From 059b654a5c4bb53e3c61cb5c2d1721a9beabab98 Mon Sep 17 00:00:00 2001 From: Christopher Chou Date: Sun, 15 Sep 2024 17:27:26 +0000 Subject: [PATCH] Keep same sys --- fastchat/serve/monitor/classify/category.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/fastchat/serve/monitor/classify/category.py b/fastchat/serve/monitor/classify/category.py index c20b0eeab..f1a6dc3df 100644 --- a/fastchat/serve/monitor/classify/category.py +++ b/fastchat/serve/monitor/classify/category.py @@ -138,15 +138,4 @@ def post_process(self, judgment): class CategoryVisionHardPrompt(CategoryHardPrompt): def __init__(self): super().__init__() - self.name_tag = "criteria_vision_v0.1" - self.pattern = re.compile(r"(\[\d(?:\,\s\d)*\])") - self.sys_prompt = "Your task is to evaluate how well the following input prompts can assess the capabilities of advanced AI assistants.\n\nFor the input prompt, please analyze it based on the following 7 criteria.\n1. Specificity: Does the prompt ask for a specific output, such as code, a mathematical solution, a logical simplification, a problem-solving strategy, or a hardware setup recommendation? This specificity allows the AI to demonstrate its ability to understand and generate precise responses.\n2. Domain Knowledge: Does the prompt cover a specific domain, such as programming, mathematics, logic, problem-solving, or hardware setup? Prompts spanning a range of topics test the AI's breadth of knowledge and its ability to apply that knowledge to different domains.\n3. Complexity: Does the prompt vary in complexity, from straightforward tasks to more complex, multi-step problems? This allows evaluators to assess the AI's capability to handle problems of varying difficulty.\n4. Problem-Solving Skills: Does the prompt directly involves the AI to demonstrate active problem-solving skills, such systemically coming up with a solution for a specific setup instead of regurgitating an existing fact? This tests the AI's ability to apply logical reasoning and provide practical solutions.\n5. Creativity: Does the prompt involve a level of creativity in approaching the problem? This criterion tests the AI's ability to provide tailored solutions that take into account the user's specific needs and limitations.\n6. Technical Accuracy: Does the prompt require technical accuracy in the response? This allows evaluators to assess the AI's precision and correctness in technical fields.\n7. Real-world Application: Does the prompt relate to real-world applications, such as setting up a functional system or writing code for a practical use case? This tests the AI's ability to provide practical and actionable information that could be implemented in real-life scenarios.\n\nYou must list the criteria numbers that the prompt satisfies in the format of a Python array. For example, \"[...]\". Do not explain your choice." - self.tags = { - 1: "specificity", - 2: "domain_knowledge", - 3: "complexity", - 4: "problem_solving", - 5: "creativity", - 6: "technical_accuracy", - 7: "real_world", - } \ No newline at end of file + self.name_tag = "criteria_vision_v0.1" \ No newline at end of file