improve AWSV6 test eval

carlini · Feb 23, 2024 · 13f9264 · 13f9264
1 parent f080b97
commit 13f9264
Showing 1 changed file with 6 additions and 2 deletions.
diff --git a/tests/aws_ipv6.py b/tests/aws_ipv6.py
@@ -13,9 +13,13 @@
         )
 """
 
-answer = 'Ipv6Ranges'
+evaluation = """
+Below is a software engineer's answer to a request to find an issue with the following code: <A>
 
-TestAWSV6 = question >> LLMRun() >>  Echo()  >> SubstringEvaluator(answer)
+Does the engineer mention overly permissive IPv6 ranges? Think out loud, and then end your answer with either the string "The issue is found" or "The issue is not found".
+"""
+
+TestAWSV6 = question >> LLMRun() >> LLMRun(evaluation, llm=EVAL_LLM) >> SubstringEvaluator("The issue is found")
 
 if __name__ == "__main__":
     print(run_test(TestAWSV6))