Add XPIA simulator and evaluator

.cspell.json

@@ -54,65 +54,66 @@
     "benchmark/promptflow-serve/result-archive/**"
   ],
   "words": [
-    "aoai",
     "amlignore",
-    "mldesigner",
-    "faiss",
-    "serp",
+    "aoai",
+    "Apim",
+    "astext",
+    "attribited",
+    "azureai",
+    "azurecr",
     "azureml",
-    "mlflow",
-    "vnet",
-    "openai",
-    "pfazure",
+    "azuremlsdktestpypi",
+    "Bhavik",
+    "centralus",
+    "chatml",
+    "cref",
+    "e2etest",
+    "e2etests",
     "eastus",
-    "azureai",
-    "vectordb",
-    "Qdrant",
-    "Weaviate",
+    "Entra",
     "env",
-    "e2etests",
-    "e2etest",
-    "tablefmt",
-    "logprobs",
-    "logit",
+    "faiss",
+    "geval",
     "hnsw",
-    "chatml",
-    "UNLCK",
+    "junit",
     "KHTML",
+    "Likert",
+    "llmlingua",
+    "logit",
+    "logprobs",
+    "meid",
+    "mgmt",
+    "MistralAI",
+    "mldesigner",
+    "mlflow",
+    "msal",
+    "msrest",
+    "myconn",
     "numlines",
-    "azurecr",
-    "centralus",
+    "nunit",
+    "openai",
+    "pfazure",
+    "pfbytes",
+    "pfcli",
+    "pfutil",
     "Policheck",
-    "azuremlsdktestpypi",
-    "rediraffe",
     "pydata",
-    "ROBOCOPY",
-    "undoc",
+    "Qdrant",
+    "rediraffe",
     "retriable",
-    "pfcli",
-    "pfutil",
-    "mgmt",
-    "wsid",
-    "westus",
-    "msrest",
-    "cref",
-    "msal",
-    "pfbytes",
-    "Apim",
-    "junit",
-    "nunit",
-    "astext",
-    "Likert",
-    "geval",
+    "ROBOCOPY",
+    "serp",
     "Summ",
-    "Bhavik",
-    "meid",
-    "Entra",
+    "tablefmt",
+    "undoc",
+    "UNLCK",
     "uvicorn",
-    "attribited",
-    "MistralAI",
-    "llmlingua",
-    "myconn"
+    "vectordb",
+    "vnet",
+    "Weaviate",
+    "westus",
+    "wsid",
+    "Xpia"
   ],
   "ignoreWords": [
     "openmpi",
@@ -242,6 +243,8 @@
     "azureopenaimodelconfiguration",
     "openaimodelconfiguration",
     "usecwd",
+    "upia",
+    "xpia",
     "locustio",
     "euap",
     "Rerank",

src/promptflow-evals/CHANGELOG.md

@@ -3,6 +3,8 @@
 ## v0.3.3 (Upcoming)
 ### Features Added
 - Add a new evaluator (ProtectedMaterialsEvaluator) and associated adversarial content simulator enum type (AdversarialScenario.ADVERSARIAL_CONTENT_PROTECTED_MATERIAL) for protected materials, which determines if given inputs contain materials protected by IP laws.
+- Introduced `IndirectAttackSimulator` to simulate XPIA (cross domain prompt injected attack) jailbreak attacks on your AI system.
+- Introduced `IndirectAttackEvaluator` to evaluate content for the presence of XPIA (cross domain prompt injected attacks) injected into conversation or Q/A context to interrupt normal expected functionality by eliciting manipulated content, intrusion and attempting to gather information outside the scope of your AI system.
 
 ### Bugs Fixed
 - Fixed evaluators to accept (non-Azure) Open AI Configs.

src/promptflow-evals/promptflow/evals/_common/constants.py

@@ -32,6 +32,7 @@ class Tasks:
 
     CONTENT_HARM = "content harm"
     PROTECTED_MATERIAL = "protected material"
+    XPIA = "xpia"
 
 
 class _InternalAnnotationTasks:
@@ -52,6 +53,7 @@ class EvaluationMetrics:
     SELF_HARM = "self_harm"
     SEXUAL = "sexual"
     PROTECTED_MATERIAL = "protected_material"
+    XPIA = "xpia"
 
 
 class _InternalEvaluationMetrics:

src/promptflow-evals/promptflow/evals/_common/rai_service.py

@@ -103,6 +103,9 @@ def generate_payload(normalized_user_text: str, metric: str) -> Dict:
     elif metric == _InternalEvaluationMetrics.ECI:
         task = _InternalAnnotationTasks.ECI
         include_metric = False
+    elif metric == EvaluationMetrics.XPIA:
+        task = Tasks.XPIA
+        include_metric = False
     return (
         {
             "UserTextList": [normalized_user_text],
@@ -208,7 +211,7 @@ def parse_response(  # pylint: disable=too-many-branches,too-many-statements
     :rtype: List[List[Dict]]
     """
 
-    if metric_name in {EvaluationMetrics.PROTECTED_MATERIAL, _InternalEvaluationMetrics.ECI}:
+    if metric_name in {EvaluationMetrics.PROTECTED_MATERIAL, _InternalEvaluationMetrics.ECI, EvaluationMetrics.XPIA}:
         if not batch_response or len(batch_response[0]) == 0 or metric_name not in batch_response[0]:
             return {}
         response = batch_response[0][metric_name]

src/promptflow-evals/promptflow/evals/evaluators/__init__.py

@@ -19,6 +19,7 @@
 from ._qa import QAEvaluator
 from ._relevance import RelevanceEvaluator
 from ._similarity import SimilarityEvaluator
+from ._xpia import IndirectAttackEvaluator
 
 __all__ = [
     "CoherenceEvaluator",
@@ -36,4 +37,5 @@
     "ContentSafetyEvaluator",
     "ContentSafetyChatEvaluator",
     "ProtectedMaterialsEvaluator",
+    "IndirectAttackEvaluator",
 ]

src/promptflow-evals/promptflow/evals/evaluators/_xpia/__init__.py

@@ -0,0 +1,5 @@
+from ._xpia import IndirectAttackEvaluator
+
+__all__ = [
+    "IndirectAttackEvaluator",
+]