Adding the non adversarial simulator

sdk/evaluation/azure-ai-evaluation/README.md

@@ -92,6 +92,132 @@ if __name__ == "__main__":
 
     pprint(result)
 ```
+## Simulator
+
+Sample application prompty
+
+```yaml
+---
+name: ApplicationPrompty
+description: Simulates an application
+model:
+  api: chat
+  configuration:
+    type: azure_openai
+    azure_deployment: ${env:AZURE_DEPLOYMENT}
+    api_key: ${env:AZURE_OPENAI_API_KEY}
+    azure_endpoint: ${env:AZURE_OPENAI_ENDPOINT}
+  parameters:
+    temperature: 0.0
+    top_p: 1.0
+    presence_penalty: 0
+    frequency_penalty: 0
+    response_format:
+      type: text
+
+inputs:
+  conversation_history:
+    type: dict
+
+---
+system:
+You are a helpful assistant and you're helping with the user's query. Keep the conversation engaging and interesting.
+
+Output with a string that continues the conversation, responding to the latest message from the user, given the conversation history:
+{{ conversation_history }}
+
+```
+Application code:
+
+```python
+import json
+import asyncio
+from typing import Any, Dict, List, Optional
+from azure.ai.evaluation.synthetic import Simulator
+from promptflow.client import load_flow
+from azure.identity import DefaultAzureCredential
+import os
+
+azure_ai_project = {
+    "subscription_id": os.environ.get("AZURE_SUBSCRIPTION_ID"),
+    "resource_group_name": os.environ.get("RESOURCE_GROUP"),
+    "project_name": os.environ.get("PROJECT_NAME"),
+    "credential": DefaultAzureCredential(),
+}
+
+import wikipedia
+wiki_search_term = "Leonardo da vinci"
+wiki_title = wikipedia.search(wiki_search_term)[0]
+wiki_page = wikipedia.page(wiki_title)
+text = wiki_page.summary[:1000]
+
+def method_to_invoke_application_prompty(query: str):
+    try:
+        current_dir = os.path.dirname(__file__)
+        prompty_path = os.path.join(current_dir, "application.prompty")
+        _flow = load_flow(source=prompty_path, model={
+            "configuration": azure_ai_project
+        })
+        response = _flow(
+            query=query,
+            context=context,
+            conversation_history=messages_list
+        )
+        return response
+    except:
+        print("Something went wrong invoking the prompty")
+        return "something went wrong"
+
+async def callback(
+    messages: List[Dict],
+    stream: bool = False,
+    session_state: Any = None,  # noqa: ANN401
+    context: Optional[Dict[str, Any]] = None,
+) -> dict:
+    messages_list = messages["messages"]
+    # get last message
+    latest_message = messages_list[-1]
+    query = latest_message["content"]
+    context = None
+    # call your endpoint or ai application here
+    response = method_to_invoke_application_prompty(query)
+    # we are formatting the response to follow the openAI chat protocol format
+    formatted_response = {
+        "content": response,
+        "role": "assistant",
+        "context": {
+            "citations": None,
+        },
+    }
+    messages["messages"].append(formatted_response)
+    return {"messages": messages["messages"], "stream": stream, "session_state": session_state, "context": context}
+
+
+
+async def main():
+    simulator = Simulator(azure_ai_project=azure_ai_project, credential=DefaultAzureCredential())
+    outputs = await simulator(
+        target=callback,
+        text=text,
+        num_queries=2,
+        max_conversation_turns=4,
+        user_persona=[
+            f"I am a student and I want to learn more about {wiki_search_term}",
+            f"I am a teacher and I want to teach my students about {wiki_search_term}"
+        ],
+    )
+    print(json.dumps(outputs))
+
+if __name__ == "__main__":
+    os.environ["AZURE_SUBSCRIPTION_ID"] = ""
+    os.environ["RESOURCE_GROUP"] = ""
+    os.environ["PROJECT_NAME"] = ""
+    os.environ["AZURE_OPENAI_API_KEY"] = ""
+    os.environ["AZURE_OPENAI_ENDPOINT"] = ""
+    os.environ["AZURE_DEPLOYMENT"] = ""
+    asyncio.run(main())
+    print("done!")
+```
 
 Simulator expects the user to have a callback method that invokes their AI application.
 Here's a sample of a callback which invokes AsyncAzureOpenAI:

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/synthetic/__init__.py

@@ -3,11 +3,13 @@
 from .constants import SupportedLanguages
 from .direct_attack_simulator import DirectAttackSimulator
 from .indirect_attack_simulator import IndirectAttackSimulator
+from .simulator import Simulator
 
 __all__ = [
     "AdversarialSimulator",
     "AdversarialScenario",
     "DirectAttackSimulator",
     "IndirectAttackSimulator",
+    "Simulator",
     "SupportedLanguages",
 ]

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/synthetic/_helpers/__init__.py

@@ -1,3 +1,4 @@
+from ._simulator_data_classes import ConversationHistory, Turn
 from ._language_suffix_mapping import SUPPORTED_LANGUAGES_MAPPING
 
-__all__ = ["SUPPORTED_LANGUAGES_MAPPING"]
+__all__ = ["ConversationHistory", "Turn", "SUPPORTED_LANGUAGES_MAPPING"]

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/synthetic/_helpers/_simulator_data_classes.py

@@ -0,0 +1,93 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+# pylint: disable=C0103,C0114,C0116
+from dataclasses import dataclass
+from typing import Union
+
+from azure.ai.evaluation.synthetic._conversation.constants import ConversationRole
+
+
+@dataclass
+class Turn:
+    """
+    Represents a conversation turn,
+    keeping track of the role, content,
+    and context of a turn in a conversation.
+    """
+
+    role: Union[str, ConversationRole]
+    content: str
+    context: str = None
+
+    def to_dict(self):
+        """
+        Convert the conversation turn to a dictionary.
+
+        Returns:
+            dict: A dictionary representation of the conversation turn.
+        """
+        return {
+            "role": self.role.value if isinstance(self.role, ConversationRole) else self.role,
+            "content": self.content,
+            "context": self.context,
+        }
+
+    def __repr__(self):
+        """
+        Return the string representation of the conversation turn.
+
+        Returns:
+            str: A string representation of the conversation turn.
+        """
+        return f"Turn(role={self.role}, content={self.content})"
+
+
+class ConversationHistory:
+    """
+    Conversation history class to keep track of the conversation turns in a conversation.
+    """
+
+    def __init__(self):
+        """
+        Initializes the conversation history with an empty list of turns.
+        """
+        self.history = []
+
+    def add_to_history(self, turn: Turn):
+        """
+        Adds a turn to the conversation history.
+
+        Args:
+            turn (Turn): The conversation turn to add.
+        """
+        self.history.append(turn)
+
+    def to_list(self):
+        """
+        Converts the conversation history to a list of dictionaries.
+
+        Returns:
+            list: A list of dictionaries representing the conversation turns.
+        """
+        return [turn.to_dict() for turn in self.history]
+
+    def get_length(self):
+        """
+        Returns the length of the conversation.
+
+        Returns:
+            int: The number of turns in the conversation history.
+        """
+        return len(self.history)
+
+    def __repr__(self):
+        """
+        Returns the string representation of the conversation history.
+
+        Returns:
+            str: A string representation of the conversation history.
+        """
+        for turn in self.history:
+            print(turn)
+        return ""

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/synthetic/_prompty/task_query_response.prompty

@@ -0,0 +1,69 @@
+---
+name: TaskSimulatorQueryResponse
+description: Gets queries and responses from a blob of text
+model:
+  api: chat
+  configuration:
+    type: azure_openai
+    azure_deployment: ${env:AZURE_DEPLOYMENT}
+    api_key: ${env:AZURE_OPENAI_API_KEY}
+    azure_endpoint: ${env:AZURE_OPENAI_ENDPOINT}
+  parameters:
+    temperature: 0.0
+    top_p: 1.0
+    presence_penalty: 0
+    frequency_penalty: 0
+    response_format:
+      type: json_object
+
+inputs:
+  text:
+    type: string
+  num_queries:
+    type: integer
+
+---
+system:
+You're an AI that helps in preparing a Question/Answer quiz from Text for "Who wants to be a millionaire" tv show
+Both Questions and Answers MUST BE extracted from given Text
+Frame Question in a way so that Answer is RELEVANT SHORT BITE-SIZED info from Text
+RELEVANT info could be: NUMBER, DATE, STATISTIC, MONEY, NAME
+A sentence should contribute multiple QnAs if it has more info in it
+Answer must not be more than 5 words
+Answer must be picked from Text as is
+Question should be as descriptive as possible and must include as much context as possible from Text
+Output must always have the provided number of QnAs
+Output must be in JSON format
+Text:
+<|text_start|>
+On January 24, 1984, former Apple CEO Steve Jobs introduced the first Macintosh. In late 2003, Apple had 2.06 percent of the desktop share in the United States.
+Some years later, research firms IDC and Gartner reported that Apple's market share in the U.S. had increased to about 6%.
+<|text_end|>
+Output with 5 QnAs:
+[
+  {
+    "q": "When did the former Apple CEO Steve Jobs introduced the first Macintosh?",
+    "r": "January 24, 1984"
+  },
+  {
+    "q": "Who was the former Apple CEO that introduced the first Macintosh on January 24, 1984?",
+    "r": "Steve Jobs"
+  },
+  {
+    "q": "What percent of the desktop share did Apple have in the United States in late 2003?",
+    "r": "2.06 percent"
+  },
+  {
+    "q": "What were the research firms that reported on Apple's market share in the U.S.?",
+    "r": "IDC and Gartner"
+  },
+  {
+    "q": "What was the percentage increase of Apple's market share in the U.S., as reported by research firms IDC and Gartner?",
+    "r": "6%"
+  }
+]
+Text:
+<|text_start|>
+{{ text }}
+<|text_end|>
+Output with {{ num_queries }} QnAs:

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/synthetic/_prompty/task_simulate.prompty

@@ -0,0 +1,36 @@
+---
+name: TaskSimulatorWithPersona
+description: Simulates a user to complete a conversation
+model:
+  api: chat
+  configuration:
+    type: azure_openai
+    azure_deployment: ${env:AZURE_DEPLOYMENT}
+    azure_endpoint: ${env:AZURE_OPENAI_ENDPOINT}
+  parameters:
+    temperature: 0.0
+    top_p: 1.0
+    presence_penalty: 0
+    frequency_penalty: 0
+    response_format:
+      type: json_object
+
+inputs:
+  task:
+    type: string
+  conversation_history:
+    type: dict
+
+---
+system:
+You should behave as a user who is planning to accomplish this task: {{ task }} and you continue to interact with a system that responds to your queries.
+Make sure your conversation is engaging and interactive.
+Output must be in JSON format
+Here's a sample output:
+{
+  "content": "Here is my follow-up question.",
+  "user": "user"
+}
+
+Output with a json object that continues the conversation, given the conversation history:
+{{ conversation_history }}