diff --git a/assets/ImageInputAssistant.png b/assets/ImageInputAssistant.png new file mode 100644 index 0000000..1dc4a9e Binary files /dev/null and b/assets/ImageInputAssistant.png differ diff --git a/samples/ImageInput/README.md b/samples/ImageInput/README.md new file mode 100644 index 0000000..3c30418 --- /dev/null +++ b/samples/ImageInput/README.md @@ -0,0 +1,31 @@ +# Sample Application using Azure OpenAI Assistants with Image Input Support (Python) + +This sample includes a simple Python [Quart](https://quart.palletsprojects.com/en/latest/) app that streams responses from OpenAI Assistant to an HTML/JS frontend using Server-Sent Events (SSEs). The application supports both image (.jpg/jpeg, .webp, .gif, .png) and text inputs. + +The sample is designed for use with [Docker containers](https://www.docker.com/), both for local development and Azure deployment. For Azure deployment to [Azure Container Apps](https://learn.microsoft.com/azure/container-apps/overview), please use this [template](https://github.com/Azure-Samples/openai-chat-app-quickstart) and replace the `src` folder content with this application. + +## Local development with Docker + +This sample includes a `docker-compose.yaml` for local development which creates a volume for the app code. That allows you to make changes to the code and see them instantly. + +1. Install [Docker Desktop](https://www.docker.com/products/docker-desktop/). If you opened this inside Github Codespaces or a Dev Container in VS Code, installation is not needed. ⚠️ If you're on an Apple M1/M2, you won't be able to run `docker` commands inside a Dev Container; either use Codespaces or do not open the Dev Container. + +2. Make sure that the `.env` file exists. + +3. Store keys and endpoint information (Azure) for the OpenAI resource in the `.env` file. The key should be stored in the `.env` file as `AZURE_OPENAI_API_KEY or OPENAI_API_KEY`. This is necessary because Docker containers don't have access to your user Azure credentials. + +4. Start the services with this command: + + ```shell + docker-compose up --build + ``` + +5. Click 'http://localhost:50505' in the browser to run the application. + +## Example run + +![image-input-screenshot](../../assets/ImageInputAssistant.png) + +## Deployment to Azure + +As mentioned earlier, please integrate this app using [template](https://github.com/Azure-Samples/openai-chat-app-quickstart) and following the Azure Container App deployment steps there. \ No newline at end of file diff --git a/samples/ImageInput/docker-compose.yaml b/samples/ImageInput/docker-compose.yaml new file mode 100644 index 0000000..0ba1933 --- /dev/null +++ b/samples/ImageInput/docker-compose.yaml @@ -0,0 +1,10 @@ +services: + app: + build: + context: ./src + env_file: + - .env + ports: + - 50505:50505 + volumes: + - ./src:/code diff --git a/samples/ImageInput/src/.dockerignore b/samples/ImageInput/src/.dockerignore new file mode 100644 index 0000000..24a736d --- /dev/null +++ b/samples/ImageInput/src/.dockerignore @@ -0,0 +1,3 @@ +.git* +.venv/ +**/*.pyc diff --git a/samples/ImageInput/src/Dockerfile b/samples/ImageInput/src/Dockerfile new file mode 100644 index 0000000..697d21d --- /dev/null +++ b/samples/ImageInput/src/Dockerfile @@ -0,0 +1,35 @@ +# ------------------- Stage 0: Base Stage ------------------------------ +FROM python:3.11-alpine AS base + +WORKDIR /code + +# Install tini, a tiny init for containers +RUN apk add --update --no-cache tini + +# Install required packages for cryptography package +# https://cryptography.io/en/latest/installation/#building-cryptography-on-linux +RUN apk add gcc musl-dev python3-dev libffi-dev openssl-dev cargo pkgconfig + +# ------------------- Stage 1: Build Stage ------------------------------ +FROM base AS build + +COPY requirements.txt . + +RUN pip3 install -r requirements.txt + +COPY . . + +# ------------------- Stage 2: Final Stage ------------------------------ +FROM base AS final + +RUN addgroup -S app && adduser -S app -G app + +COPY --from=build --chown=app:app /usr/local/lib/python3.11 /usr/local/lib/python3.11 +COPY --from=build --chown=app:app /usr/local/bin /usr/local/bin +COPY --from=build --chown=app:app /code /code + +USER app + +EXPOSE 50505 + +ENTRYPOINT ["tini", "gunicorn", "quartapp:create_app()"] diff --git a/samples/ImageInput/src/__init__.py b/samples/ImageInput/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/samples/ImageInput/src/config/image_input_assistant_config.yaml b/samples/ImageInput/src/config/image_input_assistant_config.yaml new file mode 100644 index 0000000..9351557 --- /dev/null +++ b/samples/ImageInput/src/config/image_input_assistant_config.yaml @@ -0,0 +1,19 @@ +name: image_input +instructions: You are a helpful assistant capable of answering questions. +model: gpt-4-turbo-2024-04-09 +assistant_id: +file_references: [] +tool_resources: + code_interpreter: + files: {} + file_search: + vector_stores: [] +functions: [] +file_search: false +code_interpreter: false +output_folder_path: /code/output +ai_client_type: OPEN_AI +assistant_type: assistant +completion_settings: null +assistant_role: user +config_folder: null diff --git a/samples/ImageInput/src/gunicorn.conf.py b/samples/ImageInput/src/gunicorn.conf.py new file mode 100644 index 0000000..614fc2c --- /dev/null +++ b/samples/ImageInput/src/gunicorn.conf.py @@ -0,0 +1,20 @@ +import multiprocessing +import os + +from dotenv import load_dotenv + +load_dotenv() + +max_requests = 1000 +max_requests_jitter = 50 +log_file = "-" +bind = "0.0.0.0:50505" + +if not os.getenv("RUNNING_IN_PRODUCTION"): + reload = True + +num_cpus = multiprocessing.cpu_count() +workers = 1 #(num_cpus * 2) + 1 +worker_class = "uvicorn.workers.UvicornWorker" + +timeout = 120 diff --git a/samples/ImageInput/src/pyproject.toml b/samples/ImageInput/src/pyproject.toml new file mode 100644 index 0000000..c197473 --- /dev/null +++ b/samples/ImageInput/src/pyproject.toml @@ -0,0 +1,20 @@ +[project] +name = "quartapp" +version = "1.0.0" +description = "Create a simple chat app using Quart and OpenAI" +dependencies = [ + "quart", + "werkzeug", + "gunicorn", + "uvicorn[standard]", + "openai", + "azure-identity", + "aiohttp", + "python-dotenv", + "pyyaml", + "azure-ai-assistant" + ] + +[build-system] +requires = ["flit_core<4"] +build-backend = "flit_core.buildapi" diff --git a/samples/ImageInput/src/quartapp/__init__.py b/samples/ImageInput/src/quartapp/__init__.py new file mode 100644 index 0000000..c5b2957 --- /dev/null +++ b/samples/ImageInput/src/quartapp/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) Microsoft. All rights reserved. +# Licensed under the MIT license. See LICENSE.md file in the project root for full license information. + +import logging +import os + +from quart import Quart + + +def create_app(): + if os.getenv("RUNNING_IN_PRODUCTION"): + logging.basicConfig(level=logging.INFO) + else: + logging.basicConfig(level=logging.DEBUG) + + app = Quart(__name__) + + from . import chat # noqa + + app.register_blueprint(chat.bp) + + return app diff --git a/samples/ImageInput/src/quartapp/chat.py b/samples/ImageInput/src/quartapp/chat.py new file mode 100644 index 0000000..cc8f94e --- /dev/null +++ b/samples/ImageInput/src/quartapp/chat.py @@ -0,0 +1,240 @@ +# Copyright (c) Microsoft. All rights reserved. +# Licensed under the MIT license. See LICENSE.md file in the project root for full license information. + +from azure.ai.assistant.management.async_assistant_client import AsyncAssistantClient +from azure.ai.assistant.management.ai_client_factory import AsyncAIClientType +from azure.ai.assistant.management.async_assistant_client_callbacks import AsyncAssistantClientCallbacks +from azure.ai.assistant.management.async_conversation_thread_client import AsyncConversationThreadClient +from azure.ai.assistant.management.async_message import AsyncConversationMessage +from azure.ai.assistant.management.assistant_config_manager import AssistantConfigManager +from azure.ai.assistant.management.attachment import Attachment, AttachmentType +import azure.identity.aio + +from quart import Blueprint, jsonify, request, Response, render_template, current_app + +import asyncio +import json, os, tempfile, time, random + + +bp = Blueprint("chat", __name__, template_folder="templates", static_folder="static") + +user_queues = {} + +class MyAssistantClientCallbacks(AsyncAssistantClientCallbacks): + def __init__(self, message_queue): + super().__init__() + self.message_queue = message_queue + + async def on_run_update(self, assistant_name, run_identifier, run_status, thread_name, is_first_message=False, message : AsyncConversationMessage = None): + if run_status == "streaming": + text_message_content = message.text_message.content + current_app.logger.info(f"Stream message: {text_message_content}") + await self.message_queue.put(("message", text_message_content)) + elif run_status == "completed": + current_app.logger.info("run status completed") + text_message = message.text_message + current_app.logger.info(f"message.text_message.content: {text_message.content}") + await self.message_queue.put(("completed_message", text_message.content)) + + async def on_run_end(self, assistant_name, run_identifier, run_end_time, thread_name, response=None): + await self.message_queue.put(("stream_end", "")) + + async def on_function_call_processed(self, assistant_name, run_identifier, function_name, arguments, response): + #await self.message_queue.put(("function", function_name)) + pass + +async def read_config(assistant_name): + config_path = f"config/{assistant_name}_assistant_config.yaml" + try: + # Attempt to read the configuration file + current_app.logger.info(f"Reading assistant configuration from {config_path}") + with open(config_path, "r") as file: + content = file.read() + return content + except FileNotFoundError as e: + current_app.logger.error(f"Configuration file not found at {config_path}: {e}") + return None + except Exception as e: + current_app.logger.error(f"An error occurred: {e}") + return None + +@bp.before_app_serving +async def configure_assistant_client(): + config = await read_config("image_input") + client_args = {} + if config: + if os.getenv("OPENAI_API_KEY"): + current_app.logger.info("Using OpenAI API key") + client_args["api_key"] = os.getenv("OPENAI_API_KEY") + else: + os.environ['AZURE_OPENAI_API_VERSION'] = '2024-05-01-preview' + if os.getenv("AZURE_OPENAI_API_KEY"): + # Authenticate using an Azure OpenAI API key + # This is generally discouraged, but is provided for developers + # that want to develop locally inside the Docker container. + current_app.logger.info("Using Azure OpenAI with key") + client_args["api_key"] = os.getenv("AZURE_OPENAI_API_KEY") + else: + if client_id := os.getenv("AZURE_OPENAI_CLIENT_ID"): + # Authenticate using a user-assigned managed identity on Azure + # See aca.bicep for value of AZURE_OPENAI_CLIENT_ID + current_app.logger.info( + "Using Azure OpenAI with managed identity for client ID %s", + client_id, + ) + default_credential = azure.identity.aio.ManagedIdentityCredential(client_id=client_id) + else: + # Authenticate using the default Azure credential chain + # See https://docs.microsoft.com/azure/developer/python/azure-sdk-authenticate#defaultazurecredential + # This will *not* work inside a Docker container. + current_app.logger.info("Using Azure OpenAI with default credential") + default_credential = azure.identity.aio.DefaultAzureCredential( + exclude_shared_token_cache_credential=True + ) + client_args["azure_ad_token_provider"] = azure.identity.aio.get_bearer_token_provider( + default_credential, "https://cognitiveservices.azure.com/.default" + ) + + # Create an instance of the AssistantConfigManager to save the updated assistant configuration to config folder once assistant client is initialized + assistant_config_manager = AssistantConfigManager.get_instance('config') + + # Create a new message queue for this session + message_queue = asyncio.Queue() + + # Initialize callbacks with the created message queue + callbacks = MyAssistantClientCallbacks(message_queue) + + api_version = os.getenv("AZURE_OPENAI_API_VERSION") + current_app.logger.info(f"Initializing AsyncAssistantClient with callbacks, api_version: {api_version}") + + bp.assistant_client = await AsyncAssistantClient.from_yaml(config, callbacks=callbacks, **client_args) + + current_app.logger.info(f"Assistant client id: {bp.assistant_client.assistant_config.assistant_id}") + current_app.logger.info(f"Assistant tool resources: {bp.assistant_client.assistant_config.tool_resources.to_dict()}") + + # Save the assistant configuration to the config folder + assistant_config_manager.save_config(bp.assistant_client.name) + + ai_client_type = AsyncAIClientType[bp.assistant_client.assistant_config.ai_client_type] + bp.conversation_thread_client = AsyncConversationThreadClient.get_instance(ai_client_type) + + # Create a new conversation thread and store its name + bp.thread_name = await bp.conversation_thread_client.create_conversation_thread() + current_app.logger.info(f"Conversation thread created with name: {bp.thread_name}") + + # Store the message queue for this thread name in the global dictionary + user_queues[bp.thread_name] = message_queue + else: + current_app.logger.error("Assistant configuration not found") + raise FileNotFoundError("Assistant configuration not found") + +@bp.after_app_serving +async def shutdown_assistant_client(): + if hasattr(bp, 'conversation_thread_client'): + await bp.conversation_thread_client.close() + current_app.logger.info("AsyncChatAssistantClient has been closed") + +@bp.get("/") +async def index(): + return await render_template("index.html") + +async def generate_unique_filename(base_name): + name, ext = os.path.splitext(base_name) + unique_name = f"{name}_{int(time.time())}_{random.randint(1000, 9999)}{ext}" + current_app.logger.info(f"Generated unique filename: {unique_name}") + return unique_name + +@bp.post("/chat") +async def start_chat(): + user_message = await request.form + user_files = await request.files + + attachments = [] + temp_dir = tempfile.gettempdir() + for key in user_files.keys(): + file_name = await generate_unique_filename(key) + temp_file_path = os.path.join(temp_dir, file_name) + await user_files[key].save(temp_file_path) + attachments.append(Attachment.from_dict({ + "file_name": os.path.basename(temp_file_path), + "file_path": temp_file_path, + "attachment_type": AttachmentType.IMAGE_FILE, + "tools": [], + })) + current_app.logger.info(f"Attachments: {attachments}") + timeout = 90.0 + + if not hasattr(bp, 'assistant_client'): + return jsonify({"error": "Assistant client is not initialized"}), 500 + + if not hasattr(bp, 'thread_name'): + return jsonify({"error": "Conversation thread is not initialized"}), 500 + + # Send user message to the conversation thread + if len(attachments) > 0: + await bp.conversation_thread_client.create_conversation_thread_message(user_message['message'], bp.thread_name, attachments=attachments, timeout=timeout) + else: + await bp.conversation_thread_client.create_conversation_thread_message(user_message['message'], bp.thread_name, timeout=timeout) + # Process messages in the background, do not await here + asyncio.create_task( + bp.assistant_client.process_messages(thread_name=bp.thread_name, stream=True) + ) + + return jsonify({"thread_name": bp.thread_name, "message": "Processing started"}), 200 + +def read_file(path): + with open(path, 'r') as file: + return file.read() + +@bp.route('/stream/', methods=['GET']) +async def stream_responses(thread_name): + # Set necessary headers for SSE + headers = { + 'Cache-Control': 'no-cache', + 'Connection': 'keep-alive', + 'Content-Type': 'text/event-stream' + } + + current_app.logger.info(f"Stream request received for thread: {thread_name}") + + if thread_name != bp.thread_name: + current_app.logger.error(f"Invalid thread name: {thread_name} does not match {bp.thread_name}") + return jsonify({"error": "Invalid thread name"}), 404 + + message_queue = user_queues.get(thread_name) + if not message_queue: + current_app.logger.error(f"No active session found for thread: {thread_name}") + return jsonify({"error": "No active session for this thread"}), 404 + + current_app.logger.info(f"Starting to stream events for thread: {thread_name}") + + async def event_stream(): + try: + while True: + message_type, message = await message_queue.get() + + if message_type == "message": + event_data = json.dumps({'content': message, 'type': message_type}) + yield f"data: {event_data}\n\n" + elif message_type == "completed_message": + event_data = json.dumps({'content': message, 'type': message_type}) + yield f"data: {event_data}\n\n" + elif message_type == "stream_end": + event_data = json.dumps({'content': message, 'type': message_type}) + yield f"data: {event_data}\n\n" + return + elif message_type == "function": + function_message = f"Function {message} called" + event_data = json.dumps({'content': function_message}) + yield f"data: {event_data}\n\n" + + message_queue.task_done() + + except asyncio.CancelledError: + raise + except Exception as e: + raise + finally: + pass + + return Response(event_stream(), headers=headers) \ No newline at end of file diff --git a/samples/ImageInput/src/quartapp/static/ChatClient.js b/samples/ImageInput/src/quartapp/static/ChatClient.js new file mode 100644 index 0000000..a976d36 --- /dev/null +++ b/samples/ImageInput/src/quartapp/static/ChatClient.js @@ -0,0 +1,98 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE.md file in the project root for full license information. + +class ChatClient { + constructor(ui) { + this.ui = ui; + this.messageInput = document.getElementById("message"); + this.fileInput = document.getElementById("file"); + this.eventSource = null; + } + + async sendMessage(url) { + const message = this.messageInput.value.trim(); + const files = this.fileInput.files; + + if (!message) return false; + + if (files.length > 0) { + this.ui.appendUserMessage(message, files); + } else { + this.ui.appendUserMessage(message); + } + + const formData = new FormData(); + formData.append("message", message); + for (const [i, file] of Array.from(files).entries()) { + if (file.type == "image/jpeg" || file.type == "image/png" || file.type == "image/gif" || file.type == "image/webp") { + formData.append(`${i}_${file.name}`, file); + } else { + console.error("Unsupported file type") + } + } + + const response = await fetch(url, { + method: "POST", + body: formData, + }); + + const data = await response.json(); + return data.thread_name; + } + + listenToServer(url, threadName) { + if (!this.eventSource || this.eventSource.readyState === EventSource.CLOSED) { + this.eventSource = new EventSource(`${url}/${threadName}`); + this.handleMessages(); + } + } + + handleMessages() { + let messageDiv = null; + let accumulatedContent = ''; + let isStreaming = true; + + this.eventSource.onmessage = event => { + const data = JSON.parse(event.data); + + if (data.type === "stream_end") { + this.eventSource.close(); + messageDiv = null; + accumulatedContent = ''; + } else { + if (!messageDiv) { + messageDiv = this.ui.createAssistantMessageDiv(); + if (!messageDiv) { + console.error("Failed to create message div."); + } + } + + // Check if it's a completed message + if (data.type === "completed_message") { + //console.log("Received completed message:", data.content); + // Replace the accumulated content with the completed message + this.ui.clearAssistantMessage(messageDiv); + accumulatedContent = data.content; + isStreaming = false; + } else { + //console.log("Received partial message:", data.content); + // Append the partial message to the accumulated content + accumulatedContent += data.content; + } + + this.ui.appendAssistantMessage(messageDiv, accumulatedContent, isStreaming); + } + }; + + this.eventSource.onerror = error => { + console.error("EventSource failed:", error); + this.eventSource.close(); + }; + } + + closeEventSource() { + if (this.eventSource) this.eventSource.close(); + } +} + +export default ChatClient; diff --git a/samples/ImageInput/src/quartapp/static/ChatUI.js b/samples/ImageInput/src/quartapp/static/ChatUI.js new file mode 100644 index 0000000..5b14739 --- /dev/null +++ b/samples/ImageInput/src/quartapp/static/ChatUI.js @@ -0,0 +1,119 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE.md file in the project root for full license information. + +class ChatUI { + constructor() { + this.targetContainer = document.getElementById("messages"); + this.userTemplate = document.querySelector('#message-template-user'); + this.assistantTemplate = document.querySelector('#message-template-assistant'); + if (!this.assistantTemplate) { + console.error("Assistant template not found!"); + } + } + + appendUserMessage(message, imageFiles = null) { + const userTemplateClone = this.userTemplate.content.cloneNode(true); + userTemplateClone.querySelector(".message-content").textContent = message; + + if (imageFiles) { + userTemplateClone.querySelector(".message-content").innerHTML += "

"; + for (const imageFile of imageFiles){ + if (imageFile.type == "image/jpeg" || imageFile.type == "image/png" || imageFile.type == "image/gif" || imageFile.type == "image/webp") { + userTemplateClone.querySelector(".message-content").innerHTML += ``; + } else { + console.error("Unsupported file type") + } + } + } + + this.targetContainer.appendChild(userTemplateClone); + this.scrollToBottom(); + } + + appendAssistantMessage(messageDiv, accumulatedContent, isStreaming) { + //console.log("Accumulated Content before conversion:", accumulatedContent); + const md = window.markdownit({ + html: true, + linkify: true, + typographer: true, + breaks: true + }); + + try { + // Convert the accumulated content to HTML using markdown-it + let htmlContent = md.render(accumulatedContent); + const messageTextDiv = messageDiv.querySelector(".message-text"); + if (!messageTextDiv) { + throw new Error("Message content div not found in the template."); + } + + // Set the innerHTML of the message text div to the HTML content + messageTextDiv.innerHTML = htmlContent; + + // Use requestAnimationFrame to ensure the DOM has updated before scrolling + // Only scroll if not streaming + if (!isStreaming) { + console.log("Accumulated content:", accumulatedContent); + console.log("HTML set to messageTextDiv:", messageTextDiv.innerHTML); + requestAnimationFrame(() => { + this.scrollToBottom(); + }); + } + } catch (error) { + console.error("Error in appendAssistantMessage:", error); + } + } + + clearAssistantMessage(messageDiv) { + const messageTextDiv = messageDiv.querySelector(".message-text"); + if (messageTextDiv) { + messageTextDiv.innerHTML = ''; + } + } + + createAssistantMessageDiv() { + const assistantTemplateClone = this.assistantTemplate.content.cloneNode(true); + if (!assistantTemplateClone) { + console.error("Failed to clone assistant template."); + return null; + } + + // Append the clone to the target container + this.targetContainer.appendChild(assistantTemplateClone); + + // Since the content of assistantTemplateClone is now transferred to the DOM, + // you should query the targetContainer for the elements you want to interact with. + // Specifically, you look at the last added 'toast' which is where the new content lives. + const newlyAddedToast = this.targetContainer.querySelector(".toast-container:last-child .toast:last-child"); + + if (!newlyAddedToast) { + console.error("Failed to find the newly added toast element."); + return null; + } + + // Now, find the .message-content within this newly added toast + const messageDiv = newlyAddedToast.querySelector(".message-content"); + + if (!messageDiv) { + console.error("Message content div not found in the template."); + } + + return messageDiv; + } + + scrollToBottom() { + const lastChild = this.targetContainer.lastElementChild; + if (lastChild) { + // Adjust the scroll to make sure the input box is visible + lastChild.scrollIntoView({ behavior: 'smooth', block: 'end' }); + } + + // Ensure the input box remains visible + const inputBox = document.querySelector('#chat-area'); + if (inputBox) { + inputBox.scrollIntoView({ behavior: 'smooth', block: 'end' }); + } + } +} + +export default ChatUI; diff --git a/samples/ImageInput/src/quartapp/static/main.js b/samples/ImageInput/src/quartapp/static/main.js new file mode 100644 index 0000000..c161dab --- /dev/null +++ b/samples/ImageInput/src/quartapp/static/main.js @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE.md file in the project root for full license information. + +import ChatUI from './ChatUI.js'; +import ChatClient from './ChatClient.js'; + +function initChat() { + const chatUI = new ChatUI(); + const chatClient = new ChatClient(chatUI); + + const form = document.getElementById("chat-form"); + + form.addEventListener("submit", async function(e) { + e.preventDefault(); + const threadName = await chatClient.sendMessage("/chat"); + if (threadName) { + chatClient.listenToServer("/stream", threadName); + } + chatClient.messageInput.value = ""; + }); + + window.onbeforeunload = function() { + chatClient.closeEventSource(); + }; +} + +document.addEventListener("DOMContentLoaded", initChat); diff --git a/samples/ImageInput/src/quartapp/static/styles.css b/samples/ImageInput/src/quartapp/static/styles.css new file mode 100644 index 0000000..8f07262 --- /dev/null +++ b/samples/ImageInput/src/quartapp/static/styles.css @@ -0,0 +1,137 @@ +/* Copyright (c) Microsoft. All rights reserved. + Licensed under the MIT license. See LICENSE.md file in the project root for full license information. */ + +* { + box-sizing: border-box; +} + +html, body { + height: 100%; + margin: 0; + padding: 0; + overflow-x: hidden; /* Prevent horizontal scroll */ +} + +.row { + height: 100%; +} + +/* Chat section */ +#messages { + height: calc(100% - 60px); /* Adjust height based on the input area */ + overflow-y: auto; /* Enable scrolling for overflow content */ +} + +#messages .toast-container { + margin-bottom: 12px; +} + +/* Styles for the message box */ +#messages .message-content { + /* Other styles... */ + font-size: 16px; + font-family: Arial, sans-serif; /* Example font */ +} + +#messages .message-content ol { + padding-left: 20px; + list-style-type: decimal; /* Ensures numbered lists are displayed correctly */ +} + +#messages .message-content ol li { + margin-bottom: 5px; +} + +/* Ensure consistent font size and styling for message text */ +.message-text { + font-size: 16px; + font-family: Arial, sans-serif; +} + +.message-text h1, +.message-text h2, +.message-text h3, +.message-text h4, +.message-text h5, +.message-text h6, +.message-text p, +.message-text span, +.message-text div { + font-size: 16px; + font-family: Arial, sans-serif; + margin: 0; /* Reset margin to avoid extra spacing */ + line-height: 1.5; /* Ensure consistent line height */ +} + +/* Optional: Adjust font weight for headers to distinguish them without changing size */ +.message-text h1, +.message-text h2, +.message-text h3, +.message-text h4, +.message-text h5, +.message-text h6 { + font-weight: bold; +} + +#chat-area { + height: 60px; /* Fixed height for the chat input area */ + padding: 10px; /* Padding for the input area */ +} + +/* Ensure Flexbox is applied to parent and children elements */ +.container-fluid { + display: flex; + flex-direction: row; + height: 100%; +} + +#chat-container { + flex: 1; + display: flex; + flex-direction: column; + height: 100%; +} + +.col-full { + flex: 0 0 100%; + max-width: 100%; +} + +.col-half { + flex: 0 0 50%; + max-width: 50%; +} + +.hidden { + display: none; +} + +/* Background colors for user and assistant messages */ +.background-user { + background-color: #2372cc; + color: white; /* Ensure text is readable on the background */ +} + +.background-assistant { + background-color: #2c8310; + color: white; /* Ensure text is readable on the background */ +} + +/* Styling for messages */ +.toast { + position: relative; + display: block; + margin-bottom: 0.5rem; + border-radius: 0.25rem; +} + +.toast-header { + display: flex; + align-items: center; + padding: 0.5rem 0.75rem; + color: #ffffff; +} + +.toast-body { + padding: 0.75rem; +} \ No newline at end of file diff --git a/samples/ImageInput/src/quartapp/templates/index.html b/samples/ImageInput/src/quartapp/templates/index.html new file mode 100644 index 0000000..0c3b36f --- /dev/null +++ b/samples/ImageInput/src/quartapp/templates/index.html @@ -0,0 +1,73 @@ + + + + + + + OpenAI ChatGPT Demo + + + + + + +
+
+ +
+
+ +
+ +
+
+
+ + + + + + +
+
+
+
+
+
+ + + + + + + + + + diff --git a/samples/ImageInput/src/requirements.txt b/samples/ImageInput/src/requirements.txt new file mode 100644 index 0000000..c2bbb05 --- /dev/null +++ b/samples/ImageInput/src/requirements.txt @@ -0,0 +1,175 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --output-file=requirements.txt pyproject.toml +# +aiofiles==23.2.1 + # via quart +aiohttp==3.9.3 + # via quartapp (pyproject.toml) +aiosignal==1.3.1 + # via aiohttp +annotated-types==0.6.0 + # via pydantic +anyio==4.3.0 + # via + # httpx + # openai + # watchfiles +attrs==23.2.0 + # via aiohttp +azure-core==1.30.1 + # via azure-identity +azure-identity==1.15.0 + # via quartapp (pyproject.toml) +blinker==1.7.0 + # via + # flask + # quart +certifi==2024.2.2 + # via + # httpcore + # httpx + # requests +cffi==1.16.0 + # via cryptography +charset-normalizer==3.3.2 + # via requests +click==8.1.7 + # via + # flask + # quart + # uvicorn +cryptography==42.0.5 + # via + # azure-identity + # msal + # pyjwt +distro==1.9.0 + # via openai +flask==3.0.3 + # via quart +frozenlist==1.4.1 + # via + # aiohttp + # aiosignal +gunicorn==21.2.0 + # via quartapp (pyproject.toml) +h11==0.14.0 + # via + # httpcore + # hypercorn + # uvicorn + # wsproto +h2==4.1.0 + # via hypercorn +hpack==4.0.0 + # via h2 +httpcore==1.0.5 + # via httpx +httptools==0.6.1 + # via uvicorn +httpx==0.27.0 + # via openai +hypercorn==0.16.0 + # via quart +hyperframe==6.0.1 + # via h2 +idna==3.7 + # via + # anyio + # httpx + # requests + # yarl +itsdangerous==2.1.2 + # via + # flask + # quart +jinja2==3.1.3 + # via + # flask + # quart +markupsafe==2.1.5 + # via + # jinja2 + # quart + # werkzeug +msal==1.28.0 + # via + # azure-identity + # msal-extensions +msal-extensions==1.1.0 + # via azure-identity +multidict==6.0.5 + # via + # aiohttp + # yarl +openai==1.30.1 + # via quartapp (pyproject.toml) +packaging==24.0 + # via + # gunicorn + # msal-extensions +portalocker==2.8.2 + # via msal-extensions +priority==2.0.0 + # via hypercorn +pycparser==2.22 + # via cffi +pydantic==2.6.4 + # via openai +pydantic-core==2.16.3 + # via pydantic +pyjwt[crypto]==2.8.0 + # via msal +python-dotenv==1.0.1 + # via + # quartapp (pyproject.toml) + # uvicorn +pyyaml==6.0.1 + # via + # quartapp (pyproject.toml) + # uvicorn +quart==0.19.5 + # via quartapp (pyproject.toml) +requests==2.31.0 + # via + # azure-core + # msal +six==1.16.0 + # via azure-core +sniffio==1.3.1 + # via + # anyio + # httpx + # openai +tqdm==4.66.2 + # via openai +typing-extensions==4.11.0 + # via + # azure-core + # openai + # pydantic + # pydantic-core +urllib3==2.2.1 + # via requests +uvicorn[standard]==0.29.0 + # via quartapp (pyproject.toml) +#uvloop==0.19.0 + # via uvicorn +watchfiles==0.21.0 + # via uvicorn +websockets==12.0 + # via uvicorn +werkzeug==3.0.2 + # via + # flask + # quart + # quartapp (pyproject.toml) +wsproto==1.2.0 + # via hypercorn +yarl==1.9.4 + # via aiohttp +https://github.com/Azure-Samples/azureai-assistant-tool/releases/download/v0.4.1-alpha/azure_ai_assistant-0.4.1a1-py3-none-any.whl + # via quartapp (pyproject.toml) diff --git a/sdk/azure-ai-assistant/azure/ai/assistant/_version.py b/sdk/azure-ai-assistant/azure/ai/assistant/_version.py index 68390b5..57dfec6 100644 --- a/sdk/azure-ai-assistant/azure/ai/assistant/_version.py +++ b/sdk/azure-ai-assistant/azure/ai/assistant/_version.py @@ -6,4 +6,4 @@ # Changes may cause incorrect behavior and will be lost if the code is regenerated. # -------------------------------------------------------------------------- -VERSION = "0.4.1a1" +VERSION = "0.4.2a1" diff --git a/sdk/azure-ai-assistant/azure/ai/assistant/management/async_conversation_thread_client.py b/sdk/azure-ai-assistant/azure/ai/assistant/management/async_conversation_thread_client.py index b873d25..8599555 100644 --- a/sdk/azure-ai-assistant/azure/ai/assistant/management/async_conversation_thread_client.py +++ b/sdk/azure-ai-assistant/azure/ai/assistant/management/async_conversation_thread_client.py @@ -256,18 +256,15 @@ async def create_conversation_thread_message( }) if image_attachments: - # Retrieve the conversation to check if the image file is already included - conversation = await self.retrieve_conversation(thread_name) for image_attachment in image_attachments: - # if image attachment is not already in the conversation, add it - if not conversation.contains_image_file_id(image_attachment.file_id): - content.append({ - "type": "image_file", - "image_file": { - "file_id": image_attachment.file_id, - "detail": "high" - } - }) + # add image attachment to message + content.append({ + "type": "image_file", + "image_file": { + "file_id": image_attachment.file_id, + "detail": "high" + } + }) if attachments: # Create the message with the attachments @@ -298,12 +295,14 @@ async def _update_message_attachments(self, thread_id: str, new_attachments: Lis try: existing_attachments = self._thread_config.get_attachments_of_thread(thread_id) existing_attachments_by_id = {att.file_id: att for att in existing_attachments if att.file_id} - new_file_ids = {att.file_id for att in new_attachments if att.file_id} - attachments_to_remove = [att for att in existing_attachments if att.file_id not in new_file_ids] - for attachment in attachments_to_remove: - self._thread_config.remove_attachment_from_thread(thread_id, attachment.file_id) - await self._ai_client.files.delete(file_id=attachment.file_id) + #Note: removed due to openai issue with file deletion on thread; may be added back once issue is resolved + # new_file_ids = {att.file_id for att in new_attachments if att.file_id} + # attachments_to_remove = [att for att in existing_attachments if att.file_id not in new_file_ids] + + # for attachment in attachments_to_remove: + # self._thread_config.remove_attachment_from_thread(thread_id, attachment.file_id) + # await self._ai_client.files.delete(file_id=attachment.file_id) all_updated_attachments = [] image_attachments = [] diff --git a/sdk/azure-ai-assistant/azure/ai/assistant/management/conversation_thread_client.py b/sdk/azure-ai-assistant/azure/ai/assistant/management/conversation_thread_client.py index 2690241..fff4277 100644 --- a/sdk/azure-ai-assistant/azure/ai/assistant/management/conversation_thread_client.py +++ b/sdk/azure-ai-assistant/azure/ai/assistant/management/conversation_thread_client.py @@ -253,18 +253,15 @@ def create_conversation_thread_message( }) if image_attachments: - # Retrieve the conversation to check if the image file is already included - conversation = self.retrieve_conversation(thread_name) for image_attachment in image_attachments: - # if image attachment is not already in the conversation, add it - if not conversation.contains_image_file_id(image_attachment.file_id): - content.append({ - "type": "image_file", - "image_file": { - "file_id": image_attachment.file_id, - "detail": "high" - } - }) + # add image attachment to message + content.append({ + "type": "image_file", + "image_file": { + "file_id": image_attachment.file_id, + "detail": "high" + } + }) if attachments: # Create the message with the attachments @@ -295,12 +292,14 @@ def _update_message_attachments(self, thread_id: str, new_attachments: List[Atta try: existing_attachments = self._thread_config.get_attachments_of_thread(thread_id) existing_attachments_by_id = {att.file_id: att for att in existing_attachments if att.file_id} - new_file_ids = {att.file_id for att in new_attachments if att.file_id} - attachments_to_remove = [att for att in existing_attachments if att.file_id not in new_file_ids] - for attachment in attachments_to_remove: - self._thread_config.remove_attachment_from_thread(thread_id, attachment.file_id) - self._ai_client.files.delete(file_id=attachment.file_id) + #Note: removed due to openai issue with file deletion on thread; may be added back once issue is resolved + # new_file_ids = {att.file_id for att in new_attachments if att.file_id} + # attachments_to_remove = [att for att in existing_attachments if att.file_id not in new_file_ids] + + # for attachment in attachments_to_remove: + # self._thread_config.remove_attachment_from_thread(thread_id, attachment.file_id) + # self._ai_client.files.delete(file_id=attachment.file_id) all_updated_attachments = [] image_attachments = [] diff --git a/sdk/azure-ai-assistant/test/test_assistant_client.py b/sdk/azure-ai-assistant/test/test_assistant_client.py index b3e641d..cbcf450 100644 --- a/sdk/azure-ai-assistant/test/test_assistant_client.py +++ b/sdk/azure-ai-assistant/test/test_assistant_client.py @@ -265,4 +265,30 @@ def test_assistant_client_create_thread_and_process_message_with_multi_image(): assert last_message.image_messages[1].file_id is not None assert conversation.contains_image_file_id(last_message.image_messages[1].file_id) + client.purge() + +def test_assistant_client_create_thread_and_process_multi_messages_with_image(): + config = generate_test_config() + config_json = json.dumps(config) + + client = AssistantClient.from_json(config_json) + thread_client = ConversationThreadClient.get_instance(client._get_ai_client_type(config.get('ai_client_type'))) + thread_name = thread_client.create_conversation_thread() + attachment1 = Attachment(file_path=str(RESOURCES_PATH / "scenery.png"), attachment_type=AttachmentType.IMAGE_FILE) + attachment2 = Attachment(file_path=str(RESOURCES_PATH / "scenery.png"), attachment_type=AttachmentType.IMAGE_FILE) + thread_client.create_conversation_thread_message(message="What is in the picture?", thread_name=thread_name, attachments=[attachment1]) + client.process_messages(thread_name) + thread_client.create_conversation_thread_message(message="What is in the picture?", thread_name=thread_name, attachments=[attachment1, attachment2]) + client.process_messages(thread_name) + conversation = thread_client.retrieve_conversation(thread_name) + last_message = conversation.get_last_message("user") + assert len(conversation.messages) == 4 + assert last_message is not None + assert last_message.image_messages is not None + assert len(last_message.image_messages) == 2 + assert last_message.image_messages[0].file_id is not None + assert conversation.contains_image_file_id(last_message.image_messages[0].file_id) + assert last_message.image_messages[1].file_id is not None + assert conversation.contains_image_file_id(last_message.image_messages[1].file_id) + client.purge() \ No newline at end of file diff --git a/sdk/azure-ai-assistant/test/test_async_assistant_client.py b/sdk/azure-ai-assistant/test/test_async_assistant_client.py index b69706f..65c19a9 100644 --- a/sdk/azure-ai-assistant/test/test_async_assistant_client.py +++ b/sdk/azure-ai-assistant/test/test_async_assistant_client.py @@ -146,3 +146,31 @@ async def test_assistant_client_create_thread_and_process_message_with_multi_ima await client.purge() await thread_client.close() + +@pytest.mark.asyncio +async def test_assistant_client_create_thread_and_process_multi_messages_with_image(): + config = generate_test_config() + config_json = json.dumps(config) + + client = await AsyncAssistantClient.from_json(config_json) + thread_client = AsyncConversationThreadClient.get_instance(AsyncAIClientType.OPEN_AI) + thread_name = await thread_client.create_conversation_thread() + attachment1 = Attachment(file_path=str(RESOURCES_PATH / "scenery.png"), attachment_type=AttachmentType.IMAGE_FILE) + attachment2 = Attachment(file_path=str(RESOURCES_PATH / "scenery.png"), attachment_type=AttachmentType.IMAGE_FILE) + await thread_client.create_conversation_thread_message(message="What is in the picture?", thread_name=thread_name, attachments=[attachment1]) + await client.process_messages(thread_name) + await thread_client.create_conversation_thread_message(message="What is in the picture?", thread_name=thread_name, attachments=[attachment1, attachment2]) + await client.process_messages(thread_name) + conversation = await thread_client.retrieve_conversation(thread_name) + last_message = conversation.get_last_message("user") + assert len(conversation.messages) == 4 + assert last_message is not None + assert last_message.image_messages is not None + assert len(last_message.image_messages) == 2 + assert last_message.image_messages[0].file_id is not None + assert conversation.contains_image_file_id(last_message.image_messages[0].file_id) + assert last_message.image_messages[1].file_id is not None + assert conversation.contains_image_file_id(last_message.image_messages[1].file_id) + + await client.purge() + await thread_client.close()