when UserMessage have both string and Image data，JSON deserialization cause error #7170
Description
What happened?
Describe the bug
An error occurs when UserMessage.content contains both string and Image during deserialization.
To Reproduce
This is the code for reproduction.
"""
测试图片上传功能
用于验证向LLM发送图片消息的功能
"""
import asyncio
import base64
import os
from pathlib import Path
from typing import TYPE_CHECKING
from autogen_core.models import UserMessage, SystemMessage
from autogen_core import Image as AGImage
from autogen_handoffs.utils.model_utils import get_vllm_qwen3_vl_30b
def local_img_to_base64(img_path):
"""
将本地图片路径转为OpenAI API支持的base64 url格式
:param img_path: 本地图片绝对路径/相对路径
:return: data:image/*;base64,xxx 格式的字符串
"""
img_format = img_path.split(".")[-1].lower()
if img_format not in ["jpg", "jpeg", "png", "webp"]:
raise ValueError(f"不支持的图片格式：{img_format}")
# 读取并编码
with open(img_path, "rb") as f:
base64_str = base64.b64encode(f.read()).decode("utf-8")
# 返回OpenAI API要求的格式
return base64_str
def create_image_from_bytes(img_path: str) -> AGImage:
"""创建一个autogen_core.models.Image对象"""
base64_str = local_img_to_base64(img_path)
image_obj = AGImage.from_base64(base64_str)
return image_obj
async def main():
"""主测试函数"""
print("=== 开始测试多模态模型 ===\n")
img_path = "C:\\Users\\l00621014\\Desktop\\素材\\20211007104526775.jpg"
try:
# 1. 验证图片路径
print(f"图片路径: {img_path}")
if not os.path.exists(img_path):
raise FileNotFoundError(f"图片文件不存在: {img_path}")
print(f"图片文件大小: {os.path.getsize(img_path)} bytes")
# 2. 创建Image对象
print("\n=== 创建Image对象 ===")
img_obj = create_image_from_bytes(img_path)
print(f"Image对象类型: {type(img_obj)}")
print(f"Image对象URI: {img_obj.uri[:50]}..." if hasattr(img_obj, 'uri') else "没有URI属性")
# 3. 创建SystemMessage
print("\n=== 创建消息 ===")
system_message = SystemMessage(content="你是一个图像描述助手，请详细描述图片内容和场景。")
user_message = UserMessage(content=[img_obj, "请详细描述这幅图像的内容"], source="user")
json = user_message.model_dump_json()
obj = UserMessage.model_validate_json(json)
messages = [system_message, user_message]
print(f"消息数量: {len(messages)}")
print(f"消息类型: {[type(m).__name__ for m in messages]}")
# 4. 获取模型客户端
print("\n=== 获取模型客户端 ===")
model_client = get_vllm_qwen3_vl_30b()
print(f"模型类型: {type(model_client)}")
# 5. 调用模型
print("\n=== 调用模型 ===")
print("发送请求中...")
llm_result = await model_client.create(
messages=messages,
tools=[]
)
print("\n=== 测试结果 ===")
print(f"响应内容: {llm_result.content}")
print(f"思考过程: {llm_result.thought}")
except Exception as e:
print(f"\n=== 错误信息 ===")
print(f"错误类型: {type(e).__name__}")
print(f"错误消息: {e}")
import traceback
print("\n详细堆栈:")
traceback.print_exc()
if __name__ == "__main__":
# 运行测试
asyncio.run(main())
the error is Expected dict or Image instance, got <class 'str'>
I did some debugging, and I found that the problem mainly lies in the serialization definition of the Image.
@classmethod
def __get_pydantic_core_schema__(cls, source_type: Any, handler: GetCoreSchemaHandler) -> core_schema.CoreSchema:
# Custom validation
def validate(value: Any, validation_info: ValidationInfo) -> Image:
if isinstance(value, dict):
base_64 = cast(str | None, value.get("data")) # type: ignore
if base_64 is None:
raise ValueError("Expected 'data' key in the dictionary")
return cls.from_base64(base_64)
elif isinstance(value, cls):
return value
else:
raise TypeError(f"Expected dict or {cls.__name__} instance, got {type(value)}")
It seems that the string was also processed by the Image deserialization code, causing an error. This appears to be a bug.
Expected behavior
Deserialization is normal.
Which packages was the bug in?
Python Core (autogen-core)
AutoGen library version.
Python 0.6.4
Other library version.
No response
Model used
No response
Model provider
None
Other model provider
No response
Python version
3.12
.NET version
None
Operating system
Windows