mirror of
https://github.com/NCBM/plyngent.git
synced 2026-11-07 23:47:54 +08:00
core/lmproto+agent: carry image parts on user and tool messages
A user turn's `content` may now be a part array (`ChatTextPart` + `ChatImagePart`), which chat completions take as-is; the Anthropic and Responses bridges translate it into `image` blocks and `input_image` parts. A tool result keeps its text in `content` and carries `images` beside it: Anthropic and Responses put them in the result item itself (a live probe reads both back), while chat completions cannot, so `compat.chat_messages_with_images` moves them into the user turn after the tool run. Prose-only readers go through `content_to_text`, which renders an image as `[image]`. `agent.images` sniffs the real format from the bytes and builds the data URL, capped at 8 MiB inline because every stored message is re-sent on each round.
This commit is contained in:
@@ -20,10 +20,12 @@ from plyngent.agent.messages_bridge import (
|
||||
)
|
||||
from plyngent.lmproto.anthropic.model import (
|
||||
AnthropicAssistantMessage,
|
||||
AnthropicImageContent,
|
||||
AnthropicMessageResponse,
|
||||
AnthropicResponseText,
|
||||
AnthropicResponseToolUse,
|
||||
AnthropicServerToolUseContent,
|
||||
AnthropicTextContent,
|
||||
AnthropicThinkingContent,
|
||||
AnthropicToolDefinition,
|
||||
AnthropicToolResultContent,
|
||||
@@ -37,6 +39,9 @@ from plyngent.lmproto.openai_compatible.model import (
|
||||
AssistantFunctionTool,
|
||||
AssistantFunctionToolCall,
|
||||
ChatCompletionsParam,
|
||||
ChatImagePart,
|
||||
ChatImageUrl,
|
||||
ChatTextPart,
|
||||
DeveloperChatMessage,
|
||||
ReasoningEffort,
|
||||
SystemChatMessage,
|
||||
@@ -136,6 +141,57 @@ def test_merge_consecutive_tool_results() -> None:
|
||||
assert len(items[2].content) == 2
|
||||
|
||||
|
||||
def test_a_user_turn_carries_its_images_as_blocks() -> None:
|
||||
messages = [
|
||||
UserChatMessage(
|
||||
content=[
|
||||
ChatTextPart(text="what is this?"),
|
||||
ChatImagePart(image_url=ChatImageUrl(url="data:image/png;base64,AAAA", detail="low")),
|
||||
ChatImagePart(image_url=ChatImageUrl(url="https://example.com/a.jpg")),
|
||||
]
|
||||
)
|
||||
]
|
||||
_, items = chat_messages_to_anthropic(messages)
|
||||
|
||||
assert isinstance(items[0], AnthropicUserMessage)
|
||||
blocks = items[0].content
|
||||
assert isinstance(blocks, list)
|
||||
assert blocks[0] == AnthropicTextContent(text="what is this?")
|
||||
# An inline data URL becomes a base64 source carrying the sniffed media type.
|
||||
assert blocks[1] == AnthropicImageContent(source={"type": "base64", "media_type": "image/png", "data": "AAAA"})
|
||||
# A public link stays a url source.
|
||||
assert blocks[2] == AnthropicImageContent(source={"type": "url", "url": "https://example.com/a.jpg"})
|
||||
|
||||
|
||||
def test_a_tool_result_carries_its_images_inside_the_result() -> None:
|
||||
messages = [
|
||||
UserChatMessage(content="screenshot please"),
|
||||
AssistantChatMessage(
|
||||
content=None,
|
||||
tool_calls=[
|
||||
AssistantFunctionToolCall(id="c1", function=AssistantFunctionTool(name="view_image", arguments="{}"))
|
||||
],
|
||||
),
|
||||
ToolChatMessage(
|
||||
content="image: shot.png (2 KB)",
|
||||
tool_call_id="c1",
|
||||
images=[ChatImagePart(image_url=ChatImageUrl(url="data:image/png;base64,AAAA"))],
|
||||
),
|
||||
]
|
||||
_, items = chat_messages_to_anthropic(messages)
|
||||
|
||||
result = items[2]
|
||||
assert isinstance(result, AnthropicUserMessage)
|
||||
assert isinstance(result.content, list)
|
||||
block = result.content[0]
|
||||
assert isinstance(block, AnthropicToolResultContent)
|
||||
assert block.tool_use_id == "c1"
|
||||
assert block.content == [
|
||||
AnthropicTextContent(text="image: shot.png (2 KB)"),
|
||||
AnthropicImageContent(source={"type": "base64", "media_type": "image/png", "data": "AAAA"}),
|
||||
]
|
||||
|
||||
|
||||
def test_anthropic_response_to_assistant_with_tools() -> None:
|
||||
response = AnthropicMessageResponse(
|
||||
id="msg_1",
|
||||
|
||||
Reference in New Issue
Block a user