core/lmproto+agent: carry image parts on user and tool messages

A user turn's `content` may now be a part array (`ChatTextPart` +
`ChatImagePart`), which chat completions take as-is; the Anthropic and
Responses bridges translate it into `image` blocks and `input_image` parts.

A tool result keeps its text in `content` and carries `images` beside it:
Anthropic and Responses put them in the result item itself (a live probe reads
both back), while chat completions cannot, so
`compat.chat_messages_with_images` moves them into the user turn after the tool
run. Prose-only readers go through `content_to_text`, which renders an image as
`[image]`.

`agent.images` sniffs the real format from the bytes and builds the data URL,
capped at 8 MiB inline because every stored message is re-sent on each round.
This commit is contained in:
2026-10-21 18:24:23 +08:00
parent a0aba700e2
commit d7293eeb16
13 changed files with 495 additions and 19 deletions
+56
View File
@@ -20,10 +20,12 @@ from plyngent.agent.messages_bridge import (
)
from plyngent.lmproto.anthropic.model import (
AnthropicAssistantMessage,
AnthropicImageContent,
AnthropicMessageResponse,
AnthropicResponseText,
AnthropicResponseToolUse,
AnthropicServerToolUseContent,
AnthropicTextContent,
AnthropicThinkingContent,
AnthropicToolDefinition,
AnthropicToolResultContent,
@@ -37,6 +39,9 @@ from plyngent.lmproto.openai_compatible.model import (
AssistantFunctionTool,
AssistantFunctionToolCall,
ChatCompletionsParam,
ChatImagePart,
ChatImageUrl,
ChatTextPart,
DeveloperChatMessage,
ReasoningEffort,
SystemChatMessage,
@@ -136,6 +141,57 @@ def test_merge_consecutive_tool_results() -> None:
assert len(items[2].content) == 2
def test_a_user_turn_carries_its_images_as_blocks() -> None:
messages = [
UserChatMessage(
content=[
ChatTextPart(text="what is this?"),
ChatImagePart(image_url=ChatImageUrl(url="data:image/png;base64,AAAA", detail="low")),
ChatImagePart(image_url=ChatImageUrl(url="https://example.com/a.jpg")),
]
)
]
_, items = chat_messages_to_anthropic(messages)
assert isinstance(items[0], AnthropicUserMessage)
blocks = items[0].content
assert isinstance(blocks, list)
assert blocks[0] == AnthropicTextContent(text="what is this?")
# An inline data URL becomes a base64 source carrying the sniffed media type.
assert blocks[1] == AnthropicImageContent(source={"type": "base64", "media_type": "image/png", "data": "AAAA"})
# A public link stays a url source.
assert blocks[2] == AnthropicImageContent(source={"type": "url", "url": "https://example.com/a.jpg"})
def test_a_tool_result_carries_its_images_inside_the_result() -> None:
messages = [
UserChatMessage(content="screenshot please"),
AssistantChatMessage(
content=None,
tool_calls=[
AssistantFunctionToolCall(id="c1", function=AssistantFunctionTool(name="view_image", arguments="{}"))
],
),
ToolChatMessage(
content="image: shot.png (2 KB)",
tool_call_id="c1",
images=[ChatImagePart(image_url=ChatImageUrl(url="data:image/png;base64,AAAA"))],
),
]
_, items = chat_messages_to_anthropic(messages)
result = items[2]
assert isinstance(result, AnthropicUserMessage)
assert isinstance(result.content, list)
block = result.content[0]
assert isinstance(block, AnthropicToolResultContent)
assert block.tool_use_id == "c1"
assert block.content == [
AnthropicTextContent(text="image: shot.png (2 KB)"),
AnthropicImageContent(source={"type": "base64", "media_type": "image/png", "data": "AAAA"}),
]
def test_anthropic_response_to_assistant_with_tools() -> None:
response = AnthropicMessageResponse(
id="msg_1",