core/lmproto+agent: carry image parts on user and tool messages

A user turn's `content` may now be a part array (`ChatTextPart` +
`ChatImagePart`), which chat completions take as-is; the Anthropic and
Responses bridges translate it into `image` blocks and `input_image` parts.

A tool result keeps its text in `content` and carries `images` beside it:
Anthropic and Responses put them in the result item itself (a live probe reads
both back), while chat completions cannot, so
`compat.chat_messages_with_images` moves them into the user turn after the tool
run. Prose-only readers go through `content_to_text`, which renders an image as
`[image]`.

`agent.images` sniffs the real format from the bytes and builds the data URL,
capped at 8 MiB inline because every stored message is re-sent on each round.
This commit is contained in:
2026-10-21 18:24:23 +08:00
parent a0aba700e2
commit d7293eeb16
13 changed files with 495 additions and 19 deletions
+59
View File
@@ -19,6 +19,9 @@ from plyngent.lmproto.openai_compatible.model import (
AssistantFunctionTool,
AssistantFunctionToolCall,
ChatCompletionsParam,
ChatImagePart,
ChatImageUrl,
ChatTextPart,
DeveloperChatMessage,
SystemChatMessage,
ToolChatMessage,
@@ -76,6 +79,62 @@ def test_chat_messages_to_input_and_instructions() -> None:
assert _input_item_types(items) == ["message", "reasoning", "function_call", "function_call_output"]
def test_a_user_turn_carries_its_images_as_input_parts() -> None:
messages = [
UserChatMessage(
content=[
ChatTextPart(text="what is this?"),
ChatImagePart(image_url=ChatImageUrl(url="https://example.com/a.jpg", detail="low")),
]
)
]
_, items = chat_messages_to_responses_input(messages)
assert items[0] == ResponseEasyInputMessage(
role="user",
content=[
{"type": "input_text", "text": "what is this?"},
{"type": "input_image", "image_url": "https://example.com/a.jpg", "detail": "low"},
],
)
def test_a_tool_output_carries_the_images_the_tool_returned() -> None:
messages = [
AssistantChatMessage(
content=None,
tool_calls=[
AssistantFunctionToolCall(
id="call_1", function=AssistantFunctionTool(name="view_image", arguments="{}")
)
],
),
ToolChatMessage(
content="image: shot.png (2 KB)",
tool_call_id="call_1",
images=[ChatImagePart(image_url=ChatImageUrl(url="data:image/png;base64,AAAA"))],
),
]
_, items = chat_messages_to_responses_input(messages)
output = items[-1]
assert isinstance(output, ResponseFunctionToolCallOutput)
assert output.call_id == "call_1"
assert output.output == [
{"type": "input_text", "text": "image: shot.png (2 KB)"},
{"type": "input_image", "image_url": "data:image/png;base64,AAAA"},
]
def test_a_plain_tool_output_stays_a_string() -> None:
messages = [ToolChatMessage(content="file body", tool_call_id="call_1")]
_, items = chat_messages_to_responses_input(messages)
output = items[0]
assert isinstance(output, ResponseFunctionToolCallOutput)
assert output.output == "file body"
def test_parallel_tool_outputs_group_after_the_rounds_calls() -> None:
"""Regression: DeepSeek Responses rejected an interleaved batch.