core/agent: let a tool result carry images beside its text

`ToolOutput` pairs the model-facing text with the images a tool produced.
`ToolRegistry.execute` keeps its text-only contract; `execute_output` returns
the pair and the loop stores the images on the tool message, so a screenshot
tool can answer in pixels instead of a hex placeholder.

Only the text is capped by the tool-result limit; the char estimator now counts
a present image at its 1024-token ceiling, because the shrinkers never drop one.
This commit is contained in:
2026-10-21 18:27:13 +08:00
parent d7293eeb16
commit c6a94e0b66
5 changed files with 185 additions and 20 deletions
+102
View File
@@ -0,0 +1,102 @@
"""A tool result that carries images beside its text."""
from __future__ import annotations
from msgspec import UNSET
from plyngent.agent import ToolOutput, ToolRegistry, tool
from plyngent.agent.budget import IMAGE_CHARS, estimate_message_chars
from plyngent.agent.loop import _run_one_tool
from plyngent.lmproto.openai_compatible.model import (
AssistantFunctionTool,
AssistantFunctionToolCall,
ChatImagePart,
ChatImageUrl,
ChatTextPart,
ToolChatMessage,
UserChatMessage,
)
def _image() -> ChatImagePart:
return ChatImagePart(image_url=ChatImageUrl(url="data:image/png;base64,AAAA", detail="low"))
def _registry() -> ToolRegistry:
@tool(register=False)
def shot() -> ToolOutput:
"""Pretend to take a screenshot."""
return ToolOutput(text="image: shot.png (2 KB)", images=[_image()])
@tool(register=False)
def plain() -> str:
return "just text"
return ToolRegistry([shot, plain])
def _call(name: str) -> AssistantFunctionToolCall:
return AssistantFunctionToolCall(id="call-1", function=AssistantFunctionTool(name=name, arguments="{}"))
async def test_execute_keeps_the_text_only_contract() -> None:
registry = _registry()
assert await registry.execute("shot", "{}") == "image: shot.png (2 KB)"
assert await registry.execute("plain", "{}") == "just text"
async def test_execute_output_returns_the_images_beside_the_text() -> None:
registry = _registry()
output = await registry.execute_output("shot", "{}")
assert output.text == "image: shot.png (2 KB)"
assert output.has_images
assert output.images == [_image()]
async def test_execute_output_wraps_a_plain_string_result() -> None:
output = await _registry().execute_output("plain", "{}")
assert output.text == "just text"
assert not output.has_images
async def test_an_unknown_tool_is_error_text_without_images() -> None:
output = await _registry().execute_output("missing", "{}")
assert "unknown tool" in output.text
assert not output.has_images
async def test_the_loop_keeps_tool_images_on_the_tool_message() -> None:
message, err = await _run_one_tool(_registry(), _call("shot"), max_result_chars=32_000)
assert err is None
assert isinstance(message, ToolChatMessage)
assert message.content == "image: shot.png (2 KB)"
assert message.images == [_image()]
async def test_the_loop_leaves_the_images_field_unset_for_text_tools() -> None:
message, err = await _run_one_tool(_registry(), _call("plain"), max_result_chars=32_000)
assert err is None
assert message.images is UNSET
async def test_a_truncated_tool_result_keeps_its_images() -> None:
message, _ = await _run_one_tool(_registry(), _call("shot"), max_result_chars=1)
assert "[Truncated" in message.content
assert message.images == [_image()]
def test_the_char_estimate_counts_images() -> None:
assert estimate_message_chars(UserChatMessage(content="look:")) == len("look:")
assert estimate_message_chars(UserChatMessage(content=[ChatTextPart(text="look:"), _image()])) == (
len("look:") + IMAGE_CHARS
)
assert estimate_message_chars(ToolChatMessage(content="shot", tool_call_id="call-1", images=[_image()])) == (
len("shot") + len("call-1") + IMAGE_CHARS
)