core/tools: show images instead of describing them

New `view_image` reads a local picture (JPEG, PNG, GIF, WebP, sniffed from its
bytes) and hands it to the model as an image part, with an optional `detail`
level — `low` measured 3.4x cheaper on a 1200x900 picture. `read_file` now
points at it instead of returning decode noise.

`fetch_local` returns an image response as the picture itself, and MCP image
content blocks become image parts instead of `[image: ...]` placeholders, so a
screenshot server finally reaches the model.
This commit is contained in:
2026-10-21 18:32:50 +08:00
parent c6a94e0b66
commit e9ee71e853
14 changed files with 266 additions and 26 deletions
+18
View File
@@ -11,6 +11,8 @@ import os
import sys
_FAKE_INSTRUCTIONS = os.environ.get("PLYNGENT_MCP_FAKE_INSTRUCTIONS", "")
# A real 1x1 PNG (base64), what an image-carrying tool result looks like.
_PNG_BASE64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="
def respond(request_id: int, result: object) -> None:
@@ -61,6 +63,11 @@ def main() -> None:
"description": "Sleeps past any client timeout.",
"inputSchema": {"type": "object", "properties": {}},
},
{
"name": "shot",
"description": "Returns a picture beside its text.",
"inputSchema": {"type": "object", "properties": {}},
},
]
},
)
@@ -75,6 +82,17 @@ def main() -> None:
"isError": False,
},
)
elif name == "shot":
respond(
int(request_id),
{
"content": [
{"type": "text", "text": "shot.png"},
{"type": "image", "data": _PNG_BASE64, "mimeType": "image/png"},
],
"isError": False,
},
)
elif name == "fail":
respond(
int(request_id),
+21 -6
View File
@@ -34,13 +34,28 @@ async def test_connection_handshake_lists_and_calls_tools() -> None:
assert connection.connected
assert connection.error is None
assert connection.instructions is None
assert [tool.name for tool in connection.tools] == ["echo", "fail", "slow"]
assert [tool.name for tool in connection.tools] == ["echo", "fail", "slow", "shot"]
echo = connection.tools[0]
assert echo.description == "Echo the given text back."
assert echo.input_schema["type"] == "object"
result = await connection.call_tool("echo", {"text": "hello"})
assert result == "echo: hello"
assert result.text == "echo: hello"
assert result.images == []
finally:
await connection.aclose()
async def test_an_image_content_block_becomes_an_image_part() -> None:
"""A server that returns pixels gets them to the model, not a placeholder."""
connection = McpServerConnection("fake", _server_config())
await connection.start()
try:
output = await connection.call_tool("shot", {})
assert output.text == "shot.png"
assert len(output.images) == 1
assert output.images[0].image_url.url.startswith("data:image/png;base64,")
finally:
await connection.aclose()
@@ -64,7 +79,7 @@ async def test_tool_error_returns_error_text() -> None:
await connection.start()
try:
result = await connection.call_tool("fail", {})
assert result == "error: boom"
assert result.text == "error: boom"
finally:
await connection.aclose()
@@ -122,9 +137,9 @@ async def test_manager_start_status_and_call() -> None:
manager = McpManager(McpConfig(servers={"fake": _server_config()}, disable=[]))
await manager.ensure_started()
try:
assert [(name, status, count) for name, status, count in manager.statuses()] == [("fake", "connected", 3)]
assert [(name, status, count) for name, status, count in manager.statuses()] == [("fake", "connected", 4)]
assert manager.instructions() == []
assert await manager.call("fake", "echo", {"text": "hi"}) == "echo: hi"
assert (await manager.call("fake", "echo", {"text": "hi"})).text == "echo: hi"
finally:
await manager.aclose()
assert manager.statuses() == [("fake", "not started", 0)]
@@ -157,5 +172,5 @@ async def test_manager_disable_and_restart() -> None:
await manager.restart(McpConfig(servers={"fake": _server_config()}, disable=[]))
await manager.ensure_started()
assert manager.statuses() == [("fake", "connected", 3)]
assert manager.statuses() == [("fake", "connected", 4)]
await manager.aclose()