core/tools: show images instead of describing them

New `view_image` reads a local picture (JPEG, PNG, GIF, WebP, sniffed from its
bytes) and hands it to the model as an image part, with an optional `detail`
level — `low` measured 3.4x cheaper on a 1200x900 picture. `read_file` now
points at it instead of returning decode noise.

`fetch_local` returns an image response as the picture itself, and MCP image
content blocks become image parts instead of `[image: ...]` placeholders, so a
screenshot server finally reaches the model.
This commit is contained in:
2026-10-21 18:32:50 +08:00
parent c6a94e0b66
commit e9ee71e853
14 changed files with 266 additions and 26 deletions
+1
View File
@@ -59,6 +59,7 @@ def test_read_only_classification() -> None:
read_only = {t.name for t in default_tool_definitions(surface="local") if t.tags & ToolTag.READ_ONLY}
assert read_only == {
"read_file",
"view_image",
"get_truncated",
"glob_paths",
"regex_files",
+25
View File
@@ -2,12 +2,14 @@
from __future__ import annotations
import base64
import threading
from http.server import BaseHTTPRequestHandler, HTTPServer
from typing import TYPE_CHECKING, override
import pytest
from plyngent.agent import ToolOutput
from plyngent.tools.context import read_only_context
from plyngent.tools.danger import classify_danger
from plyngent.tools.net import fetch_local, grant_private_host
@@ -29,6 +31,12 @@ if TYPE_CHECKING:
from pathlib import Path
# A real 1x1 PNG: the body must be sniffed by its bytes, not by the header.
_PNG_BYTES = base64.b64decode(
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="
)
class _Handler(BaseHTTPRequestHandler):
"""Minimal echo server for fetch_local tests."""
@@ -66,6 +74,9 @@ class _Handler(BaseHTTPRequestHandler):
if self.path.startswith("/bin"):
self._send(200, b"\x00\x01\x02\xffbinary", content_type="application/octet-stream")
return
if self.path.startswith("/png"):
self._send(200, _PNG_BYTES, content_type="image/png")
return
if self.path.startswith("/ua"):
ua = (self.headers.get("User-Agent") or "").encode()
self._send(200, ua)
@@ -200,6 +211,20 @@ async def test_fetch_get_post_put_delete(workspace: Path, http_server: str) -> N
assert "status=204" in out_del
async def test_an_image_response_comes_back_as_the_picture(workspace: Path, http_server: str) -> None:
"""A PNG body is handed to the model as an image part, not a hex preview."""
del workspace
grant_private_host("127.0.0.1", int(http_server.rsplit(":", 1)[1]))
output: object = await fetch_local.handler(f"{http_server}/png")
assert isinstance(output, ToolOutput)
assert "body_kind=image" in output.text
assert "status=200" in output.text
part = output.images[0]
assert part.image_url.url.startswith("data:image/png;base64,")
async def test_fetch_read_only_context_blocks_non_get(workspace: Path, http_server: str) -> None:
"""In a read-only context fetch_local only allows GET; outside it POST works."""
del workspace
+68
View File
@@ -0,0 +1,68 @@
"""``view_image``: a local picture reaches the model as a picture."""
from __future__ import annotations
import base64
from typing import TYPE_CHECKING
from plyngent.agent import ToolOutput
from plyngent.tools.file.read import read_file
from plyngent.tools.file.view_image import view_image
if TYPE_CHECKING:
from pathlib import Path
# A real 1x1 PNG: the format is sniffed from these bytes.
PNG = base64.b64decode(
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="
)
async def test_view_image_attaches_the_picture(workspace: Path) -> None:
(workspace / "shot.png").write_bytes(PNG)
output = await view_image.handler("shot.png")
assert isinstance(output, ToolOutput)
assert output.text == "image: shot.png (attached below)"
url = output.images[0].image_url.url
assert url.startswith("data:image/png;base64,")
assert base64.b64decode(url.partition(",")[2]) == PNG
async def test_the_detail_level_reaches_the_part(workspace: Path) -> None:
(workspace / "shot.png").write_bytes(PNG)
output = await view_image.handler("shot.png", detail="low")
assert isinstance(output, ToolOutput)
assert output.images[0].image_url.detail == "low"
async def test_an_unknown_detail_level_is_an_error(workspace: Path) -> None:
(workspace / "shot.png").write_bytes(PNG)
assert "detail must be one of" in await view_image.handler("shot.png", detail="huge")
async def test_a_non_image_file_is_an_error(workspace: Path) -> None:
(workspace / "notes.txt").write_text("hello")
result = await view_image.handler("notes.txt")
assert isinstance(result, str)
assert "not a JPEG, PNG, GIF, or WebP image" in result
async def test_a_missing_file_is_an_error(workspace: Path) -> None:
del workspace
assert await view_image.handler("nope.png") == "error: file not found: nope.png"
async def test_read_file_points_at_view_image_for_a_picture(workspace: Path) -> None:
(workspace / "shot.png").write_bytes(PNG)
result = await read_file.handler("shot.png")
assert result == "error: shot.png is an image (image/png); use view_image to look at it"