mirror of
https://github.com/NCBM/plyngent.git
synced 2026-11-07 07:17:54 +08:00
core/tools: show images instead of describing them
New `view_image` reads a local picture (JPEG, PNG, GIF, WebP, sniffed from its bytes) and hands it to the model as an image part, with an optional `detail` level — `low` measured 3.4x cheaper on a 1200x900 picture. `read_file` now points at it instead of returning decode noise. `fetch_local` returns an image response as the picture itself, and MCP image content blocks become image parts instead of `[image: ...]` placeholders, so a screenshot server finally reaches the model.
This commit is contained in:
@@ -2,12 +2,14 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import threading
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
from typing import TYPE_CHECKING, override
|
||||
|
||||
import pytest
|
||||
|
||||
from plyngent.agent import ToolOutput
|
||||
from plyngent.tools.context import read_only_context
|
||||
from plyngent.tools.danger import classify_danger
|
||||
from plyngent.tools.net import fetch_local, grant_private_host
|
||||
@@ -29,6 +31,12 @@ if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
# A real 1x1 PNG: the body must be sniffed by its bytes, not by the header.
|
||||
_PNG_BYTES = base64.b64decode(
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="
|
||||
)
|
||||
|
||||
|
||||
class _Handler(BaseHTTPRequestHandler):
|
||||
"""Minimal echo server for fetch_local tests."""
|
||||
|
||||
@@ -66,6 +74,9 @@ class _Handler(BaseHTTPRequestHandler):
|
||||
if self.path.startswith("/bin"):
|
||||
self._send(200, b"\x00\x01\x02\xffbinary", content_type="application/octet-stream")
|
||||
return
|
||||
if self.path.startswith("/png"):
|
||||
self._send(200, _PNG_BYTES, content_type="image/png")
|
||||
return
|
||||
if self.path.startswith("/ua"):
|
||||
ua = (self.headers.get("User-Agent") or "").encode()
|
||||
self._send(200, ua)
|
||||
@@ -200,6 +211,20 @@ async def test_fetch_get_post_put_delete(workspace: Path, http_server: str) -> N
|
||||
assert "status=204" in out_del
|
||||
|
||||
|
||||
async def test_an_image_response_comes_back_as_the_picture(workspace: Path, http_server: str) -> None:
|
||||
"""A PNG body is handed to the model as an image part, not a hex preview."""
|
||||
del workspace
|
||||
grant_private_host("127.0.0.1", int(http_server.rsplit(":", 1)[1]))
|
||||
|
||||
output: object = await fetch_local.handler(f"{http_server}/png")
|
||||
|
||||
assert isinstance(output, ToolOutput)
|
||||
assert "body_kind=image" in output.text
|
||||
assert "status=200" in output.text
|
||||
part = output.images[0]
|
||||
assert part.image_url.url.startswith("data:image/png;base64,")
|
||||
|
||||
|
||||
async def test_fetch_read_only_context_blocks_non_get(workspace: Path, http_server: str) -> None:
|
||||
"""In a read-only context fetch_local only allows GET; outside it POST works."""
|
||||
del workspace
|
||||
|
||||
Reference in New Issue
Block a user