From d7293eeb16cb68535604b5783bec10c253d313e0 Mon Sep 17 00:00:00 2001 From: worldmozara Date: Wed, 21 Oct 2026 18:24:23 +0800 Subject: [PATCH] core/lmproto+agent: carry image parts on user and tool messages A user turn's `content` may now be a part array (`ChatTextPart` + `ChatImagePart`), which chat completions take as-is; the Anthropic and Responses bridges translate it into `image` blocks and `input_image` parts. A tool result keeps its text in `content` and carries `images` beside it: Anthropic and Responses put them in the result item itself (a live probe reads both back), while chat completions cannot, so `compat.chat_messages_with_images` moves them into the user turn after the tool run. Prose-only readers go through `content_to_text`, which renders an image as `[image]`. `agent.images` sniffs the real format from the bytes and builds the data URL, capped at 8 MiB inline because every stored message is re-sent on each round. --- src/plyngent/agent/chat.py | 9 +- src/plyngent/agent/images.py | 76 ++++++++++++++++ src/plyngent/agent/messages_bridge.py | 55 +++++++++++- src/plyngent/agent/responses_bridge.py | 36 +++++++- src/plyngent/cli/export.py | 3 +- src/plyngent/cli/transcript.py | 7 +- src/plyngent/lmproto/openai/client.py | 6 ++ src/plyngent/lmproto/openai/model.py | 9 +- .../lmproto/openai_compatible/compat.py | 53 +++++++++-- .../lmproto/openai_compatible/model.py | 57 ++++++++++++ tests/test_agent/test_messages_bridge.py | 56 ++++++++++++ tests/test_agent/test_responses_bridge.py | 59 +++++++++++++ tests/test_lmproto/test_chat_compat.py | 88 +++++++++++++++++++ 13 files changed, 495 insertions(+), 19 deletions(-) create mode 100644 src/plyngent/agent/images.py diff --git a/src/plyngent/agent/chat.py b/src/plyngent/agent/chat.py index 63dd977..2bb09a3 100644 --- a/src/plyngent/agent/chat.py +++ b/src/plyngent/agent/chat.py @@ -15,6 +15,7 @@ from plyngent.lmproto.openai_compatible.model import ( SystemChatMessage, ToolChatMessage, UserChatMessage, + content_to_text, ) from .budget import ( @@ -162,12 +163,12 @@ def incomplete_turn_user_text(messages: Sequence[AnyChatMessage]) -> str | None: return None last = body[-1] if isinstance(last, UserChatMessage): - return last.content + return content_to_text(last.content) if isinstance(last, ToolChatMessage) or (truncated and isinstance(last, AssistantChatMessage)): # Walk back to the user that opened this turn. for msg in reversed(body): if isinstance(msg, UserChatMessage): - return msg.content + return content_to_text(msg.content) return None return None @@ -558,7 +559,9 @@ class ChatAgent: raise RuntimeError(msg) for i in range(len(self.messages) - 1, -1, -1): msg = self.messages[i] - if isinstance(msg, UserChatMessage) and msg.content == text: + # ``incomplete_turn_user_text`` returns the text view, so compare that + # (a turn carrying images is a part array, not a string). + if isinstance(msg, UserChatMessage) and content_to_text(msg.content) == text: return i msg = "nothing to retry" raise RuntimeError(msg) diff --git a/src/plyngent/agent/images.py b/src/plyngent/agent/images.py new file mode 100644 index 0000000..f26a483 --- /dev/null +++ b/src/plyngent/agent/images.py @@ -0,0 +1,76 @@ +"""Image parts for the model: data URLs, format sniffing, and the size policy. + +A model-visible image is a :class:`ChatImagePart` holding a URL — an ``http(s)`` +link or an inline ``data:`` URL. Providers sniff the real format from the bytes +(never from a file name), so this module sniffs too and builds the data URL from +what it found. + +Inline bytes are capped at :data:`MAX_INLINE_IMAGE_BYTES`, well under the 32 MiB +a provider accepts: every stored message is re-sent on each round and lives in +the session's JSON message column, so a huge screenshot would be paid for over +and over. A larger image has to be resized first. +""" + +from __future__ import annotations + +import base64 + +from plyngent.lmproto.openai_compatible.model import ChatImagePart, ChatImageUrl, ImageDetail + +# Inline payload ceiling for one image (the API allows 32 MiB; see the module docstring). +MAX_INLINE_IMAGE_BYTES = 8 * 1024 * 1024 + +# Formatted images the API accepts, by their leading bytes. +_IMAGE_SIGNATURES: tuple[tuple[bytes, str], ...] = ( + (b"\x89PNG\r\n\x1a\n", "image/png"), + (b"\xff\xd8\xff", "image/jpeg"), + (b"GIF87a", "image/gif"), + (b"GIF89a", "image/gif"), +) +# RIFF....WEBP: four bytes of container, four of size, then the form type. +_WEBP_HEADER_LEN = 12 + + +def sniff_image_media_type(data: bytes) -> str | None: + """Media type of an image *data* by its content, or ``None`` if unsupported.""" + for signature, media_type in _IMAGE_SIGNATURES: + if data.startswith(signature): + return media_type + # WebP: RIFF....WEBP + if len(data) >= _WEBP_HEADER_LEN and data[:4] == b"RIFF" and data[8:12] == b"WEBP": + return "image/webp" + return None + + +def to_data_url(media_type: str, data: bytes) -> str: + """Encode *data* as an inline ``data:`` URL of *media_type*.""" + return f"data:{media_type};base64,{base64.b64encode(data).decode('ascii')}" + + +def split_data_url(url: str) -> tuple[str, str] | None: + """``(media_type, base64_payload)`` of a ``data:`` URL, or ``None`` otherwise.""" + if not url.startswith("data:"): + return None + header, _, payload = url.partition(",") + media_type = header[len("data:") :].split(";")[0].strip() + if not media_type or not payload: + return None + return media_type, payload + + +def image_part_from_bytes(data: bytes, *, detail: ImageDetail = "auto") -> ChatImagePart | str: + """An image part for *data*; a string is the model-facing error instead.""" + media_type = sniff_image_media_type(data) + if media_type is None: + return "error: not a JPEG, PNG, GIF, or WebP image" + if len(data) > MAX_INLINE_IMAGE_BYTES: + return ( + f"error: image is {len(data)} bytes, over the {MAX_INLINE_IMAGE_BYTES}-byte inline limit; " + "resize it and try again" + ) + return ChatImagePart(image_url=ChatImageUrl(url=to_data_url(media_type, data), detail=detail)) + + +def image_part_from_url(url: str, *, detail: ImageDetail = "auto") -> ChatImagePart: + """An image part for a public ``http(s)`` URL or an inline ``data:`` URL.""" + return ChatImagePart(image_url=ChatImageUrl(url=url, detail=detail)) diff --git a/src/plyngent/agent/messages_bridge.py b/src/plyngent/agent/messages_bridge.py index ef5a792..752a11b 100644 --- a/src/plyngent/agent/messages_bridge.py +++ b/src/plyngent/agent/messages_bridge.py @@ -13,8 +13,10 @@ from typing import TYPE_CHECKING, Any, cast from msgspec import UNSET +from plyngent.agent.images import split_data_url from plyngent.lmproto.anthropic.model import ( AnthropicAssistantMessage, + AnthropicImageContent, AnthropicMessageResponse, AnthropicMessagesParam, AnthropicOutputConfig, @@ -39,8 +41,11 @@ from plyngent.lmproto.openai_compatible.model import ( ChatCompletionChoice, ChatCompletionResponse, ChatCompletionsParam, + ChatImagePart, + ChatTextPart, DeveloperChatMessage, SystemChatMessage, + ToolChatMessage, ToolFunctionItem, UserChatMessage, ) @@ -135,6 +140,52 @@ def _tool_choice_from_chat(param: ChatCompletionsParam) -> AnthropicToolChoice | return UNSET +def _image_to_anthropic(part: ChatImagePart) -> AnthropicImageContent: + """One chat image part as an Anthropic ``image`` block. + + An inline ``data:`` URL becomes a ``base64`` source (the provider then reads + the media type we sniffed); anything else is a ``url`` source. + """ + inline = split_data_url(part.image_url.url) + if inline is None: + return AnthropicImageContent(source={"type": "url", "url": part.image_url.url}) + media_type, payload = inline + return AnthropicImageContent(source={"type": "base64", "media_type": media_type, "data": payload}) + + +type _AnthropicUserBlock = AnthropicTextContent | AnthropicImageContent | AnthropicToolResultContent + + +def _user_blocks_to_anthropic( + content: str | list[ChatTextPart | ChatImagePart], +) -> str | list[_AnthropicUserBlock]: + """A chat user turn's content as Anthropic blocks (plain text stays a string).""" + if isinstance(content, str): + return content + return [ + AnthropicTextContent(text=part.text) if isinstance(part, ChatTextPart) else _image_to_anthropic(part) + for part in content + ] + + +def _tool_result_content( + message: ToolChatMessage, +) -> str | list[AnthropicTextContent | AnthropicImageContent]: + """A tool result's content: its text, plus any images the tool returned. + + Anthropic takes image blocks inside ``tool_result.content`` (a live probe + confirms the DeepSeek surface reads them), so the images ride in the result + itself instead of a separate user turn. + """ + if message.images is UNSET or not message.images: + return message.content + blocks: list[AnthropicTextContent | AnthropicImageContent] = [] + if message.content: + blocks.append(AnthropicTextContent(text=message.content)) + blocks.extend(_image_to_anthropic(part) for part in message.images) + return blocks + + def _assistant_to_anthropic(message: AssistantChatMessage) -> AnthropicAssistantMessage | None: """The assistant turn for the wire: opaque blocks, then text, then calls. @@ -211,7 +262,7 @@ def chat_messages_to_anthropic( # noqa: C901 — multi-role conversion if isinstance(message, UserChatMessage): flush_tool_results() - out.append(AnthropicUserMessage(content=message.content)) + out.append(AnthropicUserMessage(content=_user_blocks_to_anthropic(message.content))) continue if isinstance(message, AssistantChatMessage): @@ -225,7 +276,7 @@ def chat_messages_to_anthropic( # noqa: C901 — multi-role conversion pending_tool_results.append( AnthropicToolResultContent( tool_use_id=message.tool_call_id, - content=message.content, + content=_tool_result_content(message), ) ) diff --git a/src/plyngent/agent/responses_bridge.py b/src/plyngent/agent/responses_bridge.py index a76460e..7eb9bf2 100644 --- a/src/plyngent/agent/responses_bridge.py +++ b/src/plyngent/agent/responses_bridge.py @@ -29,8 +29,11 @@ from plyngent.lmproto.openai_compatible.model import ( ChatCompletionChunk, ChatCompletionResponse, ChatCompletionsParam, + ChatImagePart, + ChatTextPart, DeveloperChatMessage, SystemChatMessage, + ToolChatMessage, ToolFunctionItem, UserChatMessage, ) @@ -175,6 +178,35 @@ def _group_tool_outputs( return out +def _image_to_input_part(part: ChatImagePart) -> dict[str, Any]: + """One chat image part as a Responses ``input_image`` item.""" + item: dict[str, Any] = {"type": "input_image", "image_url": part.image_url.url} + if part.image_url.detail is not UNSET: + item["detail"] = part.image_url.detail + return item + + +def _user_content_to_input(content: str | list[ChatTextPart | ChatImagePart]) -> str | list[dict[str, Any]]: + """A chat user turn's content as ``input`` content (plain text stays a string).""" + if isinstance(content, str): + return content + return [ + {"type": "input_text", "text": part.text} if isinstance(part, ChatTextPart) else _image_to_input_part(part) + for part in content + ] + + +def _tool_output(message: ToolChatMessage) -> str | list[dict[str, Any]]: + """A tool result's ``output``: its text, plus any images the tool returned.""" + if message.images is UNSET or not message.images: + return message.content + parts: list[dict[str, Any]] = [] + if message.content: + parts.append({"type": "input_text", "text": message.content}) + parts.extend(_image_to_input_part(part) for part in message.images) + return parts + + def chat_messages_to_responses_input( messages: Sequence[AnyChatMessage], ) -> tuple[str | None, list[dict[str, Any] | ResponseEasyInputMessage | ResponseFunctionToolCallOutput]]: @@ -191,7 +223,7 @@ def chat_messages_to_responses_input( if message.content.strip(): items.append(ResponseEasyInputMessage(role="developer", content=message.content)) elif isinstance(message, UserChatMessage): - items.append(ResponseEasyInputMessage(role="user", content=message.content)) + items.append(ResponseEasyInputMessage(role="user", content=_user_content_to_input(message.content))) elif isinstance(message, AssistantChatMessage): items.extend(_assistant_to_input_items(message)) else: @@ -199,7 +231,7 @@ def chat_messages_to_responses_input( items.append( ResponseFunctionToolCallOutput( call_id=message.tool_call_id, - output=message.content, + output=_tool_output(message), ) ) diff --git a/src/plyngent/cli/export.py b/src/plyngent/cli/export.py index 21bd750..f578d5f 100644 --- a/src/plyngent/cli/export.py +++ b/src/plyngent/cli/export.py @@ -10,6 +10,7 @@ from plyngent.lmproto.openai_compatible.model import ( AssistantFunctionToolCall, ToolChatMessage, UserChatMessage, + content_to_text, ) if TYPE_CHECKING: @@ -78,7 +79,7 @@ def format_session_export_md( def _format_message_md(message: AnyChatMessage) -> list[str]: if isinstance(message, UserChatMessage): - return ["## user", "", message.content] + return ["## user", "", content_to_text(message.content)] if isinstance(message, AssistantChatMessage): parts: list[str] = ["## assistant", ""] reasoning = message.reasoning_content diff --git a/src/plyngent/cli/transcript.py b/src/plyngent/cli/transcript.py index 555a921..61e3029 100644 --- a/src/plyngent/cli/transcript.py +++ b/src/plyngent/cli/transcript.py @@ -28,6 +28,7 @@ from plyngent.lmproto.openai_compatible.model import ( SystemChatMessage, ToolChatMessage, UserChatMessage, + content_to_text, ) if TYPE_CHECKING: @@ -225,7 +226,7 @@ def build_transcript(messages: Sequence[AnyChatMessage], *, pending_retry_text: if turns and pending_retry_text is not None: user = turns[-1].user.message - if isinstance(user, UserChatMessage) and user.content == pending_retry_text: + if isinstance(user, UserChatMessage) and content_to_text(user.content) == pending_retry_text: turns[-1] = replace(turns[-1], incomplete=True) return Transcript(prelude=tuple(prelude), turns=tuple(turns)) @@ -239,7 +240,9 @@ def echo_row(row: Row, *, detail: Detail) -> None: _echo_assistant(row, reasoning=detail == "full") return click.echo(row.prefix()) - click.echo(row.message.content or "") + message = row.message + click.echo(content_to_text(message.content) if isinstance(message, UserChatMessage) else message.content) + click.echo() click.echo() diff --git a/src/plyngent/lmproto/openai/client.py b/src/plyngent/lmproto/openai/client.py index 40608a2..27cfde6 100644 --- a/src/plyngent/lmproto/openai/client.py +++ b/src/plyngent/lmproto/openai/client.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Any, Literal, cast, overload import msgspec from plyngent.lmproto.openai_compatible.client import BaseOpenAIClient +from plyngent.lmproto.openai_compatible.compat import chat_messages_with_images from plyngent.lmproto.openai_compatible.config import OpenAIConfig # noqa: TC001 from plyngent.lmproto.openai_compatible.model import ( # noqa: TC001 ChatCompletionChunk, @@ -49,6 +50,11 @@ class OpenAIClient(BaseOpenAIClient): self, param: ChatCompletionsParam, *, stream: bool = False ) -> ChatCompletionResponse | AsyncIterator[ChatCompletionChunk]: param = msgspec.structs.replace(param, stream=stream) + # Lift images into content parts, like every other chat-completions wire; + # the OpenAI API takes the ``developer`` role as-is, so only that step. + images = chat_messages_with_images(param.messages) + if images != param.messages: + param = msgspec.structs.replace(param, messages=images) data = self.encoder.encode(param) if stream: resp = await self.session.post( diff --git a/src/plyngent/lmproto/openai/model.py b/src/plyngent/lmproto/openai/model.py index 2d1a631..7f5d710 100644 --- a/src/plyngent/lmproto/openai/model.py +++ b/src/plyngent/lmproto/openai/model.py @@ -98,10 +98,15 @@ class ResponseFunctionToolCall(Struct, tag_field="type", tag="function_call"): class ResponseFunctionToolCallOutput(Struct, tag_field="type", tag="function_call_output"): - """Tool result item passed back in subsequent ``input``.""" + """Tool result item passed back in subsequent ``input``. + + ``output`` may be a part array instead of text: a tool that returns an image + puts an ``input_image`` part beside its ``input_text`` (the Responses guide + documents images there, and a live probe reads them back). + """ call_id: str - output: str + output: str | list[dict[str, Any]] id: str | Unset = UNSET status: ResponseItemStatus | Unset = UNSET diff --git a/src/plyngent/lmproto/openai_compatible/compat.py b/src/plyngent/lmproto/openai_compatible/compat.py index 77c4a3f..930c7a3 100644 --- a/src/plyngent/lmproto/openai_compatible/compat.py +++ b/src/plyngent/lmproto/openai_compatible/compat.py @@ -7,12 +7,18 @@ from typing import TYPE_CHECKING, Any, cast import msgspec from msgspec import UNSET -from .model import AssistantChatMessage, DeveloperChatMessage, SystemChatMessage +from .model import ( + AssistantChatMessage, + DeveloperChatMessage, + SystemChatMessage, + ToolChatMessage, + UserChatMessage, +) if TYPE_CHECKING: from collections.abc import Sequence - from .model import AnyChatMessage, ChatCompletionsParam + from .model import AnyChatMessage, ChatCompletionsParam, ChatImagePart def developer_to_system_message(message: DeveloperChatMessage) -> SystemChatMessage: @@ -38,6 +44,38 @@ def coerce_developer_messages_to_system(messages: Sequence[AnyChatMessage]) -> l return out if changed else list(messages) +def chat_messages_with_images(messages: Sequence[AnyChatMessage]) -> list[AnyChatMessage]: + """Return *messages* with tool-result images lifted into the chat wire. + + A user turn already carries its images in ``content`` blocks, which chat + completions take as-is. A ``tool`` role message may only carry text, so a + tool result's images move into the user turn emitted right after the run of + consecutive tool results. + + With no such images the sequence's items come back unchanged. + """ + out: list[AnyChatMessage] = [] + pending: list[ChatImagePart] = [] + changed = False + + def flush_images() -> None: + if pending: + out.append(UserChatMessage(content=list(pending))) + pending.clear() + + for message in messages: + if isinstance(message, ToolChatMessage) and message.images is not UNSET and message.images: + out.append(msgspec.structs.replace(message, images=UNSET)) + pending.extend(message.images) + changed = True + continue + if pending and not isinstance(message, ToolChatMessage): + flush_images() + out.append(message) + flush_images() + return out if changed else list(messages) + + def coerce_chat_messages(messages: Sequence[AnyChatMessage]) -> list[AnyChatMessage]: """Return *messages* as a chat-completions wire can take them. @@ -46,18 +84,19 @@ def coerce_chat_messages(messages: Sequence[AnyChatMessage]) -> list[AnyChatMess and ``AssistantChatMessage.provider_blocks`` is dropped — those blocks belong to the surface that produced them (Anthropic ``thinking`` and server-tool blocks; see ``agent.messages_bridge``) and no other provider - knows the key. + knows the key. Tool-result images are lifted last (see + :func:`chat_messages_with_images`). """ coerced = coerce_developer_messages_to_system(messages) changed = False - out: list[AnyChatMessage] = [] + stripped: list[AnyChatMessage] = [] for message in coerced: if isinstance(message, AssistantChatMessage) and message.provider_blocks is not UNSET: - out.append(msgspec.structs.replace(message, provider_blocks=UNSET)) + stripped.append(msgspec.structs.replace(message, provider_blocks=UNSET)) changed = True else: - out.append(message) - return out if changed else coerced + stripped.append(message) + return chat_messages_with_images(stripped if changed else coerced) def coerce_chat_completions_param(param: ChatCompletionsParam) -> ChatCompletionsParam: diff --git a/src/plyngent/lmproto/openai_compatible/model.py b/src/plyngent/lmproto/openai_compatible/model.py index 2da03cd..9c34364 100644 --- a/src/plyngent/lmproto/openai_compatible/model.py +++ b/src/plyngent/lmproto/openai_compatible/model.py @@ -19,6 +19,41 @@ type Verbosity = Literal["low", "medium", "high"] type GrammarSyntax = Literal["lark", "regex"] +type ImageDetail = Literal["low", "high", "original", "auto"] + + +class ChatImageUrl(Struct, omit_defaults=True): + """The ``image_url`` payload of one image part (the canonical chat shape).""" + + url: str + detail: ImageDetail | Unset = UNSET + + +class ChatImagePart(Struct, tag_field="type", tag="image_url"): + """One image a message carries: an ``http(s)`` URL or a ``data:`` URL. + + This is the shape OpenAI-compatible chat completions take. The other two + surfaces lift it into their own form — ``compat.chat_messages_with_images`` + for chat completions, ``agent.messages_bridge`` (Anthropic ``image`` + blocks) and ``agent.responses_bridge`` (``input_image`` parts). + """ + + image_url: ChatImageUrl + + +class ChatTextPart(Struct, tag_field="type", tag="text"): + """A text block beside images, when a message's content is a part array.""" + + text: str + + +type ChatContentPart = ChatTextPart | ChatImagePart +type ChatContent = str | list[ChatContentPart] + +# What a text view of a part array shows where an image sits. +IMAGE_MARKER = "[image]" + + class ChatMessage(Struct): content: str @@ -28,6 +63,12 @@ class SystemChatMessage(ChatMessage, tag_field="role", tag="system"): class UserChatMessage(ChatMessage, tag_field="role", tag="user"): + # A user turn may carry images beside its text; chat completions already take + # that shape as ``content`` blocks, and the other two surfaces translate it + # (``agent.messages_bridge``, ``agent.responses_bridge``). The wider type is + # deliberate: only a user turn takes images, so the base stays a string and + # prose-only callers read it through ``content_to_text``. + content: str | list[ChatContentPart] # pyright: ignore[reportIncompatibleVariableOverride] name: str | Unset = UNSET @@ -35,6 +76,17 @@ class DeveloperChatMessage(ChatMessage, tag_field="role", tag="developer"): name: str | Unset = UNSET +def content_to_text(content: ChatContent) -> str: + """The plain text of a message ``content``, with ``[image]`` for each image. + + Callers that only need prose (summaries, transcripts, exports, comparisons) + must not print a part array's ``repr``. + """ + if isinstance(content, str): + return content + return " ".join(part.text if isinstance(part, ChatTextPart) else IMAGE_MARKER for part in content) + + type NamedChatMessage = SystemChatMessage | UserChatMessage | DeveloperChatMessage @@ -84,6 +136,11 @@ class AssistantChatMessage(Struct, tag_field="role", tag="assistant"): class ToolChatMessage(ChatMessage, tag_field="role", tag="tool"): tool_call_id: str + # Images the tool returned. Chat completions cannot put an image in a + # ``tool`` message, so on that wire they move into the user message that + # follows the tool run (``compat.chat_messages_with_images``); the Anthropic + # and Responses surfaces carry them in the result itself. + images: list[ChatImagePart] | Unset = UNSET type AnyChatMessage = ( diff --git a/tests/test_agent/test_messages_bridge.py b/tests/test_agent/test_messages_bridge.py index a69a6e4..29c028f 100644 --- a/tests/test_agent/test_messages_bridge.py +++ b/tests/test_agent/test_messages_bridge.py @@ -20,10 +20,12 @@ from plyngent.agent.messages_bridge import ( ) from plyngent.lmproto.anthropic.model import ( AnthropicAssistantMessage, + AnthropicImageContent, AnthropicMessageResponse, AnthropicResponseText, AnthropicResponseToolUse, AnthropicServerToolUseContent, + AnthropicTextContent, AnthropicThinkingContent, AnthropicToolDefinition, AnthropicToolResultContent, @@ -37,6 +39,9 @@ from plyngent.lmproto.openai_compatible.model import ( AssistantFunctionTool, AssistantFunctionToolCall, ChatCompletionsParam, + ChatImagePart, + ChatImageUrl, + ChatTextPart, DeveloperChatMessage, ReasoningEffort, SystemChatMessage, @@ -136,6 +141,57 @@ def test_merge_consecutive_tool_results() -> None: assert len(items[2].content) == 2 +def test_a_user_turn_carries_its_images_as_blocks() -> None: + messages = [ + UserChatMessage( + content=[ + ChatTextPart(text="what is this?"), + ChatImagePart(image_url=ChatImageUrl(url="data:image/png;base64,AAAA", detail="low")), + ChatImagePart(image_url=ChatImageUrl(url="https://example.com/a.jpg")), + ] + ) + ] + _, items = chat_messages_to_anthropic(messages) + + assert isinstance(items[0], AnthropicUserMessage) + blocks = items[0].content + assert isinstance(blocks, list) + assert blocks[0] == AnthropicTextContent(text="what is this?") + # An inline data URL becomes a base64 source carrying the sniffed media type. + assert blocks[1] == AnthropicImageContent(source={"type": "base64", "media_type": "image/png", "data": "AAAA"}) + # A public link stays a url source. + assert blocks[2] == AnthropicImageContent(source={"type": "url", "url": "https://example.com/a.jpg"}) + + +def test_a_tool_result_carries_its_images_inside_the_result() -> None: + messages = [ + UserChatMessage(content="screenshot please"), + AssistantChatMessage( + content=None, + tool_calls=[ + AssistantFunctionToolCall(id="c1", function=AssistantFunctionTool(name="view_image", arguments="{}")) + ], + ), + ToolChatMessage( + content="image: shot.png (2 KB)", + tool_call_id="c1", + images=[ChatImagePart(image_url=ChatImageUrl(url="data:image/png;base64,AAAA"))], + ), + ] + _, items = chat_messages_to_anthropic(messages) + + result = items[2] + assert isinstance(result, AnthropicUserMessage) + assert isinstance(result.content, list) + block = result.content[0] + assert isinstance(block, AnthropicToolResultContent) + assert block.tool_use_id == "c1" + assert block.content == [ + AnthropicTextContent(text="image: shot.png (2 KB)"), + AnthropicImageContent(source={"type": "base64", "media_type": "image/png", "data": "AAAA"}), + ] + + def test_anthropic_response_to_assistant_with_tools() -> None: response = AnthropicMessageResponse( id="msg_1", diff --git a/tests/test_agent/test_responses_bridge.py b/tests/test_agent/test_responses_bridge.py index 7ba7203..ac76525 100644 --- a/tests/test_agent/test_responses_bridge.py +++ b/tests/test_agent/test_responses_bridge.py @@ -19,6 +19,9 @@ from plyngent.lmproto.openai_compatible.model import ( AssistantFunctionTool, AssistantFunctionToolCall, ChatCompletionsParam, + ChatImagePart, + ChatImageUrl, + ChatTextPart, DeveloperChatMessage, SystemChatMessage, ToolChatMessage, @@ -76,6 +79,62 @@ def test_chat_messages_to_input_and_instructions() -> None: assert _input_item_types(items) == ["message", "reasoning", "function_call", "function_call_output"] +def test_a_user_turn_carries_its_images_as_input_parts() -> None: + messages = [ + UserChatMessage( + content=[ + ChatTextPart(text="what is this?"), + ChatImagePart(image_url=ChatImageUrl(url="https://example.com/a.jpg", detail="low")), + ] + ) + ] + _, items = chat_messages_to_responses_input(messages) + + assert items[0] == ResponseEasyInputMessage( + role="user", + content=[ + {"type": "input_text", "text": "what is this?"}, + {"type": "input_image", "image_url": "https://example.com/a.jpg", "detail": "low"}, + ], + ) + + +def test_a_tool_output_carries_the_images_the_tool_returned() -> None: + messages = [ + AssistantChatMessage( + content=None, + tool_calls=[ + AssistantFunctionToolCall( + id="call_1", function=AssistantFunctionTool(name="view_image", arguments="{}") + ) + ], + ), + ToolChatMessage( + content="image: shot.png (2 KB)", + tool_call_id="call_1", + images=[ChatImagePart(image_url=ChatImageUrl(url="data:image/png;base64,AAAA"))], + ), + ] + _, items = chat_messages_to_responses_input(messages) + + output = items[-1] + assert isinstance(output, ResponseFunctionToolCallOutput) + assert output.call_id == "call_1" + assert output.output == [ + {"type": "input_text", "text": "image: shot.png (2 KB)"}, + {"type": "input_image", "image_url": "data:image/png;base64,AAAA"}, + ] + + +def test_a_plain_tool_output_stays_a_string() -> None: + messages = [ToolChatMessage(content="file body", tool_call_id="call_1")] + _, items = chat_messages_to_responses_input(messages) + + output = items[0] + assert isinstance(output, ResponseFunctionToolCallOutput) + assert output.output == "file body" + + def test_parallel_tool_outputs_group_after_the_rounds_calls() -> None: """Regression: DeepSeek Responses rejected an interleaved batch. diff --git a/tests/test_lmproto/test_chat_compat.py b/tests/test_lmproto/test_chat_compat.py index fa1addc..a4152ac 100644 --- a/tests/test_lmproto/test_chat_compat.py +++ b/tests/test_lmproto/test_chat_compat.py @@ -3,6 +3,7 @@ from __future__ import annotations from collections.abc import Sequence +from typing import Any import msgspec from msgspec import UNSET @@ -18,10 +19,18 @@ from plyngent.lmproto.openai_compatible.compat import ( from plyngent.lmproto.openai_compatible.model import ( AnyChatMessage, AssistantChatMessage, + AssistantFunctionTool, + AssistantFunctionToolCall, ChatCompletionsParam, + ChatImagePart, + ChatImageUrl, + ChatTextPart, DeveloperChatMessage, + ImageDetail, SystemChatMessage, + ToolChatMessage, UserChatMessage, + content_to_text, ) @@ -119,3 +128,82 @@ def test_coerce_chat_messages_drops_provider_blocks() -> None: def test_coerce_chat_messages_leaves_plain_history_alone() -> None: messages: list[AnyChatMessage] = [UserChatMessage(content="hi"), AssistantChatMessage(content="done")] assert coerce_chat_messages(messages) == messages + + +def _image(url: str = "data:image/png;base64,AAAA", detail: ImageDetail = "low") -> ChatImagePart: + return ChatImagePart(image_url=ChatImageUrl(url=url, detail=detail)) + + +def _wire(messages: Sequence[AnyChatMessage]) -> list[dict[str, Any]]: + param = ChatCompletionsParam(model="m", messages=coerce_chat_messages(messages)) + raw: dict[str, Any] = msgspec.json.decode(msgspec.json.encode(param)) + messages_raw: list[dict[str, Any]] = raw["messages"] + return messages_raw + + +def test_user_images_reach_the_chat_wire_as_blocks() -> None: + messages: list[AnyChatMessage] = [ + UserChatMessage( + content=[ + ChatTextPart(text="what is this?"), + ChatImagePart(image_url=ChatImageUrl(url="https://example.com/a.png", detail="low")), + ] + ) + ] + + assert _wire(messages) == [ + { + "role": "user", + "content": [ + {"type": "text", "text": "what is this?"}, + {"type": "image_url", "image_url": {"url": "https://example.com/a.png", "detail": "low"}}, + ], + } + ] + + +def test_tool_result_images_move_into_the_user_turn_after_the_tool_run() -> None: + messages: list[AnyChatMessage] = [ + AssistantChatMessage( + content="", + tool_calls=[ + AssistantFunctionToolCall( + id="call-1", function=AssistantFunctionTool(name="view_image", arguments="{}") + ) + ], + ), + ToolChatMessage(content="image: shot.png (1 KB)", tool_call_id="call-1", images=[_image()]), + ToolChatMessage(content="second result", tool_call_id="call-2"), + AssistantChatMessage(content="I can see it."), + ] + raw = _wire(messages) + + assert [item["role"] for item in raw] == ["assistant", "tool", "tool", "user", "assistant"] + # The tool message keeps its text and loses the field chat completions cannot send. + assert raw[1] == {"role": "tool", "content": "image: shot.png (1 KB)", "tool_call_id": "call-1"} + assert raw[3]["content"] == [ + {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA", "detail": "low"}} + ] + # The stored history keeps its own image parts. + assert messages[1].images is not UNSET + + +def test_a_tool_result_without_images_keeps_the_history_untouched() -> None: + messages: list[AnyChatMessage] = [ToolChatMessage(content="plain", tool_call_id="call-1")] + assert coerce_chat_messages(messages) == messages + + +def test_the_deepseek_param_takes_a_user_turn_with_images() -> None: + base = DeepseekChatCompletionsParam( + model="deepseek-flash", + messages=[UserChatMessage(content=[_image()])], + ) + coerced = coerce_chat_completions_param_any(base) + raw = msgspec.json.decode(msgspec.json.encode(coerced)) + + assert raw["messages"][0]["content"][0]["type"] == "image_url" + + +def test_content_to_text_replaces_images_with_a_marker() -> None: + assert content_to_text("plain") == "plain" + assert content_to_text([ChatTextPart(text="look:"), _image()]) == "look: [image]"