From 154509b929a6a1f712b64b079fb535e0e879f1b9 Mon Sep 17 00:00:00 2001 From: worldmozara Date: Wed, 21 Oct 2026 18:52:05 +0800 Subject: [PATCH] core/cli+config: attach files to a turn with @path mentions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Typing `@shot.png` in the line attaches that picture to the turn — no separate command — and `@notes.txt` appends the file as text under a `--- @path ---` header, capped like a tool result with a resumable `truncate_token`. `@"my shot.png"` is the quoted form, and Tab completes the path after `@` (directories end in `/`, a name with a space completes quoted). An image is skipped, with a note naming the key to set, on a model without `image_in`; a missing or denylisted path is a note too, and the turn still goes. The default catalogs mark what can take a picture (`deepseek-flash` yes, `deepseek-v4-pro` no, the OpenAI defaults yes). `Agent.run` takes the parts, so the turn is stored with its images and shows up as `[image]` wherever only prose is wanted. --- src/plyngent/agent/chat.py | 18 ++- src/plyngent/agent/images.py | 27 +++- src/plyngent/cli/mentions.py | 217 ++++++++++++++++++++++++++ src/plyngent/cli/readline_setup.py | 6 +- src/plyngent/cli/repl.py | 23 ++- src/plyngent/cli/retry.py | 6 +- src/plyngent/config/models.py | 15 +- tests/test_agent/test_images.py | 80 ++++++++++ tests/test_cli/test_mentions.py | 88 +++++++++++ tests/test_cli/test_readline_setup.py | 40 +++++ tests/test_cli/test_repl_commands.py | 81 +++++++++- 11 files changed, 577 insertions(+), 24 deletions(-) create mode 100644 src/plyngent/cli/mentions.py create mode 100644 tests/test_agent/test_images.py create mode 100644 tests/test_cli/test_mentions.py diff --git a/src/plyngent/agent/chat.py b/src/plyngent/agent/chat.py index 2bb09a3..cd412a3 100644 --- a/src/plyngent/agent/chat.py +++ b/src/plyngent/agent/chat.py @@ -28,6 +28,7 @@ from .directive_checkpoint import ( parse_checkpoint_bands, ) from .events import UsageEvent +from .images import user_content from .loop import DEFAULT_MAX_ROUNDS, DEFAULT_MAX_TRUNCATION_CONTINUES, run_chat_loop from .notices import TRUNCATED_NOTICE_KIND, notice_of_kind from .todo_nag import ( @@ -41,7 +42,7 @@ from .usage import TokenUsage if TYPE_CHECKING: from collections.abc import AsyncIterator, Awaitable, Callable, Sequence - from plyngent.lmproto.openai_compatible.model import AnyAssistantToolCall, AnyChatMessage + from plyngent.lmproto.openai_compatible.model import AnyAssistantToolCall, AnyChatMessage, ChatImagePart from plyngent.memory import MemoryStore from .events import AgentEvent @@ -664,11 +665,20 @@ class ChatAgent: await self._persist_range(self._persist_from, len(self.messages)) await self._persist_context_usage() - async def run(self, user_text: str) -> AsyncIterator[AgentEvent]: - """Append a user message (persist immediately), run the tool loop, yield events.""" + async def run( + self, + user_text: str, + *, + images: Sequence[ChatImagePart] | None = None, + ) -> AsyncIterator[AgentEvent]: + """Append a user message (persist immediately), run the tool loop, yield events. + + *images* ride along with the text as content parts, which is the shape + the model sees an attachment in (see ``agent.images``). + """ _ = reset_lineno_tracker() self._ensure_system_prompt() - user_msg = UserChatMessage(content=user_text) + user_msg = UserChatMessage(content=user_content(user_text, images)) self.messages.append(user_msg) await self._persist(user_msg) self._persist_from = len(self.messages) diff --git a/src/plyngent/agent/images.py b/src/plyngent/agent/images.py index f26a483..aae24b4 100644 --- a/src/plyngent/agent/images.py +++ b/src/plyngent/agent/images.py @@ -14,8 +14,19 @@ and over. A larger image has to be resized first. from __future__ import annotations import base64 +from typing import TYPE_CHECKING -from plyngent.lmproto.openai_compatible.model import ChatImagePart, ChatImageUrl, ImageDetail +from plyngent.lmproto.openai_compatible.model import ( + ChatImagePart, + ChatImageUrl, + ChatTextPart, + ImageDetail, +) + +if TYPE_CHECKING: + from collections.abc import Sequence + + from plyngent.lmproto.openai_compatible.model import ChatContentPart # Inline payload ceiling for one image (the API allows 32 MiB; see the module docstring). MAX_INLINE_IMAGE_BYTES = 8 * 1024 * 1024 @@ -74,3 +85,17 @@ def image_part_from_bytes(data: bytes, *, detail: ImageDetail = "auto") -> ChatI def image_part_from_url(url: str, *, detail: ImageDetail = "auto") -> ChatImagePart: """An image part for a public ``http(s)`` URL or an inline ``data:`` URL.""" return ChatImagePart(image_url=ChatImageUrl(url=url, detail=detail)) + + +def user_content( + text: str, + images: Sequence[ChatImagePart] | None = None, +) -> str | list[ChatContentPart]: + """A user turn's content: the text alone, or a part array when images ride along.""" + if not images: + return text + parts: list[ChatContentPart] = [] + if text: + parts.append(ChatTextPart(text=text)) + parts.extend(images) + return parts diff --git a/src/plyngent/cli/mentions.py b/src/plyngent/cli/mentions.py new file mode 100644 index 0000000..979b527 --- /dev/null +++ b/src/plyngent/cli/mentions.py @@ -0,0 +1,217 @@ +"""``@path`` mentions in a REPL line: attach the file the line refers to. + +A mention starts a whitespace-delimited token, so an e-mail address (`a@b`) is +not one. Both spellings work:: + + look at @shot.png and @src/plyngent/cli/repl.py + look at @"screens/my shot.png" + +The path resolves against the working directory (``~`` expands) and is checked +against the path denylist, like every tool path. What is attached depends on +the file: + +- an image (JPEG, PNG, GIF, WebP, sniffed from the bytes) becomes an image part, + so the model sees the picture itself. A model without + ``[providers.*.models.*].image_in`` skips it with a note instead of letting + the provider answer 400. +- any other file is read as text and appended to the message under a + ``--- @path ---`` header, capped like a tool result; the remainder stays + resumable through the ``truncate_token`` in the marker. + +Images and text both keep the typed ``@path`` in the message, so the model can +tell which attachment is which. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING + +from plyngent.agent.budget import DEFAULT_TOOL_RESULT_MAX_CHARS +from plyngent.agent.images import image_part_from_bytes, sniff_image_media_type +from plyngent.tools.truncate_token import truncate_to_memory + +if TYPE_CHECKING: + from plyngent.cli.state import ReplState + from plyngent.lmproto.openai_compatible.model import ChatImagePart + +MENTION = "@" +# One attached text file is capped like a tool result, and a turn's attachments +# together cannot outgrow this (the message is re-sent on every round). +MAX_MENTION_FILE_CHARS = DEFAULT_TOOL_RESULT_MAX_CHARS +MAX_MENTION_TOTAL_CHARS = 64_000 +MAX_PATH_COMPLETIONS = 100 + + +@dataclass(frozen=True, slots=True) +class Mention: + """One ``@path`` reference as it was typed.""" + + token: str + path: str + + +@dataclass(slots=True) +class MentionPayload: + """One turn's text plus what its mentions attached.""" + + text: str + images: list[ChatImagePart] = field(default_factory=list) + notes: list[str] = field(default_factory=list) + + +def parse_mentions(line: str) -> list[Mention]: + """Every ``@path`` mention in *line*, in the order typed.""" + mentions: list[Mention] = [] + index = 0 + length = len(line) + while index < length: + char = line[index] + if char != MENTION or (index > 0 and not line[index - 1].isspace()): + index += 1 + continue + start = index + index += 1 + if index < length and line[index] == '"': + end = line.find('"', index + 1) + if end == -1: + # An unterminated quote is still being typed: leave it alone. + break + path = line[index + 1 : end] + index = end + 1 + else: + while index < length and not line[index].isspace(): + index += 1 + path = line[start + 1 : index] + if path and not path.startswith(MENTION): + mentions.append(Mention(token=line[start:index], path=path)) + return mentions + + +def mention_token_for(path: str) -> str: + """The ``@path`` spelling for *path* (quoted when it holds a space).""" + if any(char.isspace() for char in path): + return f'{MENTION}"{path}"' + return f"{MENTION}{path}" + + +def resolve_mentions(state: ReplState, line: str) -> MentionPayload: + """Attach what *line*'s mentions point at; notes say what was skipped.""" + payload = MentionPayload(text=line) + mentions = parse_mentions(line) + seen: set[str] = set() + for mention in mentions: + if mention.path in seen: + continue + seen.add(mention.path) + budget = MAX_MENTION_TOTAL_CHARS - (len(payload.text) - len(line)) + note = _attach_mention(state, payload, mention, budget) + if note is not None: + payload.notes.append(note) + return payload + + +def _attach_mention( # noqa: PLR0911 — one early return per rejected file + state: ReplState, + payload: MentionPayload, + mention: Mention, + budget: int, +) -> str | None: + """Attach one mention; a returned string is a note for the user.""" + target = _resolve(state, mention.path) + if isinstance(target, str): + return target + if not target.exists(): + return f"{mention.token}: file not found" + if not target.is_file(): + return f"{mention.token}: not a file" + data = target.read_bytes() + if sniff_image_media_type(data) is not None: + return _attach_image(state, payload, mention, data) + if b"\x00" in data[:512]: + return f"{mention.token}: binary file skipped (only images and text are attached)" + if budget <= 0: + return f"{mention.token}: skipped, this turn's attachments are at the size cap" + return _attach_text(payload, mention, data, budget) + + +def _resolve(state: ReplState, path: str) -> Path | str: + """The absolute path for a mention, or a note when it is denylisted.""" + from plyngent.tools.context import bind_instance + from plyngent.tools.workspace import denylist_match + + target = Path(path).expanduser() + if not target.is_absolute(): + target = Path.cwd() / target + with bind_instance(state.instance_state): + denied = denylist_match(target) + if denied is not None: + return f"@{path}: matches the path denylist pattern {denied!r}" + return target + + +def _attach_image( + state: ReplState, + payload: MentionPayload, + mention: Mention, + data: bytes, +) -> str | None: + """Attach an image part, or explain why this turn cannot carry it.""" + from plyngent.config.routing import model_config_for + + model_config = model_config_for(state.provider, state.model) + if model_config is None or not model_config.image_in: + return ( + f"{mention.token}: skipped, model {state.model!r} does not accept image input " + f'(set image_in = true under [providers.{state.provider_name}.models."{state.model}"])' + ) + part = image_part_from_bytes(data, detail="auto") + if isinstance(part, str): + return f"{mention.token}: {part.removeprefix('error: ')}" + payload.images.append(part) + return None + + +def _attach_text(payload: MentionPayload, mention: Mention, data: bytes, budget: int) -> str | None: + """Append a text file to the message under its own header.""" + text = data.decode("utf-8", errors="replace") + # ``truncate_to_memory`` embeds the resumable token and marker itself. + body, token = truncate_to_memory(text, min(MAX_MENTION_FILE_CHARS, budget)) + header = f"--- {mention.token} ({len(text)} chars{', truncated' if token is not None else ''}) ---" + payload.text = f"{payload.text}\n\n{header}\n{body}" + return None + + +def complete_mention_paths(incomplete: str, *, cwd: Path | None = None) -> list[str]: + """Path candidates for the path part of a mention (directories end in ``/``).""" + base = cwd if cwd is not None else Path.cwd() + prefix = incomplete.rpartition("/")[2] + raw_dir = incomplete[: len(incomplete) - len(prefix)] + directory = Path(raw_dir).expanduser() if raw_dir else Path() + if not directory.is_absolute(): + directory = base / directory + try: + entries = sorted(os.scandir(directory), key=lambda entry: entry.name) + except OSError: + return [] + completions: list[str] = [] + for entry in entries: + if prefix and not entry.name.startswith(prefix): + continue + if not prefix and entry.name.startswith("."): + continue + suffix = "/" if entry.is_dir() else "" + completions.append(f"{raw_dir}{entry.name}{suffix}") + if len(completions) >= MAX_PATH_COMPLETIONS: + break + return completions + + +def mention_completions(token: str, *, cwd: Path | None = None) -> list[str]: + """Complete an ``@path`` token (including its ``@``) for readline.""" + if not token.startswith(MENTION): + return [] + paths = complete_mention_paths(token[len(MENTION) :], cwd=cwd) + return [mention_token_for(path) for path in paths] diff --git a/src/plyngent/cli/readline_setup.py b/src/plyngent/cli/readline_setup.py index 429cd5a..e0a31c6 100644 --- a/src/plyngent/cli/readline_setup.py +++ b/src/plyngent/cli/readline_setup.py @@ -45,12 +45,16 @@ def build_completer(state: ReplState) -> Callable[[str, int], str | None]: def completer(text: str, state_index: int) -> str | None: import readline + from plyngent.cli.mentions import mention_completions from plyngent.cli.slash import complete_slash_args buffer = readline.get_line_buffer() begidx = readline.get_begidx() + # An ``@path`` mention completes files wherever it sits in the line. + if text.startswith("@"): + options = mention_completions(text) # Completing the first token (command). - if begidx == 0 or (begidx > 0 and buffer[:begidx].strip() == ""): + elif begidx == 0 or (begidx > 0 and buffer[:begidx].strip() == ""): options = filter_prefix(text, slash_commands()) else: head = buffer[:begidx].strip() diff --git a/src/plyngent/cli/repl.py b/src/plyngent/cli/repl.py index 3b1c915..4c54496 100644 --- a/src/plyngent/cli/repl.py +++ b/src/plyngent/cli/repl.py @@ -34,6 +34,19 @@ def _echo_interrupted() -> None: click.echo() +async def _send_turn(state: ReplState, text: str) -> None: + """Send one user turn, attaching whatever its ``@path`` mentions point at.""" + from plyngent.cli.mentions import resolve_mentions + + payload = resolve_mentions(state, text) + for note in payload.notes: + click.secho(note, fg="yellow") + try: + _ = await run_user_text_with_retries(state.agent, payload.text, images=payload.images or None) + finally: + state.expire_yolo_once() + + async def run_repl(state: ReplState) -> None: """Interactive chat loop with readline editing, history, and Tab completion.""" setup_readline(state) @@ -79,16 +92,10 @@ async def run_repl(state: ReplState) -> None: text = state.pending_user_text state.pending_user_text = None _echo_user(text) - try: - _ = await run_user_text_with_retries(state.agent, text) - finally: - state.expire_yolo_once() + await _send_turn(state, text) continue _echo_user(entry) - try: - _ = await run_user_text_with_retries(state.agent, entry) - finally: - state.expire_yolo_once() + await _send_turn(state, entry) except KeyboardInterrupt: _echo_interrupted() diff --git a/src/plyngent/cli/retry.py b/src/plyngent/cli/retry.py index 28eafbc..01d1cf7 100644 --- a/src/plyngent/cli/retry.py +++ b/src/plyngent/cli/retry.py @@ -13,10 +13,11 @@ from plyngent.cli.interrupt import install_sigint_router, sigint_cancels from plyngent.cli.limits import reset_auto_continue_turn if TYPE_CHECKING: - from collections.abc import AsyncIterator, Callable, Coroutine + from collections.abc import AsyncIterator, Callable, Coroutine, Sequence from plyngent.agent import AgentEvent from plyngent.agent.chat import ChatAgent + from plyngent.lmproto.openai_compatible.model import ChatImagePart # Auto-retry budget after the first failure (10 attempts by default). DEFAULT_MAX_AUTO_RETRIES = 10 @@ -238,10 +239,11 @@ async def run_user_text_with_retries( agent: ChatAgent, text: str, *, + images: Sequence[ChatImagePart] | None = None, delays: tuple[float, ...] = DEFAULT_RETRY_DELAYS_SECONDS, ) -> bool: """Send a new user message with auto-retry (empty ``delays`` = no auto-retry).""" - return await run_turn_with_retries(agent, starter=lambda: agent.run(text), delays=delays) + return await run_turn_with_retries(agent, starter=lambda: agent.run(text, images=images), delays=delays) async def retry_pending_with_retries(agent: ChatAgent) -> bool: diff --git a/src/plyngent/config/models.py b/src/plyngent/config/models.py index e7a1eec..998b2f6 100644 --- a/src/plyngent/config/models.py +++ b/src/plyngent/config/models.py @@ -344,9 +344,9 @@ class ProviderConfig(Struct, tag_field="preset", omit_defaults=True): def _default_openai_models() -> dict[str, ModelConfig]: """Current OpenAI text catalog when TOML omits ``models``.""" return { - "gpt-5.4": ModelConfig(text=True), - "gpt-5.4-mini": ModelConfig(text=True), - "gpt-5.4-nano": ModelConfig(text=True), + "gpt-5.4": ModelConfig(text=True, image_in=True), + "gpt-5.4-mini": ModelConfig(text=True, image_in=True), + "gpt-5.4-nano": ModelConfig(text=True, image_in=True), } @@ -382,9 +382,14 @@ class AnthropicProvider(ProviderConfig, tag="anthropic"): def _default_deepseek_models() -> dict[str, ModelConfig]: - """Current DeepSeek text catalog when TOML omits ``models``.""" + """Current DeepSeek text catalog when TOML omits ``models``. + + Only ``deepseek-flash`` reads pictures (JPEG, PNG, GIF, WebP in a user + message); ``deepseek-v4-pro`` stays text-only, so ``image_in`` is the flag + that decides whether a turn may carry one. + """ return { - "deepseek-flash": ModelConfig(text=True), + "deepseek-flash": ModelConfig(text=True, image_in=True), "deepseek-v4-pro": ModelConfig(text=True), } diff --git a/tests/test_agent/test_images.py b/tests/test_agent/test_images.py new file mode 100644 index 0000000..d65dbce --- /dev/null +++ b/tests/test_agent/test_images.py @@ -0,0 +1,80 @@ +"""Image parts: format sniffing, data URLs, and the inline size policy.""" + +from __future__ import annotations + +import base64 + +from plyngent.agent.images import ( + MAX_INLINE_IMAGE_BYTES, + image_part_from_bytes, + image_part_from_url, + sniff_image_media_type, + split_data_url, + to_data_url, + user_content, +) +from plyngent.lmproto.openai_compatible.model import ChatImageUrl, ChatTextPart + +PNG = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" +) + + +def test_the_format_is_sniffed_from_the_bytes() -> None: + assert sniff_image_media_type(PNG) == "image/png" + assert sniff_image_media_type(b"\xff\xd8\xff\xe0JFIF") == "image/jpeg" + assert sniff_image_media_type(b"GIF89a....") == "image/gif" + assert sniff_image_media_type(b"RIFF\x00\x00\x00\x00WEBPVP8 ") == "image/webp" + assert sniff_image_media_type(b"not an image") is None + assert sniff_image_media_type(b"") is None + + +def test_a_data_url_round_trips() -> None: + url = to_data_url("image/png", PNG) + + assert url.startswith("data:image/png;base64,") + assert split_data_url(url) == ("image/png", base64.b64encode(PNG).decode("ascii")) + assert split_data_url("https://example.com/a.png") is None + + +def test_bytes_become_an_image_part_with_a_detail_level() -> None: + part = image_part_from_bytes(PNG, detail="low") + + assert not isinstance(part, str) + assert part.image_url.detail == "low" + assert part.image_url.url.startswith("data:image/png;base64,") + + +def test_non_image_bytes_are_an_error_text() -> None: + result = image_part_from_bytes(b"plain text") + + assert isinstance(result, str) + assert "not a JPEG, PNG, GIF, or WebP image" in result + + +def test_an_oversized_image_is_refused() -> None: + result = image_part_from_bytes(PNG + b"\x00" * (MAX_INLINE_IMAGE_BYTES + 1)) + + assert isinstance(result, str) + assert "inline limit" in result + + +def test_a_url_becomes_an_image_part() -> None: + part = image_part_from_url("https://example.com/a.jpg", detail="original") + + assert part.image_url == ChatImageUrl(url="https://example.com/a.jpg", detail="original") + + +def test_user_content_stays_a_string_without_images() -> None: + assert user_content("hello") == "hello" + assert user_content("hello", []) == "hello" + + +def test_user_content_becomes_a_part_array_with_images() -> None: + part = image_part_from_url("https://example.com/a.jpg") + + content = user_content("look", [part]) + + assert content == [ChatTextPart(text="look"), part] + # An image-only turn sends no empty text part. + assert user_content("", [part]) == [part] diff --git a/tests/test_cli/test_mentions.py b/tests/test_cli/test_mentions.py new file mode 100644 index 0000000..045daad --- /dev/null +++ b/tests/test_cli/test_mentions.py @@ -0,0 +1,88 @@ +"""``@path`` mentions: parsing, token spelling, and Tab completion.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from plyngent.cli.mentions import ( + complete_mention_paths, + mention_completions, + mention_token_for, + parse_mentions, +) + +if TYPE_CHECKING: + from pathlib import Path + + +def _paths(line: str) -> list[str]: + return [mention.path for mention in parse_mentions(line)] + + +def test_a_mention_is_a_whitespace_delimited_token() -> None: + assert _paths("look at @shot.png please") == ["shot.png"] + assert _paths("@a.png and @b/c.txt") == ["a.png", "b/c.txt"] + assert _paths("@a.png\n@b.png") == ["a.png", "b.png"] + + +def test_an_email_address_is_not_a_mention() -> None: + assert _paths("mail me at someone@example.com") == [] + + +def test_a_quoted_mention_may_hold_spaces() -> None: + assert _paths('look at @"screens/my shot.png"') == ["screens/my shot.png"] + assert _paths('@"a b.png" and @c.png') == ["a b.png", "c.png"] + + +def test_an_unterminated_quote_is_left_alone() -> None: + assert _paths('still typing @"my sh') == [] + + +def test_a_bare_at_sign_is_not_a_mention() -> None: + assert _paths("@ and @@@") == [] + + +def test_a_token_quotes_a_path_with_spaces() -> None: + assert mention_token_for("shot.png") == "@shot.png" + assert mention_token_for("my shot.png") == '@"my shot.png"' + + +def test_completion_lists_a_directory(tmp_path: Path) -> None: + (tmp_path / "shot.png").write_bytes(b"") + (tmp_path / "screens").mkdir() + + completions = complete_mention_paths("sc", cwd=tmp_path) + + assert completions == ["screens/"] + + +def test_completion_continues_through_a_directory(tmp_path: Path) -> None: + (tmp_path / "screens").mkdir() + (tmp_path / "screens" / "shot.png").write_bytes(b"") + + assert complete_mention_paths("screens/sh", cwd=tmp_path) == ["screens/shot.png"] + assert complete_mention_paths("screens/", cwd=tmp_path) == ["screens/shot.png"] + + +def test_completion_hides_dot_files_unless_asked(tmp_path: Path) -> None: + (tmp_path / ".env").write_text("x", encoding="utf-8") + + assert complete_mention_paths("", cwd=tmp_path) == [] + assert complete_mention_paths(".", cwd=tmp_path) == [".env"] + + +def test_completion_ignores_a_missing_directory(tmp_path: Path) -> None: + assert complete_mention_paths("nope/x", cwd=tmp_path) == [] + + +def test_completions_keep_the_at_sign_and_quote_spaces(tmp_path: Path) -> None: + (tmp_path / "my shot.png").write_bytes(b"") + (tmp_path / "shots").mkdir() + + assert mention_completions("@my", cwd=tmp_path) == ['@"my shot.png"'] + assert mention_completions("@shots", cwd=tmp_path) == ["@shots/"] + assert mention_completions("@", cwd=tmp_path) == ['@"my shot.png"', "@shots/"] + + +def test_mention_completions_ignores_a_plain_word() -> None: + assert mention_completions("shot") == [] diff --git a/tests/test_cli/test_readline_setup.py b/tests/test_cli/test_readline_setup.py index 4a57cca..a91d618 100644 --- a/tests/test_cli/test_readline_setup.py +++ b/tests/test_cli/test_readline_setup.py @@ -170,6 +170,46 @@ def test_completer_stream_on_off(tmp_path: object, monkeypatch: object) -> None: assert completer("o", 1) == "off" +def test_completer_path_mentions(tmp_path: object, monkeypatch: object) -> None: + """An ``@path`` token completes files, wherever it sits in the line.""" + import readline + from pathlib import Path + + import pytest + + assert isinstance(tmp_path, Path) + assert isinstance(monkeypatch, pytest.MonkeyPatch) + (tmp_path / "shot.png").write_bytes(b"") + (tmp_path / "screens").mkdir() + + state = _minimal_state(tmp_path) + completer = build_completer(state) + monkeypatch.setattr(readline, "get_line_buffer", lambda: "look at @sh") + monkeypatch.setattr(readline, "get_begidx", lambda: len("look at ")) + monkeypatch.chdir(tmp_path) + + found = [item for index in range(10) if (item := completer("@sh", index)) is not None] + assert found == ["@shot.png"] + + +def test_a_plain_word_is_still_a_slash_argument(tmp_path: object, monkeypatch: object) -> None: + """The mention branch must not swallow ordinary argument completion.""" + import readline + from pathlib import Path + + import pytest + + assert isinstance(tmp_path, Path) + assert isinstance(monkeypatch, pytest.MonkeyPatch) + + state = _minimal_state(tmp_path) + completer = build_completer(state) + monkeypatch.setattr(readline, "get_line_buffer", lambda: "/provider r") + monkeypatch.setattr(readline, "get_begidx", lambda: len("/provider ")) + + assert completer("r", 0) == "remote" + + def test_complete_slash_args_from_registry(tmp_path: object) -> None: from pathlib import Path diff --git a/tests/test_cli/test_repl_commands.py b/tests/test_cli/test_repl_commands.py index f7c480f..bbb570d 100644 --- a/tests/test_cli/test_repl_commands.py +++ b/tests/test_cli/test_repl_commands.py @@ -1,14 +1,17 @@ from __future__ import annotations +import base64 from typing import TYPE_CHECKING, Literal, overload import pytest import tomlkit from plyngent.agent import ChatAgent, Notice +from plyngent.cli.mentions import resolve_mentions +from plyngent.cli.repl import _send_turn from plyngent.cli.slash import handle_slash from plyngent.cli.state import ReplState -from plyngent.config.models import DatabaseConfig, OpenAIProvider +from plyngent.config.models import DatabaseConfig, ModelConfig, OpenAIProvider from plyngent.config.store import ConfigStore from plyngent.lmproto.openai_compatible.model import ( AssistantChatMessage, @@ -16,6 +19,9 @@ from plyngent.lmproto.openai_compatible.model import ( ChatCompletionChunk, ChatCompletionResponse, ChatCompletionsParam, + ChatImagePart, + ChatTextPart, + UserChatMessage, ) from plyngent.memory import MemoryStore @@ -24,6 +30,17 @@ if TYPE_CHECKING: from pathlib import Path +# A real 1x1 PNG: ``/image`` sniffs the format from these bytes. +_PNG = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" +) + + +def _accept_images(state: ReplState) -> None: + """Mark the fixture's model as taking pictures (the config default is text-only).""" + state.provider.models[state.model] = ModelConfig(image_in=True) + + class DummyClient: @overload async def chat_completions( @@ -543,7 +560,7 @@ async def test_repl_ctrl_c_from_turn_returns_to_prompt( raise EOFError return "hello" - async def interrupted_turn(_agent: object, _text: str) -> bool: + async def interrupted_turn(_agent: object, _text: str, **_kwargs: object) -> bool: raise KeyboardInterrupt monkeypatch.setattr(repl_mod, "setup_readline", lambda _state: None) @@ -573,7 +590,7 @@ async def test_repl_second_ctrl_c_after_cancel_returns_to_prompt( raise EOFError return reads[calls["reads"] - 1] - async def cancel_turn(_agent: object, _text: str) -> bool: + async def cancel_turn(_agent: object, _text: str, **_kwargs: object) -> bool: raise KeyboardInterrupt # first Ctrl+C: cancels every turn def echo_user(text: str) -> None: @@ -780,3 +797,61 @@ async def test_todos_update_takes_a_listing_number(state: ReplState, capsys: pyt assert await handle_slash(state, "/todos pop") is True assert "popped TOP group (first, second)" in capsys.readouterr().out + + +async def test_a_mention_attaches_the_picture_to_the_turn(state: ReplState, tmp_path: Path) -> None: + _accept_images(state) + shot = tmp_path / "shot.png" + _ = shot.write_bytes(_PNG) + state.agent.stream = False + + payload = resolve_mentions(state, f"look at @{shot}") + assert payload.notes == [] + assert len(payload.images) == 1 + assert payload.text == f"look at @{shot}" + + await _send_turn(state, payload.text) + user = next(m for m in state.agent.messages if isinstance(m, UserChatMessage)) + assert isinstance(user.content, list) + text_part, image_part = user.content + assert isinstance(text_part, ChatTextPart) + assert text_part.text == f"look at @{shot}" + assert isinstance(image_part, ChatImagePart) + assert image_part.image_url.url.startswith("data:image/png;base64,") + + +async def test_a_mention_inlines_a_text_file(state: ReplState, tmp_path: Path) -> None: + notes = tmp_path / "notes.txt" + _ = notes.write_text("the deploy needs acme", encoding="utf-8") + + payload = resolve_mentions(state, f"read @{notes}") + + assert payload.images == [] + assert payload.notes == [] + assert payload.text.startswith(f"read @{notes}") + assert f"--- @{notes} (21 chars) ---" in payload.text + assert payload.text.endswith("the deploy needs acme") + + +async def test_a_mention_of_a_missing_file_is_a_note(state: ReplState, tmp_path: Path) -> None: + payload = resolve_mentions(state, "@nope.png is gone") + + assert payload.images == [] + assert payload.notes == ["@nope.png: file not found"] + assert payload.text == "@nope.png is gone" + + +async def test_a_picture_mention_is_skipped_for_a_text_only_model( + state: ReplState, + tmp_path: Path, +) -> None: + """``image_in`` is the gate: a text-only model never receives a picture.""" + shot = tmp_path / "shot.png" + _ = shot.write_bytes(_PNG) + + payload = resolve_mentions(state, f"@{shot}") + + assert payload.images == [] + note = payload.notes[0] + assert "does not accept image input" in note + assert "image_in = true" in note