mirror of
https://github.com/NCBM/plyngent.git
synced 2026-11-07 23:47:54 +08:00
core/cli+config: attach files to a turn with @path mentions
Typing `@shot.png` in the line attaches that picture to the turn — no separate command — and `@notes.txt` appends the file as text under a `--- @path ---` header, capped like a tool result with a resumable `truncate_token`. `@"my shot.png"` is the quoted form, and Tab completes the path after `@` (directories end in `/`, a name with a space completes quoted). An image is skipped, with a note naming the key to set, on a model without `image_in`; a missing or denylisted path is a note too, and the turn still goes. The default catalogs mark what can take a picture (`deepseek-flash` yes, `deepseek-v4-pro` no, the OpenAI defaults yes). `Agent.run` takes the parts, so the turn is stored with its images and shows up as `[image]` wherever only prose is wanted.
This commit is contained in:
@@ -0,0 +1,80 @@
|
||||
"""Image parts: format sniffing, data URLs, and the inline size policy."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
|
||||
from plyngent.agent.images import (
|
||||
MAX_INLINE_IMAGE_BYTES,
|
||||
image_part_from_bytes,
|
||||
image_part_from_url,
|
||||
sniff_image_media_type,
|
||||
split_data_url,
|
||||
to_data_url,
|
||||
user_content,
|
||||
)
|
||||
from plyngent.lmproto.openai_compatible.model import ChatImageUrl, ChatTextPart
|
||||
|
||||
PNG = base64.b64decode(
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="
|
||||
)
|
||||
|
||||
|
||||
def test_the_format_is_sniffed_from_the_bytes() -> None:
|
||||
assert sniff_image_media_type(PNG) == "image/png"
|
||||
assert sniff_image_media_type(b"\xff\xd8\xff\xe0JFIF") == "image/jpeg"
|
||||
assert sniff_image_media_type(b"GIF89a....") == "image/gif"
|
||||
assert sniff_image_media_type(b"RIFF\x00\x00\x00\x00WEBPVP8 ") == "image/webp"
|
||||
assert sniff_image_media_type(b"not an image") is None
|
||||
assert sniff_image_media_type(b"") is None
|
||||
|
||||
|
||||
def test_a_data_url_round_trips() -> None:
|
||||
url = to_data_url("image/png", PNG)
|
||||
|
||||
assert url.startswith("data:image/png;base64,")
|
||||
assert split_data_url(url) == ("image/png", base64.b64encode(PNG).decode("ascii"))
|
||||
assert split_data_url("https://example.com/a.png") is None
|
||||
|
||||
|
||||
def test_bytes_become_an_image_part_with_a_detail_level() -> None:
|
||||
part = image_part_from_bytes(PNG, detail="low")
|
||||
|
||||
assert not isinstance(part, str)
|
||||
assert part.image_url.detail == "low"
|
||||
assert part.image_url.url.startswith("data:image/png;base64,")
|
||||
|
||||
|
||||
def test_non_image_bytes_are_an_error_text() -> None:
|
||||
result = image_part_from_bytes(b"plain text")
|
||||
|
||||
assert isinstance(result, str)
|
||||
assert "not a JPEG, PNG, GIF, or WebP image" in result
|
||||
|
||||
|
||||
def test_an_oversized_image_is_refused() -> None:
|
||||
result = image_part_from_bytes(PNG + b"\x00" * (MAX_INLINE_IMAGE_BYTES + 1))
|
||||
|
||||
assert isinstance(result, str)
|
||||
assert "inline limit" in result
|
||||
|
||||
|
||||
def test_a_url_becomes_an_image_part() -> None:
|
||||
part = image_part_from_url("https://example.com/a.jpg", detail="original")
|
||||
|
||||
assert part.image_url == ChatImageUrl(url="https://example.com/a.jpg", detail="original")
|
||||
|
||||
|
||||
def test_user_content_stays_a_string_without_images() -> None:
|
||||
assert user_content("hello") == "hello"
|
||||
assert user_content("hello", []) == "hello"
|
||||
|
||||
|
||||
def test_user_content_becomes_a_part_array_with_images() -> None:
|
||||
part = image_part_from_url("https://example.com/a.jpg")
|
||||
|
||||
content = user_content("look", [part])
|
||||
|
||||
assert content == [ChatTextPart(text="look"), part]
|
||||
# An image-only turn sends no empty text part.
|
||||
assert user_content("", [part]) == [part]
|
||||
@@ -0,0 +1,88 @@
|
||||
"""``@path`` mentions: parsing, token spelling, and Tab completion."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from plyngent.cli.mentions import (
|
||||
complete_mention_paths,
|
||||
mention_completions,
|
||||
mention_token_for,
|
||||
parse_mentions,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _paths(line: str) -> list[str]:
|
||||
return [mention.path for mention in parse_mentions(line)]
|
||||
|
||||
|
||||
def test_a_mention_is_a_whitespace_delimited_token() -> None:
|
||||
assert _paths("look at @shot.png please") == ["shot.png"]
|
||||
assert _paths("@a.png and @b/c.txt") == ["a.png", "b/c.txt"]
|
||||
assert _paths("@a.png\n@b.png") == ["a.png", "b.png"]
|
||||
|
||||
|
||||
def test_an_email_address_is_not_a_mention() -> None:
|
||||
assert _paths("mail me at someone@example.com") == []
|
||||
|
||||
|
||||
def test_a_quoted_mention_may_hold_spaces() -> None:
|
||||
assert _paths('look at @"screens/my shot.png"') == ["screens/my shot.png"]
|
||||
assert _paths('@"a b.png" and @c.png') == ["a b.png", "c.png"]
|
||||
|
||||
|
||||
def test_an_unterminated_quote_is_left_alone() -> None:
|
||||
assert _paths('still typing @"my sh') == []
|
||||
|
||||
|
||||
def test_a_bare_at_sign_is_not_a_mention() -> None:
|
||||
assert _paths("@ and @@@") == []
|
||||
|
||||
|
||||
def test_a_token_quotes_a_path_with_spaces() -> None:
|
||||
assert mention_token_for("shot.png") == "@shot.png"
|
||||
assert mention_token_for("my shot.png") == '@"my shot.png"'
|
||||
|
||||
|
||||
def test_completion_lists_a_directory(tmp_path: Path) -> None:
|
||||
(tmp_path / "shot.png").write_bytes(b"")
|
||||
(tmp_path / "screens").mkdir()
|
||||
|
||||
completions = complete_mention_paths("sc", cwd=tmp_path)
|
||||
|
||||
assert completions == ["screens/"]
|
||||
|
||||
|
||||
def test_completion_continues_through_a_directory(tmp_path: Path) -> None:
|
||||
(tmp_path / "screens").mkdir()
|
||||
(tmp_path / "screens" / "shot.png").write_bytes(b"")
|
||||
|
||||
assert complete_mention_paths("screens/sh", cwd=tmp_path) == ["screens/shot.png"]
|
||||
assert complete_mention_paths("screens/", cwd=tmp_path) == ["screens/shot.png"]
|
||||
|
||||
|
||||
def test_completion_hides_dot_files_unless_asked(tmp_path: Path) -> None:
|
||||
(tmp_path / ".env").write_text("x", encoding="utf-8")
|
||||
|
||||
assert complete_mention_paths("", cwd=tmp_path) == []
|
||||
assert complete_mention_paths(".", cwd=tmp_path) == [".env"]
|
||||
|
||||
|
||||
def test_completion_ignores_a_missing_directory(tmp_path: Path) -> None:
|
||||
assert complete_mention_paths("nope/x", cwd=tmp_path) == []
|
||||
|
||||
|
||||
def test_completions_keep_the_at_sign_and_quote_spaces(tmp_path: Path) -> None:
|
||||
(tmp_path / "my shot.png").write_bytes(b"")
|
||||
(tmp_path / "shots").mkdir()
|
||||
|
||||
assert mention_completions("@my", cwd=tmp_path) == ['@"my shot.png"']
|
||||
assert mention_completions("@shots", cwd=tmp_path) == ["@shots/"]
|
||||
assert mention_completions("@", cwd=tmp_path) == ['@"my shot.png"', "@shots/"]
|
||||
|
||||
|
||||
def test_mention_completions_ignores_a_plain_word() -> None:
|
||||
assert mention_completions("shot") == []
|
||||
@@ -170,6 +170,46 @@ def test_completer_stream_on_off(tmp_path: object, monkeypatch: object) -> None:
|
||||
assert completer("o", 1) == "off"
|
||||
|
||||
|
||||
def test_completer_path_mentions(tmp_path: object, monkeypatch: object) -> None:
|
||||
"""An ``@path`` token completes files, wherever it sits in the line."""
|
||||
import readline
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
assert isinstance(tmp_path, Path)
|
||||
assert isinstance(monkeypatch, pytest.MonkeyPatch)
|
||||
(tmp_path / "shot.png").write_bytes(b"")
|
||||
(tmp_path / "screens").mkdir()
|
||||
|
||||
state = _minimal_state(tmp_path)
|
||||
completer = build_completer(state)
|
||||
monkeypatch.setattr(readline, "get_line_buffer", lambda: "look at @sh")
|
||||
monkeypatch.setattr(readline, "get_begidx", lambda: len("look at "))
|
||||
monkeypatch.chdir(tmp_path)
|
||||
|
||||
found = [item for index in range(10) if (item := completer("@sh", index)) is not None]
|
||||
assert found == ["@shot.png"]
|
||||
|
||||
|
||||
def test_a_plain_word_is_still_a_slash_argument(tmp_path: object, monkeypatch: object) -> None:
|
||||
"""The mention branch must not swallow ordinary argument completion."""
|
||||
import readline
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
assert isinstance(tmp_path, Path)
|
||||
assert isinstance(monkeypatch, pytest.MonkeyPatch)
|
||||
|
||||
state = _minimal_state(tmp_path)
|
||||
completer = build_completer(state)
|
||||
monkeypatch.setattr(readline, "get_line_buffer", lambda: "/provider r")
|
||||
monkeypatch.setattr(readline, "get_begidx", lambda: len("/provider "))
|
||||
|
||||
assert completer("r", 0) == "remote"
|
||||
|
||||
|
||||
def test_complete_slash_args_from_registry(tmp_path: object) -> None:
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
@@ -1,14 +1,17 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
from typing import TYPE_CHECKING, Literal, overload
|
||||
|
||||
import pytest
|
||||
import tomlkit
|
||||
|
||||
from plyngent.agent import ChatAgent, Notice
|
||||
from plyngent.cli.mentions import resolve_mentions
|
||||
from plyngent.cli.repl import _send_turn
|
||||
from plyngent.cli.slash import handle_slash
|
||||
from plyngent.cli.state import ReplState
|
||||
from plyngent.config.models import DatabaseConfig, OpenAIProvider
|
||||
from plyngent.config.models import DatabaseConfig, ModelConfig, OpenAIProvider
|
||||
from plyngent.config.store import ConfigStore
|
||||
from plyngent.lmproto.openai_compatible.model import (
|
||||
AssistantChatMessage,
|
||||
@@ -16,6 +19,9 @@ from plyngent.lmproto.openai_compatible.model import (
|
||||
ChatCompletionChunk,
|
||||
ChatCompletionResponse,
|
||||
ChatCompletionsParam,
|
||||
ChatImagePart,
|
||||
ChatTextPart,
|
||||
UserChatMessage,
|
||||
)
|
||||
from plyngent.memory import MemoryStore
|
||||
|
||||
@@ -24,6 +30,17 @@ if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
# A real 1x1 PNG: ``/image`` sniffs the format from these bytes.
|
||||
_PNG = base64.b64decode(
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="
|
||||
)
|
||||
|
||||
|
||||
def _accept_images(state: ReplState) -> None:
|
||||
"""Mark the fixture's model as taking pictures (the config default is text-only)."""
|
||||
state.provider.models[state.model] = ModelConfig(image_in=True)
|
||||
|
||||
|
||||
class DummyClient:
|
||||
@overload
|
||||
async def chat_completions(
|
||||
@@ -543,7 +560,7 @@ async def test_repl_ctrl_c_from_turn_returns_to_prompt(
|
||||
raise EOFError
|
||||
return "hello"
|
||||
|
||||
async def interrupted_turn(_agent: object, _text: str) -> bool:
|
||||
async def interrupted_turn(_agent: object, _text: str, **_kwargs: object) -> bool:
|
||||
raise KeyboardInterrupt
|
||||
|
||||
monkeypatch.setattr(repl_mod, "setup_readline", lambda _state: None)
|
||||
@@ -573,7 +590,7 @@ async def test_repl_second_ctrl_c_after_cancel_returns_to_prompt(
|
||||
raise EOFError
|
||||
return reads[calls["reads"] - 1]
|
||||
|
||||
async def cancel_turn(_agent: object, _text: str) -> bool:
|
||||
async def cancel_turn(_agent: object, _text: str, **_kwargs: object) -> bool:
|
||||
raise KeyboardInterrupt # first Ctrl+C: cancels every turn
|
||||
|
||||
def echo_user(text: str) -> None:
|
||||
@@ -780,3 +797,61 @@ async def test_todos_update_takes_a_listing_number(state: ReplState, capsys: pyt
|
||||
|
||||
assert await handle_slash(state, "/todos pop") is True
|
||||
assert "popped TOP group (first, second)" in capsys.readouterr().out
|
||||
|
||||
|
||||
async def test_a_mention_attaches_the_picture_to_the_turn(state: ReplState, tmp_path: Path) -> None:
|
||||
_accept_images(state)
|
||||
shot = tmp_path / "shot.png"
|
||||
_ = shot.write_bytes(_PNG)
|
||||
state.agent.stream = False
|
||||
|
||||
payload = resolve_mentions(state, f"look at @{shot}")
|
||||
assert payload.notes == []
|
||||
assert len(payload.images) == 1
|
||||
assert payload.text == f"look at @{shot}"
|
||||
|
||||
await _send_turn(state, payload.text)
|
||||
user = next(m for m in state.agent.messages if isinstance(m, UserChatMessage))
|
||||
assert isinstance(user.content, list)
|
||||
text_part, image_part = user.content
|
||||
assert isinstance(text_part, ChatTextPart)
|
||||
assert text_part.text == f"look at @{shot}"
|
||||
assert isinstance(image_part, ChatImagePart)
|
||||
assert image_part.image_url.url.startswith("data:image/png;base64,")
|
||||
|
||||
|
||||
async def test_a_mention_inlines_a_text_file(state: ReplState, tmp_path: Path) -> None:
|
||||
notes = tmp_path / "notes.txt"
|
||||
_ = notes.write_text("the deploy needs acme", encoding="utf-8")
|
||||
|
||||
payload = resolve_mentions(state, f"read @{notes}")
|
||||
|
||||
assert payload.images == []
|
||||
assert payload.notes == []
|
||||
assert payload.text.startswith(f"read @{notes}")
|
||||
assert f"--- @{notes} (21 chars) ---" in payload.text
|
||||
assert payload.text.endswith("the deploy needs acme")
|
||||
|
||||
|
||||
async def test_a_mention_of_a_missing_file_is_a_note(state: ReplState, tmp_path: Path) -> None:
|
||||
payload = resolve_mentions(state, "@nope.png is gone")
|
||||
|
||||
assert payload.images == []
|
||||
assert payload.notes == ["@nope.png: file not found"]
|
||||
assert payload.text == "@nope.png is gone"
|
||||
|
||||
|
||||
async def test_a_picture_mention_is_skipped_for_a_text_only_model(
|
||||
state: ReplState,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""``image_in`` is the gate: a text-only model never receives a picture."""
|
||||
shot = tmp_path / "shot.png"
|
||||
_ = shot.write_bytes(_PNG)
|
||||
|
||||
payload = resolve_mentions(state, f"@{shot}")
|
||||
|
||||
assert payload.images == []
|
||||
note = payload.notes[0]
|
||||
assert "does not accept image input" in note
|
||||
assert "image_in = true" in note
|
||||
|
||||
Reference in New Issue
Block a user