mirror of
https://github.com/NCBM/plyngent.git
synced 2026-11-07 23:47:54 +08:00
core/cli+config: attach files to a turn with @path mentions
Typing `@shot.png` in the line attaches that picture to the turn — no separate command — and `@notes.txt` appends the file as text under a `--- @path ---` header, capped like a tool result with a resumable `truncate_token`. `@"my shot.png"` is the quoted form, and Tab completes the path after `@` (directories end in `/`, a name with a space completes quoted). An image is skipped, with a note naming the key to set, on a model without `image_in`; a missing or denylisted path is a note too, and the turn still goes. The default catalogs mark what can take a picture (`deepseek-flash` yes, `deepseek-v4-pro` no, the OpenAI defaults yes). `Agent.run` takes the parts, so the turn is stored with its images and shows up as `[image]` wherever only prose is wanted.
This commit is contained in:
@@ -28,6 +28,7 @@ from .directive_checkpoint import (
|
|||||||
parse_checkpoint_bands,
|
parse_checkpoint_bands,
|
||||||
)
|
)
|
||||||
from .events import UsageEvent
|
from .events import UsageEvent
|
||||||
|
from .images import user_content
|
||||||
from .loop import DEFAULT_MAX_ROUNDS, DEFAULT_MAX_TRUNCATION_CONTINUES, run_chat_loop
|
from .loop import DEFAULT_MAX_ROUNDS, DEFAULT_MAX_TRUNCATION_CONTINUES, run_chat_loop
|
||||||
from .notices import TRUNCATED_NOTICE_KIND, notice_of_kind
|
from .notices import TRUNCATED_NOTICE_KIND, notice_of_kind
|
||||||
from .todo_nag import (
|
from .todo_nag import (
|
||||||
@@ -41,7 +42,7 @@ from .usage import TokenUsage
|
|||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from collections.abc import AsyncIterator, Awaitable, Callable, Sequence
|
from collections.abc import AsyncIterator, Awaitable, Callable, Sequence
|
||||||
|
|
||||||
from plyngent.lmproto.openai_compatible.model import AnyAssistantToolCall, AnyChatMessage
|
from plyngent.lmproto.openai_compatible.model import AnyAssistantToolCall, AnyChatMessage, ChatImagePart
|
||||||
from plyngent.memory import MemoryStore
|
from plyngent.memory import MemoryStore
|
||||||
|
|
||||||
from .events import AgentEvent
|
from .events import AgentEvent
|
||||||
@@ -664,11 +665,20 @@ class ChatAgent:
|
|||||||
await self._persist_range(self._persist_from, len(self.messages))
|
await self._persist_range(self._persist_from, len(self.messages))
|
||||||
await self._persist_context_usage()
|
await self._persist_context_usage()
|
||||||
|
|
||||||
async def run(self, user_text: str) -> AsyncIterator[AgentEvent]:
|
async def run(
|
||||||
"""Append a user message (persist immediately), run the tool loop, yield events."""
|
self,
|
||||||
|
user_text: str,
|
||||||
|
*,
|
||||||
|
images: Sequence[ChatImagePart] | None = None,
|
||||||
|
) -> AsyncIterator[AgentEvent]:
|
||||||
|
"""Append a user message (persist immediately), run the tool loop, yield events.
|
||||||
|
|
||||||
|
*images* ride along with the text as content parts, which is the shape
|
||||||
|
the model sees an attachment in (see ``agent.images``).
|
||||||
|
"""
|
||||||
_ = reset_lineno_tracker()
|
_ = reset_lineno_tracker()
|
||||||
self._ensure_system_prompt()
|
self._ensure_system_prompt()
|
||||||
user_msg = UserChatMessage(content=user_text)
|
user_msg = UserChatMessage(content=user_content(user_text, images))
|
||||||
self.messages.append(user_msg)
|
self.messages.append(user_msg)
|
||||||
await self._persist(user_msg)
|
await self._persist(user_msg)
|
||||||
self._persist_from = len(self.messages)
|
self._persist_from = len(self.messages)
|
||||||
|
|||||||
@@ -14,8 +14,19 @@ and over. A larger image has to be resized first.
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import base64
|
import base64
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
from plyngent.lmproto.openai_compatible.model import ChatImagePart, ChatImageUrl, ImageDetail
|
from plyngent.lmproto.openai_compatible.model import (
|
||||||
|
ChatImagePart,
|
||||||
|
ChatImageUrl,
|
||||||
|
ChatTextPart,
|
||||||
|
ImageDetail,
|
||||||
|
)
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from collections.abc import Sequence
|
||||||
|
|
||||||
|
from plyngent.lmproto.openai_compatible.model import ChatContentPart
|
||||||
|
|
||||||
# Inline payload ceiling for one image (the API allows 32 MiB; see the module docstring).
|
# Inline payload ceiling for one image (the API allows 32 MiB; see the module docstring).
|
||||||
MAX_INLINE_IMAGE_BYTES = 8 * 1024 * 1024
|
MAX_INLINE_IMAGE_BYTES = 8 * 1024 * 1024
|
||||||
@@ -74,3 +85,17 @@ def image_part_from_bytes(data: bytes, *, detail: ImageDetail = "auto") -> ChatI
|
|||||||
def image_part_from_url(url: str, *, detail: ImageDetail = "auto") -> ChatImagePart:
|
def image_part_from_url(url: str, *, detail: ImageDetail = "auto") -> ChatImagePart:
|
||||||
"""An image part for a public ``http(s)`` URL or an inline ``data:`` URL."""
|
"""An image part for a public ``http(s)`` URL or an inline ``data:`` URL."""
|
||||||
return ChatImagePart(image_url=ChatImageUrl(url=url, detail=detail))
|
return ChatImagePart(image_url=ChatImageUrl(url=url, detail=detail))
|
||||||
|
|
||||||
|
|
||||||
|
def user_content(
|
||||||
|
text: str,
|
||||||
|
images: Sequence[ChatImagePart] | None = None,
|
||||||
|
) -> str | list[ChatContentPart]:
|
||||||
|
"""A user turn's content: the text alone, or a part array when images ride along."""
|
||||||
|
if not images:
|
||||||
|
return text
|
||||||
|
parts: list[ChatContentPart] = []
|
||||||
|
if text:
|
||||||
|
parts.append(ChatTextPart(text=text))
|
||||||
|
parts.extend(images)
|
||||||
|
return parts
|
||||||
|
|||||||
@@ -0,0 +1,217 @@
|
|||||||
|
"""``@path`` mentions in a REPL line: attach the file the line refers to.
|
||||||
|
|
||||||
|
A mention starts a whitespace-delimited token, so an e-mail address (`a@b`) is
|
||||||
|
not one. Both spellings work::
|
||||||
|
|
||||||
|
look at @shot.png and @src/plyngent/cli/repl.py
|
||||||
|
look at @"screens/my shot.png"
|
||||||
|
|
||||||
|
The path resolves against the working directory (``~`` expands) and is checked
|
||||||
|
against the path denylist, like every tool path. What is attached depends on
|
||||||
|
the file:
|
||||||
|
|
||||||
|
- an image (JPEG, PNG, GIF, WebP, sniffed from the bytes) becomes an image part,
|
||||||
|
so the model sees the picture itself. A model without
|
||||||
|
``[providers.*.models.*].image_in`` skips it with a note instead of letting
|
||||||
|
the provider answer 400.
|
||||||
|
- any other file is read as text and appended to the message under a
|
||||||
|
``--- @path ---`` header, capped like a tool result; the remainder stays
|
||||||
|
resumable through the ``truncate_token`` in the marker.
|
||||||
|
|
||||||
|
Images and text both keep the typed ``@path`` in the message, so the model can
|
||||||
|
tell which attachment is which.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
from plyngent.agent.budget import DEFAULT_TOOL_RESULT_MAX_CHARS
|
||||||
|
from plyngent.agent.images import image_part_from_bytes, sniff_image_media_type
|
||||||
|
from plyngent.tools.truncate_token import truncate_to_memory
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from plyngent.cli.state import ReplState
|
||||||
|
from plyngent.lmproto.openai_compatible.model import ChatImagePart
|
||||||
|
|
||||||
|
MENTION = "@"
|
||||||
|
# One attached text file is capped like a tool result, and a turn's attachments
|
||||||
|
# together cannot outgrow this (the message is re-sent on every round).
|
||||||
|
MAX_MENTION_FILE_CHARS = DEFAULT_TOOL_RESULT_MAX_CHARS
|
||||||
|
MAX_MENTION_TOTAL_CHARS = 64_000
|
||||||
|
MAX_PATH_COMPLETIONS = 100
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class Mention:
|
||||||
|
"""One ``@path`` reference as it was typed."""
|
||||||
|
|
||||||
|
token: str
|
||||||
|
path: str
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(slots=True)
|
||||||
|
class MentionPayload:
|
||||||
|
"""One turn's text plus what its mentions attached."""
|
||||||
|
|
||||||
|
text: str
|
||||||
|
images: list[ChatImagePart] = field(default_factory=list)
|
||||||
|
notes: list[str] = field(default_factory=list)
|
||||||
|
|
||||||
|
|
||||||
|
def parse_mentions(line: str) -> list[Mention]:
|
||||||
|
"""Every ``@path`` mention in *line*, in the order typed."""
|
||||||
|
mentions: list[Mention] = []
|
||||||
|
index = 0
|
||||||
|
length = len(line)
|
||||||
|
while index < length:
|
||||||
|
char = line[index]
|
||||||
|
if char != MENTION or (index > 0 and not line[index - 1].isspace()):
|
||||||
|
index += 1
|
||||||
|
continue
|
||||||
|
start = index
|
||||||
|
index += 1
|
||||||
|
if index < length and line[index] == '"':
|
||||||
|
end = line.find('"', index + 1)
|
||||||
|
if end == -1:
|
||||||
|
# An unterminated quote is still being typed: leave it alone.
|
||||||
|
break
|
||||||
|
path = line[index + 1 : end]
|
||||||
|
index = end + 1
|
||||||
|
else:
|
||||||
|
while index < length and not line[index].isspace():
|
||||||
|
index += 1
|
||||||
|
path = line[start + 1 : index]
|
||||||
|
if path and not path.startswith(MENTION):
|
||||||
|
mentions.append(Mention(token=line[start:index], path=path))
|
||||||
|
return mentions
|
||||||
|
|
||||||
|
|
||||||
|
def mention_token_for(path: str) -> str:
|
||||||
|
"""The ``@path`` spelling for *path* (quoted when it holds a space)."""
|
||||||
|
if any(char.isspace() for char in path):
|
||||||
|
return f'{MENTION}"{path}"'
|
||||||
|
return f"{MENTION}{path}"
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_mentions(state: ReplState, line: str) -> MentionPayload:
|
||||||
|
"""Attach what *line*'s mentions point at; notes say what was skipped."""
|
||||||
|
payload = MentionPayload(text=line)
|
||||||
|
mentions = parse_mentions(line)
|
||||||
|
seen: set[str] = set()
|
||||||
|
for mention in mentions:
|
||||||
|
if mention.path in seen:
|
||||||
|
continue
|
||||||
|
seen.add(mention.path)
|
||||||
|
budget = MAX_MENTION_TOTAL_CHARS - (len(payload.text) - len(line))
|
||||||
|
note = _attach_mention(state, payload, mention, budget)
|
||||||
|
if note is not None:
|
||||||
|
payload.notes.append(note)
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
def _attach_mention( # noqa: PLR0911 — one early return per rejected file
|
||||||
|
state: ReplState,
|
||||||
|
payload: MentionPayload,
|
||||||
|
mention: Mention,
|
||||||
|
budget: int,
|
||||||
|
) -> str | None:
|
||||||
|
"""Attach one mention; a returned string is a note for the user."""
|
||||||
|
target = _resolve(state, mention.path)
|
||||||
|
if isinstance(target, str):
|
||||||
|
return target
|
||||||
|
if not target.exists():
|
||||||
|
return f"{mention.token}: file not found"
|
||||||
|
if not target.is_file():
|
||||||
|
return f"{mention.token}: not a file"
|
||||||
|
data = target.read_bytes()
|
||||||
|
if sniff_image_media_type(data) is not None:
|
||||||
|
return _attach_image(state, payload, mention, data)
|
||||||
|
if b"\x00" in data[:512]:
|
||||||
|
return f"{mention.token}: binary file skipped (only images and text are attached)"
|
||||||
|
if budget <= 0:
|
||||||
|
return f"{mention.token}: skipped, this turn's attachments are at the size cap"
|
||||||
|
return _attach_text(payload, mention, data, budget)
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve(state: ReplState, path: str) -> Path | str:
|
||||||
|
"""The absolute path for a mention, or a note when it is denylisted."""
|
||||||
|
from plyngent.tools.context import bind_instance
|
||||||
|
from plyngent.tools.workspace import denylist_match
|
||||||
|
|
||||||
|
target = Path(path).expanduser()
|
||||||
|
if not target.is_absolute():
|
||||||
|
target = Path.cwd() / target
|
||||||
|
with bind_instance(state.instance_state):
|
||||||
|
denied = denylist_match(target)
|
||||||
|
if denied is not None:
|
||||||
|
return f"@{path}: matches the path denylist pattern {denied!r}"
|
||||||
|
return target
|
||||||
|
|
||||||
|
|
||||||
|
def _attach_image(
|
||||||
|
state: ReplState,
|
||||||
|
payload: MentionPayload,
|
||||||
|
mention: Mention,
|
||||||
|
data: bytes,
|
||||||
|
) -> str | None:
|
||||||
|
"""Attach an image part, or explain why this turn cannot carry it."""
|
||||||
|
from plyngent.config.routing import model_config_for
|
||||||
|
|
||||||
|
model_config = model_config_for(state.provider, state.model)
|
||||||
|
if model_config is None or not model_config.image_in:
|
||||||
|
return (
|
||||||
|
f"{mention.token}: skipped, model {state.model!r} does not accept image input "
|
||||||
|
f'(set image_in = true under [providers.{state.provider_name}.models."{state.model}"])'
|
||||||
|
)
|
||||||
|
part = image_part_from_bytes(data, detail="auto")
|
||||||
|
if isinstance(part, str):
|
||||||
|
return f"{mention.token}: {part.removeprefix('error: ')}"
|
||||||
|
payload.images.append(part)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _attach_text(payload: MentionPayload, mention: Mention, data: bytes, budget: int) -> str | None:
|
||||||
|
"""Append a text file to the message under its own header."""
|
||||||
|
text = data.decode("utf-8", errors="replace")
|
||||||
|
# ``truncate_to_memory`` embeds the resumable token and marker itself.
|
||||||
|
body, token = truncate_to_memory(text, min(MAX_MENTION_FILE_CHARS, budget))
|
||||||
|
header = f"--- {mention.token} ({len(text)} chars{', truncated' if token is not None else ''}) ---"
|
||||||
|
payload.text = f"{payload.text}\n\n{header}\n{body}"
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def complete_mention_paths(incomplete: str, *, cwd: Path | None = None) -> list[str]:
|
||||||
|
"""Path candidates for the path part of a mention (directories end in ``/``)."""
|
||||||
|
base = cwd if cwd is not None else Path.cwd()
|
||||||
|
prefix = incomplete.rpartition("/")[2]
|
||||||
|
raw_dir = incomplete[: len(incomplete) - len(prefix)]
|
||||||
|
directory = Path(raw_dir).expanduser() if raw_dir else Path()
|
||||||
|
if not directory.is_absolute():
|
||||||
|
directory = base / directory
|
||||||
|
try:
|
||||||
|
entries = sorted(os.scandir(directory), key=lambda entry: entry.name)
|
||||||
|
except OSError:
|
||||||
|
return []
|
||||||
|
completions: list[str] = []
|
||||||
|
for entry in entries:
|
||||||
|
if prefix and not entry.name.startswith(prefix):
|
||||||
|
continue
|
||||||
|
if not prefix and entry.name.startswith("."):
|
||||||
|
continue
|
||||||
|
suffix = "/" if entry.is_dir() else ""
|
||||||
|
completions.append(f"{raw_dir}{entry.name}{suffix}")
|
||||||
|
if len(completions) >= MAX_PATH_COMPLETIONS:
|
||||||
|
break
|
||||||
|
return completions
|
||||||
|
|
||||||
|
|
||||||
|
def mention_completions(token: str, *, cwd: Path | None = None) -> list[str]:
|
||||||
|
"""Complete an ``@path`` token (including its ``@``) for readline."""
|
||||||
|
if not token.startswith(MENTION):
|
||||||
|
return []
|
||||||
|
paths = complete_mention_paths(token[len(MENTION) :], cwd=cwd)
|
||||||
|
return [mention_token_for(path) for path in paths]
|
||||||
@@ -45,12 +45,16 @@ def build_completer(state: ReplState) -> Callable[[str, int], str | None]:
|
|||||||
def completer(text: str, state_index: int) -> str | None:
|
def completer(text: str, state_index: int) -> str | None:
|
||||||
import readline
|
import readline
|
||||||
|
|
||||||
|
from plyngent.cli.mentions import mention_completions
|
||||||
from plyngent.cli.slash import complete_slash_args
|
from plyngent.cli.slash import complete_slash_args
|
||||||
|
|
||||||
buffer = readline.get_line_buffer()
|
buffer = readline.get_line_buffer()
|
||||||
begidx = readline.get_begidx()
|
begidx = readline.get_begidx()
|
||||||
|
# An ``@path`` mention completes files wherever it sits in the line.
|
||||||
|
if text.startswith("@"):
|
||||||
|
options = mention_completions(text)
|
||||||
# Completing the first token (command).
|
# Completing the first token (command).
|
||||||
if begidx == 0 or (begidx > 0 and buffer[:begidx].strip() == ""):
|
elif begidx == 0 or (begidx > 0 and buffer[:begidx].strip() == ""):
|
||||||
options = filter_prefix(text, slash_commands())
|
options = filter_prefix(text, slash_commands())
|
||||||
else:
|
else:
|
||||||
head = buffer[:begidx].strip()
|
head = buffer[:begidx].strip()
|
||||||
|
|||||||
@@ -34,6 +34,19 @@ def _echo_interrupted() -> None:
|
|||||||
click.echo()
|
click.echo()
|
||||||
|
|
||||||
|
|
||||||
|
async def _send_turn(state: ReplState, text: str) -> None:
|
||||||
|
"""Send one user turn, attaching whatever its ``@path`` mentions point at."""
|
||||||
|
from plyngent.cli.mentions import resolve_mentions
|
||||||
|
|
||||||
|
payload = resolve_mentions(state, text)
|
||||||
|
for note in payload.notes:
|
||||||
|
click.secho(note, fg="yellow")
|
||||||
|
try:
|
||||||
|
_ = await run_user_text_with_retries(state.agent, payload.text, images=payload.images or None)
|
||||||
|
finally:
|
||||||
|
state.expire_yolo_once()
|
||||||
|
|
||||||
|
|
||||||
async def run_repl(state: ReplState) -> None:
|
async def run_repl(state: ReplState) -> None:
|
||||||
"""Interactive chat loop with readline editing, history, and Tab completion."""
|
"""Interactive chat loop with readline editing, history, and Tab completion."""
|
||||||
setup_readline(state)
|
setup_readline(state)
|
||||||
@@ -79,16 +92,10 @@ async def run_repl(state: ReplState) -> None:
|
|||||||
text = state.pending_user_text
|
text = state.pending_user_text
|
||||||
state.pending_user_text = None
|
state.pending_user_text = None
|
||||||
_echo_user(text)
|
_echo_user(text)
|
||||||
try:
|
await _send_turn(state, text)
|
||||||
_ = await run_user_text_with_retries(state.agent, text)
|
|
||||||
finally:
|
|
||||||
state.expire_yolo_once()
|
|
||||||
continue
|
continue
|
||||||
|
|
||||||
_echo_user(entry)
|
_echo_user(entry)
|
||||||
try:
|
await _send_turn(state, entry)
|
||||||
_ = await run_user_text_with_retries(state.agent, entry)
|
|
||||||
finally:
|
|
||||||
state.expire_yolo_once()
|
|
||||||
except KeyboardInterrupt:
|
except KeyboardInterrupt:
|
||||||
_echo_interrupted()
|
_echo_interrupted()
|
||||||
|
|||||||
@@ -13,10 +13,11 @@ from plyngent.cli.interrupt import install_sigint_router, sigint_cancels
|
|||||||
from plyngent.cli.limits import reset_auto_continue_turn
|
from plyngent.cli.limits import reset_auto_continue_turn
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from collections.abc import AsyncIterator, Callable, Coroutine
|
from collections.abc import AsyncIterator, Callable, Coroutine, Sequence
|
||||||
|
|
||||||
from plyngent.agent import AgentEvent
|
from plyngent.agent import AgentEvent
|
||||||
from plyngent.agent.chat import ChatAgent
|
from plyngent.agent.chat import ChatAgent
|
||||||
|
from plyngent.lmproto.openai_compatible.model import ChatImagePart
|
||||||
|
|
||||||
# Auto-retry budget after the first failure (10 attempts by default).
|
# Auto-retry budget after the first failure (10 attempts by default).
|
||||||
DEFAULT_MAX_AUTO_RETRIES = 10
|
DEFAULT_MAX_AUTO_RETRIES = 10
|
||||||
@@ -238,10 +239,11 @@ async def run_user_text_with_retries(
|
|||||||
agent: ChatAgent,
|
agent: ChatAgent,
|
||||||
text: str,
|
text: str,
|
||||||
*,
|
*,
|
||||||
|
images: Sequence[ChatImagePart] | None = None,
|
||||||
delays: tuple[float, ...] = DEFAULT_RETRY_DELAYS_SECONDS,
|
delays: tuple[float, ...] = DEFAULT_RETRY_DELAYS_SECONDS,
|
||||||
) -> bool:
|
) -> bool:
|
||||||
"""Send a new user message with auto-retry (empty ``delays`` = no auto-retry)."""
|
"""Send a new user message with auto-retry (empty ``delays`` = no auto-retry)."""
|
||||||
return await run_turn_with_retries(agent, starter=lambda: agent.run(text), delays=delays)
|
return await run_turn_with_retries(agent, starter=lambda: agent.run(text, images=images), delays=delays)
|
||||||
|
|
||||||
|
|
||||||
async def retry_pending_with_retries(agent: ChatAgent) -> bool:
|
async def retry_pending_with_retries(agent: ChatAgent) -> bool:
|
||||||
|
|||||||
@@ -344,9 +344,9 @@ class ProviderConfig(Struct, tag_field="preset", omit_defaults=True):
|
|||||||
def _default_openai_models() -> dict[str, ModelConfig]:
|
def _default_openai_models() -> dict[str, ModelConfig]:
|
||||||
"""Current OpenAI text catalog when TOML omits ``models``."""
|
"""Current OpenAI text catalog when TOML omits ``models``."""
|
||||||
return {
|
return {
|
||||||
"gpt-5.4": ModelConfig(text=True),
|
"gpt-5.4": ModelConfig(text=True, image_in=True),
|
||||||
"gpt-5.4-mini": ModelConfig(text=True),
|
"gpt-5.4-mini": ModelConfig(text=True, image_in=True),
|
||||||
"gpt-5.4-nano": ModelConfig(text=True),
|
"gpt-5.4-nano": ModelConfig(text=True, image_in=True),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -382,9 +382,14 @@ class AnthropicProvider(ProviderConfig, tag="anthropic"):
|
|||||||
|
|
||||||
|
|
||||||
def _default_deepseek_models() -> dict[str, ModelConfig]:
|
def _default_deepseek_models() -> dict[str, ModelConfig]:
|
||||||
"""Current DeepSeek text catalog when TOML omits ``models``."""
|
"""Current DeepSeek text catalog when TOML omits ``models``.
|
||||||
|
|
||||||
|
Only ``deepseek-flash`` reads pictures (JPEG, PNG, GIF, WebP in a user
|
||||||
|
message); ``deepseek-v4-pro`` stays text-only, so ``image_in`` is the flag
|
||||||
|
that decides whether a turn may carry one.
|
||||||
|
"""
|
||||||
return {
|
return {
|
||||||
"deepseek-flash": ModelConfig(text=True),
|
"deepseek-flash": ModelConfig(text=True, image_in=True),
|
||||||
"deepseek-v4-pro": ModelConfig(text=True),
|
"deepseek-v4-pro": ModelConfig(text=True),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,80 @@
|
|||||||
|
"""Image parts: format sniffing, data URLs, and the inline size policy."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import base64
|
||||||
|
|
||||||
|
from plyngent.agent.images import (
|
||||||
|
MAX_INLINE_IMAGE_BYTES,
|
||||||
|
image_part_from_bytes,
|
||||||
|
image_part_from_url,
|
||||||
|
sniff_image_media_type,
|
||||||
|
split_data_url,
|
||||||
|
to_data_url,
|
||||||
|
user_content,
|
||||||
|
)
|
||||||
|
from plyngent.lmproto.openai_compatible.model import ChatImageUrl, ChatTextPart
|
||||||
|
|
||||||
|
PNG = base64.b64decode(
|
||||||
|
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_format_is_sniffed_from_the_bytes() -> None:
|
||||||
|
assert sniff_image_media_type(PNG) == "image/png"
|
||||||
|
assert sniff_image_media_type(b"\xff\xd8\xff\xe0JFIF") == "image/jpeg"
|
||||||
|
assert sniff_image_media_type(b"GIF89a....") == "image/gif"
|
||||||
|
assert sniff_image_media_type(b"RIFF\x00\x00\x00\x00WEBPVP8 ") == "image/webp"
|
||||||
|
assert sniff_image_media_type(b"not an image") is None
|
||||||
|
assert sniff_image_media_type(b"") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_data_url_round_trips() -> None:
|
||||||
|
url = to_data_url("image/png", PNG)
|
||||||
|
|
||||||
|
assert url.startswith("data:image/png;base64,")
|
||||||
|
assert split_data_url(url) == ("image/png", base64.b64encode(PNG).decode("ascii"))
|
||||||
|
assert split_data_url("https://example.com/a.png") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_bytes_become_an_image_part_with_a_detail_level() -> None:
|
||||||
|
part = image_part_from_bytes(PNG, detail="low")
|
||||||
|
|
||||||
|
assert not isinstance(part, str)
|
||||||
|
assert part.image_url.detail == "low"
|
||||||
|
assert part.image_url.url.startswith("data:image/png;base64,")
|
||||||
|
|
||||||
|
|
||||||
|
def test_non_image_bytes_are_an_error_text() -> None:
|
||||||
|
result = image_part_from_bytes(b"plain text")
|
||||||
|
|
||||||
|
assert isinstance(result, str)
|
||||||
|
assert "not a JPEG, PNG, GIF, or WebP image" in result
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_oversized_image_is_refused() -> None:
|
||||||
|
result = image_part_from_bytes(PNG + b"\x00" * (MAX_INLINE_IMAGE_BYTES + 1))
|
||||||
|
|
||||||
|
assert isinstance(result, str)
|
||||||
|
assert "inline limit" in result
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_url_becomes_an_image_part() -> None:
|
||||||
|
part = image_part_from_url("https://example.com/a.jpg", detail="original")
|
||||||
|
|
||||||
|
assert part.image_url == ChatImageUrl(url="https://example.com/a.jpg", detail="original")
|
||||||
|
|
||||||
|
|
||||||
|
def test_user_content_stays_a_string_without_images() -> None:
|
||||||
|
assert user_content("hello") == "hello"
|
||||||
|
assert user_content("hello", []) == "hello"
|
||||||
|
|
||||||
|
|
||||||
|
def test_user_content_becomes_a_part_array_with_images() -> None:
|
||||||
|
part = image_part_from_url("https://example.com/a.jpg")
|
||||||
|
|
||||||
|
content = user_content("look", [part])
|
||||||
|
|
||||||
|
assert content == [ChatTextPart(text="look"), part]
|
||||||
|
# An image-only turn sends no empty text part.
|
||||||
|
assert user_content("", [part]) == [part]
|
||||||
@@ -0,0 +1,88 @@
|
|||||||
|
"""``@path`` mentions: parsing, token spelling, and Tab completion."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
from plyngent.cli.mentions import (
|
||||||
|
complete_mention_paths,
|
||||||
|
mention_completions,
|
||||||
|
mention_token_for,
|
||||||
|
parse_mentions,
|
||||||
|
)
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
def _paths(line: str) -> list[str]:
|
||||||
|
return [mention.path for mention in parse_mentions(line)]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_mention_is_a_whitespace_delimited_token() -> None:
|
||||||
|
assert _paths("look at @shot.png please") == ["shot.png"]
|
||||||
|
assert _paths("@a.png and @b/c.txt") == ["a.png", "b/c.txt"]
|
||||||
|
assert _paths("@a.png\n@b.png") == ["a.png", "b.png"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_email_address_is_not_a_mention() -> None:
|
||||||
|
assert _paths("mail me at someone@example.com") == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_quoted_mention_may_hold_spaces() -> None:
|
||||||
|
assert _paths('look at @"screens/my shot.png"') == ["screens/my shot.png"]
|
||||||
|
assert _paths('@"a b.png" and @c.png') == ["a b.png", "c.png"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_unterminated_quote_is_left_alone() -> None:
|
||||||
|
assert _paths('still typing @"my sh') == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_bare_at_sign_is_not_a_mention() -> None:
|
||||||
|
assert _paths("@ and @@@") == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_token_quotes_a_path_with_spaces() -> None:
|
||||||
|
assert mention_token_for("shot.png") == "@shot.png"
|
||||||
|
assert mention_token_for("my shot.png") == '@"my shot.png"'
|
||||||
|
|
||||||
|
|
||||||
|
def test_completion_lists_a_directory(tmp_path: Path) -> None:
|
||||||
|
(tmp_path / "shot.png").write_bytes(b"")
|
||||||
|
(tmp_path / "screens").mkdir()
|
||||||
|
|
||||||
|
completions = complete_mention_paths("sc", cwd=tmp_path)
|
||||||
|
|
||||||
|
assert completions == ["screens/"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_completion_continues_through_a_directory(tmp_path: Path) -> None:
|
||||||
|
(tmp_path / "screens").mkdir()
|
||||||
|
(tmp_path / "screens" / "shot.png").write_bytes(b"")
|
||||||
|
|
||||||
|
assert complete_mention_paths("screens/sh", cwd=tmp_path) == ["screens/shot.png"]
|
||||||
|
assert complete_mention_paths("screens/", cwd=tmp_path) == ["screens/shot.png"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_completion_hides_dot_files_unless_asked(tmp_path: Path) -> None:
|
||||||
|
(tmp_path / ".env").write_text("x", encoding="utf-8")
|
||||||
|
|
||||||
|
assert complete_mention_paths("", cwd=tmp_path) == []
|
||||||
|
assert complete_mention_paths(".", cwd=tmp_path) == [".env"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_completion_ignores_a_missing_directory(tmp_path: Path) -> None:
|
||||||
|
assert complete_mention_paths("nope/x", cwd=tmp_path) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_completions_keep_the_at_sign_and_quote_spaces(tmp_path: Path) -> None:
|
||||||
|
(tmp_path / "my shot.png").write_bytes(b"")
|
||||||
|
(tmp_path / "shots").mkdir()
|
||||||
|
|
||||||
|
assert mention_completions("@my", cwd=tmp_path) == ['@"my shot.png"']
|
||||||
|
assert mention_completions("@shots", cwd=tmp_path) == ["@shots/"]
|
||||||
|
assert mention_completions("@", cwd=tmp_path) == ['@"my shot.png"', "@shots/"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_mention_completions_ignores_a_plain_word() -> None:
|
||||||
|
assert mention_completions("shot") == []
|
||||||
@@ -170,6 +170,46 @@ def test_completer_stream_on_off(tmp_path: object, monkeypatch: object) -> None:
|
|||||||
assert completer("o", 1) == "off"
|
assert completer("o", 1) == "off"
|
||||||
|
|
||||||
|
|
||||||
|
def test_completer_path_mentions(tmp_path: object, monkeypatch: object) -> None:
|
||||||
|
"""An ``@path`` token completes files, wherever it sits in the line."""
|
||||||
|
import readline
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
assert isinstance(tmp_path, Path)
|
||||||
|
assert isinstance(monkeypatch, pytest.MonkeyPatch)
|
||||||
|
(tmp_path / "shot.png").write_bytes(b"")
|
||||||
|
(tmp_path / "screens").mkdir()
|
||||||
|
|
||||||
|
state = _minimal_state(tmp_path)
|
||||||
|
completer = build_completer(state)
|
||||||
|
monkeypatch.setattr(readline, "get_line_buffer", lambda: "look at @sh")
|
||||||
|
monkeypatch.setattr(readline, "get_begidx", lambda: len("look at "))
|
||||||
|
monkeypatch.chdir(tmp_path)
|
||||||
|
|
||||||
|
found = [item for index in range(10) if (item := completer("@sh", index)) is not None]
|
||||||
|
assert found == ["@shot.png"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_plain_word_is_still_a_slash_argument(tmp_path: object, monkeypatch: object) -> None:
|
||||||
|
"""The mention branch must not swallow ordinary argument completion."""
|
||||||
|
import readline
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
assert isinstance(tmp_path, Path)
|
||||||
|
assert isinstance(monkeypatch, pytest.MonkeyPatch)
|
||||||
|
|
||||||
|
state = _minimal_state(tmp_path)
|
||||||
|
completer = build_completer(state)
|
||||||
|
monkeypatch.setattr(readline, "get_line_buffer", lambda: "/provider r")
|
||||||
|
monkeypatch.setattr(readline, "get_begidx", lambda: len("/provider "))
|
||||||
|
|
||||||
|
assert completer("r", 0) == "remote"
|
||||||
|
|
||||||
|
|
||||||
def test_complete_slash_args_from_registry(tmp_path: object) -> None:
|
def test_complete_slash_args_from_registry(tmp_path: object) -> None:
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
|||||||
@@ -1,14 +1,17 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import base64
|
||||||
from typing import TYPE_CHECKING, Literal, overload
|
from typing import TYPE_CHECKING, Literal, overload
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
import tomlkit
|
import tomlkit
|
||||||
|
|
||||||
from plyngent.agent import ChatAgent, Notice
|
from plyngent.agent import ChatAgent, Notice
|
||||||
|
from plyngent.cli.mentions import resolve_mentions
|
||||||
|
from plyngent.cli.repl import _send_turn
|
||||||
from plyngent.cli.slash import handle_slash
|
from plyngent.cli.slash import handle_slash
|
||||||
from plyngent.cli.state import ReplState
|
from plyngent.cli.state import ReplState
|
||||||
from plyngent.config.models import DatabaseConfig, OpenAIProvider
|
from plyngent.config.models import DatabaseConfig, ModelConfig, OpenAIProvider
|
||||||
from plyngent.config.store import ConfigStore
|
from plyngent.config.store import ConfigStore
|
||||||
from plyngent.lmproto.openai_compatible.model import (
|
from plyngent.lmproto.openai_compatible.model import (
|
||||||
AssistantChatMessage,
|
AssistantChatMessage,
|
||||||
@@ -16,6 +19,9 @@ from plyngent.lmproto.openai_compatible.model import (
|
|||||||
ChatCompletionChunk,
|
ChatCompletionChunk,
|
||||||
ChatCompletionResponse,
|
ChatCompletionResponse,
|
||||||
ChatCompletionsParam,
|
ChatCompletionsParam,
|
||||||
|
ChatImagePart,
|
||||||
|
ChatTextPart,
|
||||||
|
UserChatMessage,
|
||||||
)
|
)
|
||||||
from plyngent.memory import MemoryStore
|
from plyngent.memory import MemoryStore
|
||||||
|
|
||||||
@@ -24,6 +30,17 @@ if TYPE_CHECKING:
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
# A real 1x1 PNG: ``/image`` sniffs the format from these bytes.
|
||||||
|
_PNG = base64.b64decode(
|
||||||
|
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _accept_images(state: ReplState) -> None:
|
||||||
|
"""Mark the fixture's model as taking pictures (the config default is text-only)."""
|
||||||
|
state.provider.models[state.model] = ModelConfig(image_in=True)
|
||||||
|
|
||||||
|
|
||||||
class DummyClient:
|
class DummyClient:
|
||||||
@overload
|
@overload
|
||||||
async def chat_completions(
|
async def chat_completions(
|
||||||
@@ -543,7 +560,7 @@ async def test_repl_ctrl_c_from_turn_returns_to_prompt(
|
|||||||
raise EOFError
|
raise EOFError
|
||||||
return "hello"
|
return "hello"
|
||||||
|
|
||||||
async def interrupted_turn(_agent: object, _text: str) -> bool:
|
async def interrupted_turn(_agent: object, _text: str, **_kwargs: object) -> bool:
|
||||||
raise KeyboardInterrupt
|
raise KeyboardInterrupt
|
||||||
|
|
||||||
monkeypatch.setattr(repl_mod, "setup_readline", lambda _state: None)
|
monkeypatch.setattr(repl_mod, "setup_readline", lambda _state: None)
|
||||||
@@ -573,7 +590,7 @@ async def test_repl_second_ctrl_c_after_cancel_returns_to_prompt(
|
|||||||
raise EOFError
|
raise EOFError
|
||||||
return reads[calls["reads"] - 1]
|
return reads[calls["reads"] - 1]
|
||||||
|
|
||||||
async def cancel_turn(_agent: object, _text: str) -> bool:
|
async def cancel_turn(_agent: object, _text: str, **_kwargs: object) -> bool:
|
||||||
raise KeyboardInterrupt # first Ctrl+C: cancels every turn
|
raise KeyboardInterrupt # first Ctrl+C: cancels every turn
|
||||||
|
|
||||||
def echo_user(text: str) -> None:
|
def echo_user(text: str) -> None:
|
||||||
@@ -780,3 +797,61 @@ async def test_todos_update_takes_a_listing_number(state: ReplState, capsys: pyt
|
|||||||
|
|
||||||
assert await handle_slash(state, "/todos pop") is True
|
assert await handle_slash(state, "/todos pop") is True
|
||||||
assert "popped TOP group (first, second)" in capsys.readouterr().out
|
assert "popped TOP group (first, second)" in capsys.readouterr().out
|
||||||
|
|
||||||
|
|
||||||
|
async def test_a_mention_attaches_the_picture_to_the_turn(state: ReplState, tmp_path: Path) -> None:
|
||||||
|
_accept_images(state)
|
||||||
|
shot = tmp_path / "shot.png"
|
||||||
|
_ = shot.write_bytes(_PNG)
|
||||||
|
state.agent.stream = False
|
||||||
|
|
||||||
|
payload = resolve_mentions(state, f"look at @{shot}")
|
||||||
|
assert payload.notes == []
|
||||||
|
assert len(payload.images) == 1
|
||||||
|
assert payload.text == f"look at @{shot}"
|
||||||
|
|
||||||
|
await _send_turn(state, payload.text)
|
||||||
|
user = next(m for m in state.agent.messages if isinstance(m, UserChatMessage))
|
||||||
|
assert isinstance(user.content, list)
|
||||||
|
text_part, image_part = user.content
|
||||||
|
assert isinstance(text_part, ChatTextPart)
|
||||||
|
assert text_part.text == f"look at @{shot}"
|
||||||
|
assert isinstance(image_part, ChatImagePart)
|
||||||
|
assert image_part.image_url.url.startswith("data:image/png;base64,")
|
||||||
|
|
||||||
|
|
||||||
|
async def test_a_mention_inlines_a_text_file(state: ReplState, tmp_path: Path) -> None:
|
||||||
|
notes = tmp_path / "notes.txt"
|
||||||
|
_ = notes.write_text("the deploy needs acme", encoding="utf-8")
|
||||||
|
|
||||||
|
payload = resolve_mentions(state, f"read @{notes}")
|
||||||
|
|
||||||
|
assert payload.images == []
|
||||||
|
assert payload.notes == []
|
||||||
|
assert payload.text.startswith(f"read @{notes}")
|
||||||
|
assert f"--- @{notes} (21 chars) ---" in payload.text
|
||||||
|
assert payload.text.endswith("the deploy needs acme")
|
||||||
|
|
||||||
|
|
||||||
|
async def test_a_mention_of_a_missing_file_is_a_note(state: ReplState, tmp_path: Path) -> None:
|
||||||
|
payload = resolve_mentions(state, "@nope.png is gone")
|
||||||
|
|
||||||
|
assert payload.images == []
|
||||||
|
assert payload.notes == ["@nope.png: file not found"]
|
||||||
|
assert payload.text == "@nope.png is gone"
|
||||||
|
|
||||||
|
|
||||||
|
async def test_a_picture_mention_is_skipped_for_a_text_only_model(
|
||||||
|
state: ReplState,
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
"""``image_in`` is the gate: a text-only model never receives a picture."""
|
||||||
|
shot = tmp_path / "shot.png"
|
||||||
|
_ = shot.write_bytes(_PNG)
|
||||||
|
|
||||||
|
payload = resolve_mentions(state, f"@{shot}")
|
||||||
|
|
||||||
|
assert payload.images == []
|
||||||
|
note = payload.notes[0]
|
||||||
|
assert "does not accept image input" in note
|
||||||
|
assert "image_in = true" in note
|
||||||
|
|||||||
Reference in New Issue
Block a user