diff --git a/src/plyngent/agent/tools.py b/src/plyngent/agent/tools.py index 40fff2b..ead611f 100644 --- a/src/plyngent/agent/tools.py +++ b/src/plyngent/agent/tools.py @@ -58,6 +58,27 @@ class ToolOutput(Struct): return bool(self.images) +def _without_refused_images(output: ToolOutput) -> ToolOutput: + """Drop a picture the selected model cannot take, and say why. + + ``view_image`` and ``fetch_local`` refuse on their own, before any bytes are + read, so the model gets a clean error. This is the safety net for the rest — + an MCP server's image content, a plugin's ``ToolOutput`` — where the picture + would otherwise reach a model that answers ``[Unsupported Image]``, or a + gateway that hands it raw base64 (see ``doc/images.md``). + """ + if not output.images: + return output + from plyngent.tools.workspace import image_input_refusal + + refusal = image_input_refusal() + if refusal is None: + return output + note = refusal.removeprefix("error: ") + text = f"{output.text}\n(error: image dropped: {note})" if output.text else f"error: image dropped: {note}" + return ToolOutput(text=text) + + @dataclass(frozen=True, slots=True) class ToolConfirmRequest: """One call of a batch that needs a soft confirm before it may run.""" @@ -737,7 +758,7 @@ class ToolRegistry: if denied is not None: return ToolOutput(text=denied) result = await self._invoke(definition, args) - return result if isinstance(result, ToolOutput) else ToolOutput(text=result) + return _without_refused_images(result if isinstance(result, ToolOutput) else ToolOutput(text=result)) # Bind host-provided state for tools that read contextvars (todo, workspace). # Tag enforcement remains gated by ``auto_bind_state``; binding is always useful diff --git a/src/plyngent/cli/app.py b/src/plyngent/cli/app.py index 12b60c3..4fbf5d3 100644 --- a/src/plyngent/cli/app.py +++ b/src/plyngent/cli/app.py @@ -351,6 +351,7 @@ async def _run_chat( # noqa: C901, PLR0915 — chat orchestration # instance.workspace (no process bag). state.instance_state.workspace.path_denylist = tuple(store.agent_config.path_denylist or ()) state.instance_state.workspace.trash_enabled = store.trash_config.enabled + state.sync_image_capability() from plyngent.tools.net import set_ssrf_assume_public_cidrs # Clash Fake-IP etc.: treat configured CIDRs as public for fetch_local SSRF. diff --git a/src/plyngent/cli/state.py b/src/plyngent/cli/state.py index cb04005..ba6a7db 100644 --- a/src/plyngent/cli/state.py +++ b/src/plyngent/cli/state.py @@ -507,6 +507,7 @@ class ReplState: # Preserve live stream toggle if agent already exists. if hasattr(self, "agent"): self.stream_enabled = self.agent.stream + self.sync_image_capability() self.client = create_client(self.provider, model=self.model, reasoning=self.reasoning) self.agent = self._make_agent() # Restore history without re-marking already-stored messages as dirty. @@ -517,6 +518,21 @@ class ReplState: if self._remote_models_key is not None and self._remote_models_key != self._models_cache_key(): self.invalidate_remote_models() + def sync_image_capability(self) -> None: + """Publish the selected model's ``image_in`` to the image-carrying tools. + + Same verdict the ``@path`` mention gate uses: only a model declared with + ``image_in = true`` takes pictures, so a text-only model (or one behind a + gateway that cannot read image blocks) is answered with an error instead + of a picture it could never describe. + """ + from plyngent.config.routing import model_config_for + + model_config = model_config_for(self.provider, self.model) + policy = self.instance_state.workspace + policy.image_in = model_config is not None and bool(model_config.image_in) + policy.image_model = (self.provider_name, self.model) + def _models_cache_key(self) -> tuple[str, str]: # Remote model lists belong to the provider endpoint used for discovery, # not model-level overrides (the model is not selected yet there). diff --git a/src/plyngent/tools/__init__.py b/src/plyngent/tools/__init__.py index 8212649..67a9a8c 100644 --- a/src/plyngent/tools/__init__.py +++ b/src/plyngent/tools/__init__.py @@ -105,6 +105,7 @@ from .workspace import get_trash_enabled as get_trash_enabled from .workspace import get_workspace_root as get_workspace_root from .workspace import grant_policy_command as grant_policy_command from .workspace import granted_access_mode as granted_access_mode +from .workspace import image_input_refusal as image_input_refusal from .workspace import list_workspace_allowlist as list_workspace_allowlist from .workspace import match_command_denylist as match_command_denylist from .workspace import mode_covers as mode_covers @@ -113,6 +114,7 @@ from .workspace import pop_owned_temporary_workspaces as pop_owned_temporary_wor from .workspace import remove_workspace_allowlist as remove_workspace_allowlist from .workspace import resolve_path as resolve_path from .workspace import set_command_denylist as set_command_denylist +from .workspace import set_image_in as set_image_in from .workspace import set_path_denylist as set_path_denylist from .workspace import set_policy_confirm_hook as set_policy_confirm_hook from .workspace import set_policy_confirm_timeout as set_policy_confirm_timeout diff --git a/src/plyngent/tools/file/view_image.py b/src/plyngent/tools/file/view_image.py index 97e6f2f..98ab7aa 100644 --- a/src/plyngent/tools/file/view_image.py +++ b/src/plyngent/tools/file/view_image.py @@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, cast from plyngent.agent import ToolOutput, ToolTag, tool from plyngent.agent.images import image_part_from_bytes -from plyngent.tools.workspace import AccessMode, resolve_path +from plyngent.tools.workspace import AccessMode, image_input_refusal, resolve_path if TYPE_CHECKING: from plyngent.lmproto.openai_compatible.model import ImageDetail @@ -27,7 +27,9 @@ async def view_image(path: str, detail: str = "auto") -> str | ToolOutput: picture as it is, and ``auto`` (the default) lets the provider decide. The format is read from the file's own bytes, so a misnamed file still - works; anything else comes back as an error. + works; anything else comes back as an error. A model whose + ``[providers.*.models.*].image_in`` is off gets an error naming that key + instead of an attachment it could never read. """ if detail not in _DETAILS: return f"error: detail must be one of {', '.join(sorted(_DETAILS))}" @@ -36,6 +38,9 @@ async def view_image(path: str, detail: str = "auto") -> str | ToolOutput: return f"error: file not found: {path}" if not target.is_file(): return f"error: not a file: {path}" + refusal = image_input_refusal() + if refusal is not None: + return refusal part = image_part_from_bytes(target.read_bytes(), detail=cast("ImageDetail", detail)) if isinstance(part, str): return f"error: {path}: {part.removeprefix('error: ')}" diff --git a/src/plyngent/tools/net/fetch_local.py b/src/plyngent/tools/net/fetch_local.py index 3f36d7f..9b448ac 100644 --- a/src/plyngent/tools/net/fetch_local.py +++ b/src/plyngent/tools/net/fetch_local.py @@ -19,6 +19,7 @@ from plyngent.tools.net.policy import ( soft_confirm_reason, ) from plyngent.tools.truncate_token import truncate_to_memory +from plyngent.tools.workspace import image_input_refusal def format_fetch_result( @@ -147,10 +148,14 @@ def _attach_image(text: str, result: FetchResult) -> str | ToolOutput: """An image response is handed over as a picture, beside the usual metadata. The preview line stays in the text so ``body_kind=image`` and the byte count - are still visible to the model. + are still visible to the model. A model without ``image_in`` keeps the + metadata and is told why the picture stayed behind. """ if result.body_kind != "image" or not result.body_image: return text + refusal = image_input_refusal() + if refusal is not None: + return f"{text}\n{refusal}" part = image_part_from_bytes(result.body_image) if isinstance(part, str): return f"{text}\n{part}" diff --git a/src/plyngent/tools/workspace.py b/src/plyngent/tools/workspace.py index d94301c..67eb4de 100644 --- a/src/plyngent/tools/workspace.py +++ b/src/plyngent/tools/workspace.py @@ -122,6 +122,14 @@ class WorkspacePolicy: static_read: list[Path] = field(default_factory=list) # ``[trash] enabled``: deletes go to the system trash unless cleared. trash_enabled: bool = True + # ``[providers.*.models.*].image_in`` of the model this host is talking to: + # ``None`` = the host did not say, so a picture is passed through unchanged. + # ``False`` makes every image-carrying tool refuse, instead of sending bytes + # the model cannot read (the provider answers ``[Unsupported Image]``, or a + # gateway that ignores the block hands the model raw base64). + image_in: bool | None = None + # ``(provider name, model id)`` that verdict came from, for the message. + image_model: tuple[str, str] | None = None def _bound_instance() -> InstanceState | None: @@ -200,6 +208,40 @@ def get_trash_enabled() -> bool: return active_workspace_policy().trash_enabled +def set_image_in(*, image_in: bool | None, provider: str | None = None, model: str | None = None) -> None: + """Record whether the current model takes pictures (``image_in``). + + ``None`` means the host did not declare it, and an image is sent as usual; + ``False`` is the explicit opt-out the image-carrying tools and the agent's + tool registry check through :func:`image_input_refusal`. + """ + policy = active_workspace_policy() + policy.image_in = image_in + policy.image_model = (provider, model) if provider and model else None + + +def image_input_refusal() -> str | None: + """``error: …`` text for an image the current model cannot take, else ``None``. + + Mirrors the ``@path`` mention gate (``cli/mentions.py``): a picture is not + sent to a model whose ``image_in`` is off. The message names the config key + that turns it on, so the fix is one line away. With no instance bound the + host has declared nothing, and the picture passes through. + """ + instance = _bound_instance() + if instance is None: + return None + policy = instance.workspace + if policy.image_in is not False: + return None + named = policy.image_model + if named is None: + return "error: this model does not accept image input (set image_in = true in the provider config)" + provider, model = named + hint = f'set image_in = true under [providers.{provider}.models."{model}"]' + return f"error: model {model!r} does not accept image input ({hint})" + + def set_command_denylist(names: list[str] | tuple[str, ...] | frozenset[str] | None) -> None: """Set denied command basenames (None restores defaults).""" policy = active_workspace_policy()