core/tools+cli: split fetch into remote fetch and fetch_local

This commit is contained in:
2026-10-17 17:06:43 +08:00
parent 37e130a7dc
commit cfe36b024b
28 changed files with 465 additions and 125 deletions
+2 -2
View File
@@ -26,8 +26,8 @@ use `run_argv_batch` for ordered pipelines.
- `regex_files`: `pattern` is a Python `re` regex (not a glob, so `*.py` fails) and \ - `regex_files`: `pattern` is a Python `re` regex (not a glob, so `*.py` fails) and \
`path` is a literal file/dir (no glob expansion). `path` is a literal file/dir (no glob expansion).
- Truncated results carry a `truncate_token=...`; continue with `get_truncated`. - Truncated results carry a `truncate_token=...`; continue with `get_truncated`.
- Prefer `fetch` for HTTP(S); `offset` (0-based) skips leading body characters; \ - Prefer `fetch` for public pages (clean Markdown); `fetch_local` for localhost/LAN, \
private/LAN hosts need human policy allow (not YOLO). non-GET methods, headers, or raw status; `offset` (0-based) skips leading body characters.
- PTY secrets only via `ask_into_pty`; denylists and confirms still apply. - PTY secrets only via `ask_into_pty`; denylists and confirms still apply.
- `wait` pauses; pressing Enter disturbs early (optional reason). - `wait` pauses; pressing Enter disturbs early (optional reason).
- Todo stack: open items mean unfinished work; the stack does not auto-clear — \ - Todo stack: open items mean unfinished work; the stack does not auto-clear — \
+2 -2
View File
@@ -115,8 +115,8 @@ def truncated_turn_notice() -> Notice:
_ASIDE_BODIES: dict[str, str] = { _ASIDE_BODIES: dict[str, str] = {
"read": ( "read": (
"Side question: this exchange is not saved to the session transcript and does not " "Side question: this exchange is not saved to the session transcript and does not "
"change the main conversation. Tools here are read-only (file/VCS reads, ask, fetch " "change the main conversation. Tools here are read-only (file/VCS reads, ask, fetch/"
"GET): you cannot edit files or run state-changing commands, so describe a change " "fetch_local GET): you cannot edit files or run state-changing commands, so describe a change "
"instead of applying it." "instead of applying it."
), ),
"no": ( "no": (
+1 -1
View File
@@ -341,7 +341,7 @@ async def _run_chat( # noqa: C901, PLR0915 — chat orchestration
state.instance_state.workspace.path_denylist = tuple(store.agent_config.path_denylist or ()) state.instance_state.workspace.path_denylist = tuple(store.agent_config.path_denylist or ())
from plyngent.tools.net import set_ssrf_assume_public_cidrs from plyngent.tools.net import set_ssrf_assume_public_cidrs
# Clash Fake-IP etc.: treat configured CIDRs as public for fetch SSRF. # Clash Fake-IP etc.: treat configured CIDRs as public for fetch_local SSRF.
_ = set_ssrf_assume_public_cidrs(store.networking_config.ssrf_assume_public_cidrs) _ = set_ssrf_assume_public_cidrs(store.networking_config.ssrf_assume_public_cidrs)
if interactive: if interactive:
from plyngent.cli.limits import prompt_policy_command_confirm, prompt_policy_fetch_confirm from plyngent.cli.limits import prompt_policy_command_confirm, prompt_policy_fetch_confirm
+24 -2
View File
@@ -452,13 +452,34 @@ def _http_status_fg(status: str) -> str | None:
def _fetch_prefix(args_json: str) -> str: def _fetch_prefix(args_json: str) -> str:
"""Remote ``fetch``: the URL is the whole call."""
url = _json_str_arg(args_json, "url") or "?"
return _pretty_prefix(f"* Fetch {url} ")
def _fetch_detail(args_json: str, result: str) -> str:
"""Remote ``fetch`` outcome: fetched size, or an error.
A row written before the local tool was renamed still carries the local
``status=...`` block; render it with the local formatter.
"""
if result.startswith("error:"):
return _pretty_segments((f"({result})", "red"))
fields = _status_fields(result)
if "status" in fields:
return _fetch_local_detail(args_json, result)
chars = fields.get("chars", "")
return _pretty_segments((f"({chars} chars)" if chars else "(done)", "green"))
def _fetch_local_prefix(args_json: str) -> str:
method = _json_str_arg(args_json, "method") or "GET" method = _json_str_arg(args_json, "method") or "GET"
url = _json_str_arg(args_json, "url") or "?" url = _json_str_arg(args_json, "url") or "?"
return _pretty_prefix(f"* Fetch {method} {url} ") return _pretty_prefix(f"* Fetch {method} {url} ")
def _fetch_detail(_args_json: str, result: str) -> str: def _fetch_local_detail(_args_json: str, result: str) -> str:
"""``fetch`` outcome: HTTP status, or error.""" """``fetch_local`` outcome: HTTP status, or error."""
if result.startswith("error:"): if result.startswith("error:"):
return _pretty_segments((f"({result})", "red")) return _pretty_segments((f"({result})", "red"))
status = _status_fields(result).get("status", "") status = _status_fields(result).get("status", "")
@@ -888,6 +909,7 @@ _PRETTY_LINES: dict[str, PrettyLine] = {
"run_argv": PrettyLine(prefix=_run_argv_prefix, detail=_run_argv_detail), "run_argv": PrettyLine(prefix=_run_argv_prefix, detail=_run_argv_detail),
"run_argv_batch": PrettyLine(prefix=_run_argv_batch_prefix, detail=_run_argv_batch_detail), "run_argv_batch": PrettyLine(prefix=_run_argv_batch_prefix, detail=_run_argv_batch_detail),
"fetch": PrettyLine(prefix=_fetch_prefix, detail=_fetch_detail), "fetch": PrettyLine(prefix=_fetch_prefix, detail=_fetch_detail),
"fetch_local": PrettyLine(prefix=_fetch_local_prefix, detail=_fetch_local_detail),
"web_search": PrettyLine(prefix=_web_search_prefix, detail=_web_search_detail), "web_search": PrettyLine(prefix=_web_search_prefix, detail=_web_search_detail),
"edit_replace": _mutator_line("edit_replace"), "edit_replace": _mutator_line("edit_replace"),
"edit_lineno": _mutator_line("edit_lineno"), "edit_lineno": _mutator_line("edit_lineno"),
+3 -3
View File
@@ -311,14 +311,14 @@ def prompt_policy_fetch_confirm(
url: str, url: str,
timeout_seconds: float, timeout_seconds: float,
) -> bool: ) -> bool:
"""Ask whether to allow fetch to a private/loopback host (timed). """Ask whether to allow fetch_local to a private/loopback host (timed).
Independent of YOLO: always prompts when interactive. Default is **deny**. Independent of YOLO: always prompts when interactive. Default is **deny**.
Timeout, cancel, or non-interactive → False. On approve, the host:port is Timeout, cancel, or non-interactive → False. On approve, the host:port is
granted for the rest of this process (see net grants). granted for the rest of this process (see net grants).
""" """
reason = ( reason = (
f"fetch targets private/loopback host {host!r} port {port}\n" f"fetch_local targets private/loopback host {host!r} port {port}\n"
f"url: {url}\n" f"url: {url}\n"
f"Allow this host:port for the rest of this process?\n" f"Allow this host:port for the rest of this process?\n"
f"(timeout {timeout_seconds:g}s defaults to DENY; not skipped by YOLO)" f"(timeout {timeout_seconds:g}s defaults to DENY; not skipped by YOLO)"
@@ -332,7 +332,7 @@ def prompt_policy_fetch_confirm(
backend.secho(format_tool_confirm_box("policy", reason), fg="yellow") backend.secho(format_tool_confirm_box("policy", reason), fg="yellow")
backend.echo() backend.echo()
grant = f"{host}:{port}" grant = f"{host}:{port}"
prompt = f"[policy] allow fetch {grant!r}? [y/N] (timeout {timeout_seconds:g}s): " prompt = f"[policy] allow fetch_local {grant!r}? [y/N] (timeout {timeout_seconds:g}s): "
with contextlib.suppress(OSError): with contextlib.suppress(OSError):
_ = sys.stderr.write(prompt) _ = sys.stderr.write(prompt)
_ = sys.stderr.flush() _ = sys.stderr.flush()
+34 -17
View File
@@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Literal
from plyngent.agent import ChatAgent, ToolRegistry from plyngent.agent import ChatAgent, ToolRegistry
from plyngent.agent.loop import DEFAULT_MAX_ROUNDS from plyngent.agent.loop import DEFAULT_MAX_ROUNDS
from plyngent.agent.todo_stack import TodoStack from plyngent.agent.todo_stack import TodoStack
from plyngent.lmproto.deepseek.webtools import DeepseekWebToolsClient
if TYPE_CHECKING: if TYPE_CHECKING:
from datetime import datetime from datetime import datetime
@@ -382,20 +383,43 @@ class ReplState:
search_tool = self._web_search_tool() search_tool = self._web_search_tool()
if search_tool is not None: if search_tool is not None:
registry.register(search_tool) registry.register(search_tool)
fetch_tool = self._remote_fetch_tool()
if fetch_tool is not None:
registry.register(fetch_tool)
return registry return registry
def _deepseek_web_client(self) -> DeepseekWebToolsClient | None:
"""The Search/Fetch API client for the active provider (None otherwise)."""
from plyngent.config.routing import deepseek_api_base_url
from plyngent.lmproto.openai_compatible.config import OpenAIConfig
provider = self.effective_provider
if provider.preset != "deepseek":
return None
return DeepseekWebToolsClient(
OpenAIConfig(
access_key_or_token=provider.access_key_or_token,
base_url=deepseek_api_base_url(provider),
timeout=normalize_http_timeout(provider.timeout),
)
)
def _remote_fetch_tool(self) -> ToolDefinition | None:
"""``fetch`` through DeepSeek's Fetch API (DeepSeek providers only)."""
from plyngent.tools.net.fetch_remote import build_fetch_tool
client = self._deepseek_web_client()
if client is None:
return None
return build_fetch_tool(client)
def _web_search_tool(self) -> ToolDefinition | None: def _web_search_tool(self) -> ToolDefinition | None:
"""Local ``web_search`` for a provider whose surface cannot host one. """Local ``web_search`` for a provider whose surface cannot host one.
The sources come from ``provider.search_sources``, in fallback order The sources come from ``provider.search_sources``, in fallback order
(see :func:`~plyngent.config.routing.local_web_search`). (see :func:`~plyngent.config.routing.local_web_search`).
""" """
from plyngent.config.routing import ( from plyngent.config.routing import local_web_search
deepseek_api_base_url,
local_web_search,
)
from plyngent.lmproto.deepseek.search import DeepseekSearchClient
from plyngent.lmproto.openai_compatible.config import OpenAIConfig
from plyngent.tools.net.bing import BingSearchBackend from plyngent.tools.net.bing import BingSearchBackend
from plyngent.tools.net.search import ( from plyngent.tools.net.search import (
DeepseekSearchBackend, DeepseekSearchBackend,
@@ -408,17 +432,10 @@ class ReplState:
backends: list[SearchBackend] = [] backends: list[SearchBackend] = []
for source in provider.search_sources: for source in provider.search_sources:
if source == "deepseek": if source == "deepseek":
backends.append( client = self._deepseek_web_client()
DeepseekSearchBackend( if client is None:
DeepseekSearchClient( continue
OpenAIConfig( backends.append(DeepseekSearchBackend(client))
access_key_or_token=provider.access_key_or_token,
base_url=deepseek_api_base_url(provider),
timeout=normalize_http_timeout(provider.timeout),
)
)
)
)
elif source == "bing": elif source == "bing":
backends.append(BingSearchBackend()) backends.append(BingSearchBackend())
if not backends: if not backends:
+14 -8
View File
@@ -71,18 +71,24 @@ with it to keep reading (chains through truncations).
a `-c` one-liner, a script (`python x.py`), or a bare interactive shell alike — and each \ a `-c` one-liner, a script (`python x.py`), or a bare interactive shell alike — and each \
confirm blocks the turn, so it costs time. Invoke the target command directly (argv) instead \ confirm blocks the turn, so it costs time. Invoke the target command directly (argv) instead \
of wrapping it in `bash -c` / `python -c` / `node`, and reach for a dedicated tool \ of wrapping it in `bash -c` / `python -c` / `node`, and reach for a dedicated tool \
(`read_file` / `edit_*` / `vcs_*` / `fetch` / …) when one already does the job. (`read_file` / `edit_*` / `vcs_*` / `fetch` / `fetch_local` / …) when one already does the job.
- Several `run_argv` calls in one step may run in parallel; use `run_argv_batch` \ - Several `run_argv` calls in one step may run in parallel; use `run_argv_batch` \
for ordered pipelines (`pipe_out` / `mix_stderr` as needed). for ordered pipelines (`pipe_out` / `mix_stderr` as needed).
- Prefer `vcs_*` for status/diff/log/branch when enough. - Prefer `vcs_*` for status/diff/log/branch when enough.
### Network ### Network
- Prefer `fetch` (GET/POST/PUT/DELETE) for HTTP(S) docs/APIs over curl/wget via shell; \ - `fetch` reads a public page as clean Markdown through the provider's Fetch API \
`offset` (0-based) skips leading body characters to page through a large body. (when available, e.g. DeepSeek); `fetch_local` is the raw HTTP client \
- Set `user_agent` (or a User-Agent header) when the remote expects a specific client; \ (GET/POST/PUT/DELETE) from this host for localhost/LAN, APIs, custom headers, and \
otherwise a small default is used. Never rely on shell to spoof identity. status/redirect inspection.
- Private/loopback/LAN hosts need an explicit human policy allow (not skipped by YOLO). - `offset` (0-based) skips leading body characters to page through a large body; \
- Prefer provider hosted search (e.g. web_search) for open-ended research; use `fetch` for known URLs. truncated bodies resume with `get_truncated` and are not re-requested.
- Set `user_agent` (or a User-Agent header) on `fetch_local` when the remote expects a \
specific client; otherwise a small default is used. Never rely on shell to spoof identity.
- Private/loopback/LAN hosts need an explicit human policy allow (not skipped by YOLO); \
this applies to `fetch_local`, since `fetch` leaves from the provider's network.
- Prefer provider hosted search (e.g. web_search) for open-ended research; use `fetch` \
or `fetch_local` for known URLs.
- Respect denials, size limits, and HTTPS→HTTP redirect blocks; do not re-fetch the same URL repeatedly. - Respect denials, size limits, and HTTPS→HTTP redirect blocks; do not re-fetch the same URL repeatedly.
### Humans & safety ### Humans & safety
@@ -232,7 +238,7 @@ class McpConfig(Struct, omit_defaults=True):
class NetworkingConfig(Struct, omit_defaults=True): class NetworkingConfig(Struct, omit_defaults=True):
"""Host-side network policy for tools such as ``fetch``. """Host-side network policy for tools such as ``fetch_local``.
``ssrf_assume_public_cidrs``: CIDR strings whose resolved addresses are treated ``ssrf_assume_public_cidrs``: CIDR strings whose resolved addresses are treated
as **public for SSRF policy** (skip private-host grants). Use for Clash/meta as **public for SSRF policy** (skip private-host grants). Use for Clash/meta
@@ -1,8 +1,12 @@
from .client import DeepseekSearchClient as DeepseekSearchClient from .client import DeepseekWebToolsClient as DeepseekWebToolsClient
from .model import DEFAULT_SEARCH_COUNT as DEFAULT_SEARCH_COUNT from .model import DEFAULT_SEARCH_COUNT as DEFAULT_SEARCH_COUNT
from .model import FETCH_PATH as FETCH_PATH
from .model import MAX_FETCH_URL_CHARS as MAX_FETCH_URL_CHARS
from .model import MAX_QUERY_CHARS as MAX_QUERY_CHARS from .model import MAX_QUERY_CHARS as MAX_QUERY_CHARS
from .model import MAX_SEARCH_COUNT as MAX_SEARCH_COUNT from .model import MAX_SEARCH_COUNT as MAX_SEARCH_COUNT
from .model import SEARCH_PATH as SEARCH_PATH from .model import SEARCH_PATH as SEARCH_PATH
from .model import FetchRequest as FetchRequest
from .model import FetchResponse as FetchResponse
from .model import SearchRequest as SearchRequest from .model import SearchRequest as SearchRequest
from .model import SearchResponse as SearchResponse from .model import SearchResponse as SearchResponse
from .model import SearchResult as SearchResult from .model import SearchResult as SearchResult
@@ -1,23 +1,27 @@
"""DeepSeek's Search API: one request per query, no model round. """DeepSeek's Search and Fetch APIs: plain JSON, one bearer key.
The endpoint is a plain JSON API on the platform base URL, authenticated with ``POST /v1/tools/search`` answers a query in one request, without spending a
the same bearer key as the model APIs. It replaces the older server-side search model round — the endpoint replaces the older server-side search turn, which
turn (``POST /messages`` with a hosted ``web_search`` tool), which spent a whole cost a whole completion round (and its input tokens) just to pick the query and
completion round — and its input tokens — just to pick the query, and could only could only return titles and URLs. A Search response already carries a snippet
return titles and URLs. A Search API response already carries a snippet and a and a model-facing summary, so the local ``web_search`` tool can hand the model
model-facing summary, so the local ``web_search`` tool can hand the model
something readable on the first call. something readable on the first call.
``POST /v1/tools/fetch`` reads one public page and returns its title and content
as Markdown, for the remote ``fetch`` tool. Both endpoints share the session,
authentication, error shape, retry policy and rate limits.
``count`` does not affect billing (one successful request costs the same for 1 ``count`` does not affect billing (one successful request costs the same for 1
or 50 results); only successful requests are billable, so the retries below are or 50 results); only successful requests are billable, so the retries below are
safe. The account-level limit is 5 QPS and is shared across the Search and Fetch safe. The account-level limit is 5 QPS and is shared across the Search and Fetch
APIs independently of model concurrency. APIs independently of model concurrency; Fetch also has a per-domain limit.
""" """
from __future__ import annotations from __future__ import annotations
import asyncio import asyncio
from typing import TYPE_CHECKING from typing import TYPE_CHECKING
from urllib.parse import urlsplit
import msgspec import msgspec
import niquests import niquests
@@ -31,9 +35,13 @@ from plyngent.lmproto.search import SearchHit
from .model import ( from .model import (
DEFAULT_SEARCH_COUNT, DEFAULT_SEARCH_COUNT,
FETCH_PATH,
MAX_FETCH_URL_CHARS,
MAX_QUERY_CHARS, MAX_QUERY_CHARS,
MAX_SEARCH_COUNT, MAX_SEARCH_COUNT,
SEARCH_PATH, SEARCH_PATH,
FetchRequest,
FetchResponse,
SearchRequest, SearchRequest,
SearchResponse, SearchResponse,
SearchResult, SearchResult,
@@ -51,12 +59,13 @@ RETRY_DELAYS: tuple[float, ...] = (0.5, 1.0)
_SNIPPET_FALLBACK_CHARS = 200 _SNIPPET_FALLBACK_CHARS = 200
class DeepseekSearchClient: class DeepseekWebToolsClient:
"""``POST /v1/tools/search``: one query in, ranked hits out.""" """``POST /v1/tools/search`` and ``/v1/tools/fetch`` on one session."""
session: niquests.AsyncSession session: niquests.AsyncSession
encoder: msgspec.json.Encoder encoder: msgspec.json.Encoder
decoder: msgspec.json.Decoder[SearchResponse] decoder: msgspec.json.Decoder[SearchResponse]
fetch_decoder: msgspec.json.Decoder[FetchResponse]
def __init__(self, config: OpenAIConfig) -> None: def __init__(self, config: OpenAIConfig) -> None:
self.session = niquests.AsyncSession( self.session = niquests.AsyncSession(
@@ -66,6 +75,7 @@ class DeepseekSearchClient:
) )
self.encoder = msgspec.json.Encoder() self.encoder = msgspec.json.Encoder()
self.decoder = msgspec.json.Decoder(SearchResponse) self.decoder = msgspec.json.Decoder(SearchResponse)
self.fetch_decoder = msgspec.json.Decoder(FetchResponse)
async def search( async def search(
self, self,
@@ -97,6 +107,19 @@ class DeepseekSearchClient:
parsed = self.decoder.decode(body) parsed = self.decoder.decode(body)
return [hit for hit in map(_to_hit, parsed.results) if hit.url] return [hit for hit in map(_to_hit, parsed.results) if hit.url]
async def fetch(self, url: str) -> FetchResponse:
"""Read *url* and return its title and Markdown content.
The page is fetched from DeepSeek's network, not from this host: local
SSRF checks do not apply, and neither do private-host grants — a LAN or
loopback target is simply unreachable there (the endpoint answers 403 or
404). The URL is still validated here so a malformed request fails
before it costs a round trip.
"""
_validate_fetch_url(url)
body = await self._post(FETCH_PATH, self.encoder.encode(FetchRequest(url=url)))
return self.fetch_decoder.decode(body)
async def _post(self, path: str, data: bytes) -> bytes: async def _post(self, path: str, data: bytes) -> bytes:
"""POST *data*, retrying the statuses that mean "wait and try again".""" """POST *data*, retrying the statuses that mean "wait and try again"."""
for delay in RETRY_DELAYS: for delay in RETRY_DELAYS:
@@ -150,4 +173,22 @@ def _shorten(text: str) -> str:
return text[: _SNIPPET_FALLBACK_CHARS - 1].rstrip() + "…" return text[: _SNIPPET_FALLBACK_CHARS - 1].rstrip() + "…"
__all__ = ["RETRY_DELAYS", "DeepseekSearchClient"] def _validate_fetch_url(url: str) -> None:
"""Reject a URL the endpoint would refuse, before paying for the request."""
if not url or len(url) > MAX_FETCH_URL_CHARS:
msg = f"fetch url must be 1-{MAX_FETCH_URL_CHARS} characters"
raise ValueError(msg)
try:
parts = urlsplit(url)
except ValueError:
msg = "fetch url is malformed"
raise ValueError(msg) from None
if parts.scheme not in {"http", "https"} or not parts.hostname:
msg = "fetch url must be an absolute http(s) URL with a host"
raise ValueError(msg)
if parts.username or parts.password:
msg = "fetch url must not carry a username or password"
raise ValueError(msg)
__all__ = ["RETRY_DELAYS", "DeepseekWebToolsClient"]
@@ -1,13 +1,15 @@
"""Wire models for DeepSeek's Search API (``POST /v1/tools/search``).""" """Wire models for DeepSeek's Search and Fetch APIs."""
from __future__ import annotations from __future__ import annotations
import msgspec import msgspec
SEARCH_PATH = "/v1/tools/search" SEARCH_PATH = "/v1/tools/search"
FETCH_PATH = "/v1/tools/fetch"
DEFAULT_SEARCH_COUNT = 10 DEFAULT_SEARCH_COUNT = 10
MAX_SEARCH_COUNT = 50 MAX_SEARCH_COUNT = 50
MAX_QUERY_CHARS = 128 MAX_QUERY_CHARS = 128
MAX_FETCH_URL_CHARS = 8192
class SearchRequest(msgspec.Struct, omit_defaults=True): class SearchRequest(msgspec.Struct, omit_defaults=True):
@@ -38,11 +40,28 @@ class SearchResponse(msgspec.Struct):
results: list[SearchResult] results: list[SearchResult]
class FetchRequest(msgspec.Struct):
"""Body of one fetch request: a single absolute http(s) URL."""
url: str
class FetchResponse(msgspec.Struct):
"""A fetched page: its title and its content rendered as Markdown."""
title: str = ""
content: str = ""
__all__ = [ __all__ = [
"DEFAULT_SEARCH_COUNT", "DEFAULT_SEARCH_COUNT",
"FETCH_PATH",
"MAX_FETCH_URL_CHARS",
"MAX_QUERY_CHARS", "MAX_QUERY_CHARS",
"MAX_SEARCH_COUNT", "MAX_SEARCH_COUNT",
"SEARCH_PATH", "SEARCH_PATH",
"FetchRequest",
"FetchResponse",
"SearchRequest", "SearchRequest",
"SearchResponse", "SearchResponse",
"SearchResult", "SearchResult",
+1 -1
View File
@@ -46,7 +46,7 @@ from .file import write_file as write_file
from .net import NET_TOOLS as NET_TOOLS from .net import NET_TOOLS as NET_TOOLS
from .net import clear_private_grants as clear_private_grants from .net import clear_private_grants as clear_private_grants
from .net import clear_ssrf_assume_public_cidrs as clear_ssrf_assume_public_cidrs from .net import clear_ssrf_assume_public_cidrs as clear_ssrf_assume_public_cidrs
from .net import fetch as fetch from .net import fetch_local as fetch_local
from .net import grant_private_host as grant_private_host from .net import grant_private_host as grant_private_host
from .net import set_fetch_policy_confirm_hook as set_fetch_policy_confirm_hook from .net import set_fetch_policy_confirm_hook as set_fetch_policy_confirm_hook
from .net import set_ssrf_assume_public_cidrs as set_ssrf_assume_public_cidrs from .net import set_ssrf_assume_public_cidrs as set_ssrf_assume_public_cidrs
+1 -1
View File
@@ -155,7 +155,7 @@ _read_only: ContextVar[bool] = ContextVar("plyngent_read_only_context", default=
def read_only_context() -> Generator[None]: def read_only_context() -> Generator[None]:
"""Mark the current context as read-only for tool execution. """Mark the current context as read-only for tool execution.
Tools that may mutate depending on arguments (e.g. ``fetch`` with a Tools that may mutate depending on arguments (e.g. ``fetch_local`` with a
non-GET method) check :func:`is_read_only_context` and refuse to mutate non-GET method) check :func:`is_read_only_context` and refuse to mutate
while it is set. while it is set.
""" """
+4 -4
View File
@@ -192,8 +192,8 @@ def _open_pty_reason(args: Mapping[str, object]) -> str | None:
def _fetch_reason(args: Mapping[str, object]) -> str | None: def _fetch_reason(args: Mapping[str, object]) -> str | None:
"""Soft-confirm cleartext HTTP and mutating fetch methods (not private-host policy).""" """Soft-confirm cleartext HTTP and mutating fetch_local methods (not private-host policy)."""
from plyngent.tools.net.fetch import fetch_soft_reason from plyngent.tools.net.fetch_local import fetch_soft_reason
method_obj = args.get("method", "GET") method_obj = args.get("method", "GET")
url_obj = args.get("url", "") url_obj = args.get("url", "")
@@ -211,7 +211,7 @@ def classify_danger(name: str, args: Mapping[str, object]) -> str | None: # noq
soft confirms for mutating tools and risky command launches — interpreters soft confirms for mutating tools and risky command launches — interpreters
(however invoked) and detached runners such as ``nohup``, resolved past (however invoked) and detached runners such as ``nohup``, resolved past
environment wrappers and ``<launcher> run``. environment wrappers and ``<launcher> run``.
Private/loopback fetch targets use a separate policy grant (not YOLO). Private/loopback fetch_local targets use a separate policy grant (not YOLO).
""" """
if name == "delete_path": if name == "delete_path":
return _delete_path_reason(args) return _delete_path_reason(args)
@@ -227,6 +227,6 @@ def classify_danger(name: str, args: Mapping[str, object]) -> str | None: # noq
return _run_argv_batch_reason(args) return _run_argv_batch_reason(args)
if name == "open_pty": if name == "open_pty":
return _open_pty_reason(args) return _open_pty_reason(args)
if name == "fetch": if name == "fetch_local":
return _fetch_reason(args) return _fetch_reason(args)
return None return None
+9 -8
View File
@@ -60,14 +60,15 @@ async def get_truncated(token: str, *, max_chars: int = DEFAULT_TOOL_RESULT_MAX_
"""Fetch the next chunk of a truncated result using its ``truncate_token``. """Fetch the next chunk of a truncated result using its ``truncate_token``.
Pass the token shown at the end of a truncated ``read_file``, ``fetch``, Pass the token shown at the end of a truncated ``read_file``, ``fetch``,
``run_argv``, or earlier ``get_truncated`` result to keep reading without ``fetch_local``, ``run_argv``, or earlier ``get_truncated`` result to keep
re-requesting the whole source or raising limits. Truncated output carries a reading without re-requesting the whole source or raising limits. Truncated
fresh token, so chunks chain indefinitely. File chunks resume in the same output carries a fresh token, so chunks chain indefinitely. File chunks
view as the original read: numbered reads continue as numbered lines resume in the same view as the original read: numbered reads continue as
(editable via ``edit_lineno``), plain reads as raw text with a 1-based numbered lines (editable via ``edit_lineno``), plain reads as raw text with
``L{begin}-{end}`` range. Memory chunks (generic tool output, and ``fetch`` a 1-based ``L{begin}-{end}`` range. Memory chunks (generic tool output, and
bodies) are served from a short-lived in-memory store that is forgotten when ``fetch``/``fetch_local`` bodies) are served from a short-lived in-memory
the agent exits, so resuming a fetch never repeats the request. store that is forgotten when the agent exits, so resuming a fetch never
repeats the request.
""" """
parsed = decode_truncate_token(token) parsed = decode_truncate_token(token)
if parsed is None: if parsed is None:
+2 -2
View File
@@ -1,4 +1,4 @@
from .fetch import fetch as fetch from .fetch_local import fetch_local as fetch_local
from .grants import clear_private_grants as clear_private_grants from .grants import clear_private_grants as clear_private_grants
from .grants import grant_private_host as grant_private_host from .grants import grant_private_host as grant_private_host
from .grants import set_fetch_policy_confirm_hook as set_fetch_policy_confirm_hook from .grants import set_fetch_policy_confirm_hook as set_fetch_policy_confirm_hook
@@ -6,5 +6,5 @@ from .policy import clear_ssrf_assume_public_cidrs as clear_ssrf_assume_public_c
from .policy import set_ssrf_assume_public_cidrs as set_ssrf_assume_public_cidrs from .policy import set_ssrf_assume_public_cidrs as set_ssrf_assume_public_cidrs
NET_TOOLS = [ NET_TOOLS = [
fetch, fetch_local,
] ]
+1 -1
View File
@@ -3,7 +3,7 @@
Bing answers a query with a plain RSS document (``?format=rss``): no key, no Bing answers a query with a plain RSS document (``?format=rss``): no key, no
page scraping, and each ``<item>`` already carries a title, a link and a page scraping, and each ``<item>`` already carries a title, a link and a
description (the snippet DeepSeek's own index does not hand out). The request description (the snippet DeepSeek's own index does not hand out). The request
goes through the fetch client the ``fetch`` tool uses, so the SSRF and goes through the fetch client the ``fetch_local`` tool uses, so the SSRF and
private-host grant policy applies to it as well. private-host grant policy applies to it as well.
""" """
@@ -1,4 +1,4 @@
"""Model-facing ``fetch`` tool (HTTP GET/POST/PUT/DELETE).""" """Model-facing ``fetch_local`` tool (HTTP GET/POST/PUT/DELETE from this host)."""
from __future__ import annotations from __future__ import annotations
@@ -62,7 +62,7 @@ def format_fetch_result(
@tool(tags=ToolTag.LOCAL | ToolTag.INSTANCE_STATE | ToolTag.YOLO | ToolTag.TRUSTABLE | ToolTag.READ_ONLY) @tool(tags=ToolTag.LOCAL | ToolTag.INSTANCE_STATE | ToolTag.YOLO | ToolTag.TRUSTABLE | ToolTag.READ_ONLY)
async def fetch( async def fetch_local(
url: str, url: str,
*, *,
method: str = "GET", method: str = "GET",
@@ -97,7 +97,7 @@ async def fetch(
try: try:
verb = normalize_method(method) verb = normalize_method(method)
if is_read_only_context() and verb != "GET": if is_read_only_context() and verb != "GET":
return "error: fetch method must be GET in a read-only context (POST/PUT/DELETE mutate)" return "error: fetch_local method must be GET in a read-only context (POST/PUT/DELETE mutate)"
parsed = parse_fetch_url(url) parsed = parse_fetch_url(url)
hdrs = normalize_request_headers(headers, user_agent=user_agent) hdrs = normalize_request_headers(headers, user_agent=user_agent)
body_bytes: bytes | None = None body_bytes: bytes | None = None
@@ -119,7 +119,7 @@ async def fetch(
except FetchPolicyError as exc: except FetchPolicyError as exc:
return f"error: {exc}" return f"error: {exc}"
except Exception as exc: # noqa: BLE001 — tool surface returns error text except Exception as exc: # noqa: BLE001 — tool surface returns error text
return f"error: fetch failed: {exc}" return f"error: fetch_local failed: {exc}"
return format_fetch_result( return format_fetch_result(
status=result.status, status=result.status,
+88
View File
@@ -0,0 +1,88 @@
"""Model-facing ``fetch``: read a public page through DeepSeek's Fetch API.
The page is rendered to Markdown server-side, so it is the friendly counterpart
of ``fetch_local``: use it for public articles/docs, and keep ``fetch_local``
for localhost/LAN targets, non-GET requests, custom headers, and raw status or
redirect inspection. The request leaves from DeepSeek's network, not this host,
so local SSRF checks and private-host grants do not apply here.
"""
from __future__ import annotations
import inspect
from typing import TYPE_CHECKING
from plyngent.agent.tools import ToolDefinition, ToolTag, schema_from_callable
from plyngent.tools.net.policy import DEFAULT_MAX_CHARS
from plyngent.tools.truncate_token import truncate_to_memory
if TYPE_CHECKING:
from plyngent.lmproto.deepseek.webtools import DeepseekWebToolsClient
FETCH_TOOL_NAME = "fetch"
def format_page_result(
*,
title: str,
url: str,
content: str,
max_chars: int,
offset: int = 0,
) -> str:
"""Render a fetched page the way the other structured tool results read."""
start = max(0, offset)
body, token = truncate_to_memory(content[start:], max_chars)
parts = [
f"title={' '.join(title.split()) or '(untitled)'}",
f"url={url}",
f"chars={len(content)}",
f"truncated={'true' if token is not None else 'false'}",
"--- content ---",
body,
]
return "\n".join(parts)
def build_fetch_tool(
client: DeepseekWebToolsClient,
*,
default_max_chars: int = DEFAULT_MAX_CHARS,
) -> ToolDefinition:
"""A ``fetch`` definition backed by DeepSeek's Fetch API."""
async def fetch(url: str, offset: int = 0, max_chars: int = default_max_chars) -> str:
"""Read a public web page as Markdown through DeepSeek's Fetch API.
Use for public articles and docs when clean Markdown is enough. The page
is fetched from DeepSeek's network, so localhost/LAN targets are
unreachable here — use ``fetch_local`` for those, for non-GET methods, or
when you need the raw HTTP status/headers. A successful fetch is billed
per request (failures are not).
``offset`` (0-based) skips leading characters of the content. A body cut
by ``max_chars`` keeps its remainder in memory, resumable via its
``truncate_token`` without another request (and without another charge).
"""
try:
page = await client.fetch(url)
except Exception as exc: # noqa: BLE001 — tool surface returns error text
return f"error: fetch failed: {exc}"
return format_page_result(
title=page.title,
url=url,
content=page.content,
max_chars=max_chars,
offset=offset,
)
return ToolDefinition(
name=FETCH_TOOL_NAME,
description=inspect.getdoc(fetch) or "",
parameters=schema_from_callable(fetch),
handler=fetch,
tags=ToolTag.LOCAL | ToolTag.READ_ONLY,
)
__all__ = ["FETCH_TOOL_NAME", "build_fetch_tool", "format_page_result"]
+5 -5
View File
@@ -77,7 +77,7 @@ async def ensure_host_allowed(
try: try:
inst = instance if instance is not None else require_bound_instance() inst = instance if instance is not None else require_bound_instance()
except WorkspaceError as exc: except WorkspaceError as exc:
msg = f"instance state is not bound for fetch policy: {exc}" msg = f"instance state is not bound for fetch_local policy: {exc}"
raise FetchPolicyError(msg) from exc raise FetchPolicyError(msg) from exc
parsed = parse_fetch_url(url) parsed = parse_fetch_url(url)
@@ -85,7 +85,7 @@ async def ensure_host_allowed(
if assessment.classification is HostClass.PUBLIC: if assessment.classification is HostClass.PUBLIC:
return return
if assessment.classification is HostClass.FORBIDDEN: if assessment.classification is HostClass.FORBIDDEN:
msg = f"fetch blocked forbidden host {parsed.host!r} (addresses: {', '.join(assessment.addresses)})" msg = f"fetch_local blocked forbidden host {parsed.host!r} (addresses: {', '.join(assessment.addresses)})"
raise FetchPolicyError(msg) raise FetchPolicyError(msg)
# PRIVATE # PRIVATE
@@ -100,18 +100,18 @@ async def ensure_host_allowed(
hook = get_fetch_policy_confirm_hook(inst) hook = get_fetch_policy_confirm_hook(inst)
if hook is None: if hook is None:
msg = ( msg = (
f"fetch blocked private/loopback host {parsed.host}:{parsed.port} " f"fetch_local blocked private/loopback host {parsed.host}:{parsed.port} "
f"({assessment.reason}; no policy confirm hook / non-interactive deny)" f"({assessment.reason}; no policy confirm hook / non-interactive deny)"
) )
raise FetchPolicyError(msg) raise FetchPolicyError(msg)
try: try:
allowed = bool(hook(parsed.host, parsed.port, parsed.url, timeout)) allowed = bool(hook(parsed.host, parsed.port, parsed.url, timeout))
except Exception as exc: except Exception as exc:
msg = f"fetch blocked private/loopback host {parsed.host}:{parsed.port} (confirm failed: {exc})" msg = f"fetch_local blocked private/loopback host {parsed.host}:{parsed.port} (confirm failed: {exc})"
raise FetchPolicyError(msg) from exc raise FetchPolicyError(msg) from exc
if not allowed: if not allowed:
msg = ( msg = (
f"fetch blocked private/loopback host {parsed.host}:{parsed.port} " f"fetch_local blocked private/loopback host {parsed.host}:{parsed.port} "
f"(user declined or timed out after {timeout:g}s; not skipped by YOLO)" f"(user declined or timed out after {timeout:g}s; not skipped by YOLO)"
) )
raise FetchPolicyError(msg) raise FetchPolicyError(msg)
+1 -1
View File
@@ -1,4 +1,4 @@
"""URL / host policy for the fetch tool (SSRF, private grants, cleartext).""" """URL / host policy for the fetch_local tool (SSRF, private grants, cleartext)."""
from __future__ import annotations from __future__ import annotations
+7 -5
View File
@@ -9,7 +9,8 @@ the handler asks the configured search sources behind the scenes —
one fails or finds nothing. DeepSeek's own source is the Search API one fails or finds nothing. DeepSeek's own source is the Search API
(``POST /v1/tools/search``), which answers in one request instead of a model (``POST /v1/tools/search``), which answers in one request instead of a model
round. A hit is a title, a URL and (when the source has one) a snippet: read a round. A hit is a title, a URL and (when the source has one) a snippet: read a
page with ``fetch`` before relying on its content. page with ``fetch`` (or ``fetch_local`` for local/LAN targets) before relying on
its content.
""" """
from __future__ import annotations from __future__ import annotations
@@ -22,7 +23,7 @@ from plyngent.agent.tools import ToolDefinition, ToolTag, schema_from_callable
if TYPE_CHECKING: if TYPE_CHECKING:
from collections.abc import Sequence from collections.abc import Sequence
from plyngent.lmproto.deepseek.search import DeepseekSearchClient from plyngent.lmproto.deepseek.webtools import DeepseekWebToolsClient
from plyngent.lmproto.search import SearchHit from plyngent.lmproto.search import SearchHit
WEB_SEARCH_TOOL_NAME = "web_search" WEB_SEARCH_TOOL_NAME = "web_search"
@@ -38,9 +39,9 @@ class SearchBackend(Protocol):
class DeepseekSearchBackend: class DeepseekSearchBackend:
"""DeepSeek's own index: one Search API request per query.""" """DeepSeek's own index: one Search API request per query."""
_client: DeepseekSearchClient _client: DeepseekWebToolsClient
def __init__(self, client: DeepseekSearchClient) -> None: def __init__(self, client: DeepseekWebToolsClient) -> None:
self._client = client self._client = client
async def search(self, query: str) -> Sequence[SearchHit]: async def search(self, query: str) -> Sequence[SearchHit]:
@@ -82,7 +83,8 @@ def build_web_search_tool(
"""Search the web and return the pages that matched. """Search the web and return the pages that matched.
One entry per hit (``title — url``, plus the snippet when the source has One entry per hit (``title — url``, plus the snippet when the source has
one): read a page with ``fetch`` before relying on its content. one): read a page with ``fetch`` (or ``fetch_local`` for local targets)
before relying on its content.
""" """
found, failures = await search_in_order(backends, query) found, failures = await search_in_order(backends, query)
if not found: if not found:
+5 -5
View File
@@ -2,8 +2,8 @@
Every truncation site (the agent loop's generic tool-result cap, Every truncation site (the agent loop's generic tool-result cap,
``run_argv`` / ``run_argv_batch`` per-stream caps, ``read_file``, ``run_argv`` / ``run_argv_batch`` per-stream caps, ``read_file``,
``fetch``, and request-time compact shrinks) appends the same compact marker ``fetch`` / ``fetch_local``, and request-time compact shrinks) appends the same
when output is cut short:: compact marker when output is cut short::
[Truncated (32000 chars max.; 500 omitted). truncate_token=...] [Truncated (32000 chars max.; 500 omitted). truncate_token=...]
@@ -14,9 +14,9 @@ source is exhausted (then no marker is emitted).
A ``file`` remainder is re-read from disk by ``location`` + ``offset``, so it A ``file`` remainder is re-read from disk by ``location`` + ``offset``, so it
survives for as long as the file does. Everything else — arbitrary tool output survives for as long as the file does. Everything else — arbitrary tool output
(run_argv stdout, todo renders, …) and ``fetch`` bodies — has no source worth (run_argv stdout, todo renders, …) and ``fetch`` / ``fetch_local`` bodies — have
re-running (a re-fetch costs a round trip and re-hits the remote), so its no source worth re-running (a re-fetch costs a round trip and re-hits the remote),
remainder lives in a short-lived in-memory store (``kind="memory"``) that is so its remainder lives in a short-lived in-memory store (``kind="memory"``) that is
forgotten when the agent process exits; the model must re-run the tool after forgotten when the agent process exits; the model must re-run the tool after
that. that.
""" """
+25 -5
View File
@@ -544,7 +544,7 @@ async def test_pretty_fetch(capsys: pytest.CaptureFixture[str]) -> None:
"status=200\nmethod=GET\nfinal_url=https://example.com\ncontent_type=application/json\n" "status=200\nmethod=GET\nfinal_url=https://example.com\ncontent_type=application/json\n"
"body_kind=json\nbytes=1229\ntruncated=false\nredirects=0\nsecurity=public\n--- body ---\n{...}\n" "body_kind=json\nbytes=1229\ntruncated=false\nredirects=0\nsecurity=public\n--- body ---\n{...}\n"
) )
await render_events(_aiter([_pretty_call("fetch", '{"url": "https://example.com"}'), _result(result)])) await render_events(_aiter([_pretty_call("fetch_local", '{"url": "https://example.com"}'), _result(result)]))
out = capsys.readouterr().out out = capsys.readouterr().out
assert "* Fetch GET https://example.com (200)" in out assert "* Fetch GET https://example.com (200)" in out
assert "[tool]" not in out assert "[tool]" not in out
@@ -555,7 +555,7 @@ async def test_pretty_fetch_error_status(capsys: pytest.CaptureFixture[str]) ->
"status=404\nmethod=GET\nfinal_url=https://example.com/nope\ncontent_type=text/html\n" "status=404\nmethod=GET\nfinal_url=https://example.com/nope\ncontent_type=text/html\n"
"body_kind=html\nbytes=45\ntruncated=false\nredirects=0\nsecurity=public\n--- body ---\nNot found\n" "body_kind=html\nbytes=45\ntruncated=false\nredirects=0\nsecurity=public\n--- body ---\nNot found\n"
) )
await render_events(_aiter([_pretty_call("fetch", '{"url": "https://example.com/nope"}'), _result(result)])) await render_events(_aiter([_pretty_call("fetch_local", '{"url": "https://example.com/nope"}'), _result(result)]))
out = capsys.readouterr().out out = capsys.readouterr().out
assert "* Fetch GET https://example.com/nope (404)" in out assert "* Fetch GET https://example.com/nope (404)" in out
@@ -565,17 +565,37 @@ async def test_pretty_fetch_truncated(capsys: pytest.CaptureFixture[str]) -> Non
"status=200\nmethod=GET\nfinal_url=https://example.com\ncontent_type=text/plain\n" "status=200\nmethod=GET\nfinal_url=https://example.com\ncontent_type=text/plain\n"
"body_kind=text\nbytes=50000\ntruncated=true\nredirects=0\nsecurity=public\n--- body ---\n..." "body_kind=text\nbytes=50000\ntruncated=true\nredirects=0\nsecurity=public\n--- body ---\n..."
) )
await render_events(_aiter([_pretty_call("fetch", '{"url": "https://example.com"}'), _result(result)])) await render_events(_aiter([_pretty_call("fetch_local", '{"url": "https://example.com"}'), _result(result)]))
out = capsys.readouterr().out out = capsys.readouterr().out
assert "* Fetch GET https://example.com (200)" in out assert "* Fetch GET https://example.com (200)" in out
async def test_pretty_fetch_error(capsys: pytest.CaptureFixture[str]) -> None: async def test_pretty_fetch_error(capsys: pytest.CaptureFixture[str]) -> None:
await render_events( await render_events(
_aiter([_pretty_call("fetch", '{"url": "https://example.com"}'), _result("error: fetch failed: boom")]) _aiter(
[_pretty_call("fetch_local", '{"url": "https://example.com"}'), _result("error: fetch_local failed: boom")]
)
) )
out = capsys.readouterr().out out = capsys.readouterr().out
assert "* Fetch GET https://example.com (error: fetch failed: boom)" in out assert "* Fetch GET https://example.com (error: fetch_local failed: boom)" in out
async def test_pretty_remote_fetch(capsys: pytest.CaptureFixture[str]) -> None:
result = "title=Example article\nurl=https://example.com\nchars=1234\ntruncated=false\n--- content ---\n..."
await render_events(_aiter([_pretty_call("fetch", '{"url": "https://example.com"}'), _result(result)]))
out = capsys.readouterr().out
assert "* Fetch https://example.com (1234 chars)" in out
async def test_pretty_fetch_reads_a_legacy_local_row(capsys: pytest.CaptureFixture[str]) -> None:
# Rows written before the local tool was renamed carry the local status block.
result = (
"status=200\nmethod=GET\nfinal_url=https://example.com\ncontent_type=text/plain\n"
"body_kind=text\nbytes=12\ntruncated=false\nredirects=0\nsecurity=public\n--- body ---\nhi\n"
)
await render_events(_aiter([_pretty_call("fetch", '{"url": "https://example.com"}'), _result(result)]))
out = capsys.readouterr().out
assert "* Fetch https://example.com (200)" in out
async def test_pretty_web_search(capsys: pytest.CaptureFixture[str]) -> None: async def test_pretty_web_search(capsys: pytest.CaptureFixture[str]) -> None:
@@ -1,12 +1,12 @@
"""DeepSeek's Search API client (``POST /v1/tools/search``).""" """DeepSeek's Search and Fetch APIs (``/v1/tools/search``, ``/v1/tools/fetch``)."""
from __future__ import annotations from __future__ import annotations
import msgspec import msgspec
import pytest import pytest
from plyngent.lmproto.deepseek.search import ( from plyngent.lmproto.deepseek.webtools import (
DeepseekSearchClient, DeepseekWebToolsClient,
SearchRequest, SearchRequest,
) )
from plyngent.lmproto.openai_compatible.config import OpenAIConfig from plyngent.lmproto.openai_compatible.config import OpenAIConfig
@@ -23,8 +23,8 @@ def _body(*results: dict[str, object]) -> bytes:
return msgspec.json.encode({"results": list(results)}) return msgspec.json.encode({"results": list(results)})
def _client() -> DeepseekSearchClient: def _client() -> DeepseekWebToolsClient:
return DeepseekSearchClient( return DeepseekWebToolsClient(
OpenAIConfig(access_key_or_token="sk-test", base_url="https://api.deepseek.com"), OpenAIConfig(access_key_or_token="sk-test", base_url="https://api.deepseek.com"),
) )
@@ -166,7 +166,7 @@ async def test_the_endpoint_error_detail_is_raised(monkeypatch: pytest.MonkeyPat
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_a_retryable_status_is_tried_again(monkeypatch: pytest.MonkeyPatch) -> None: async def test_a_retryable_status_is_tried_again(monkeypatch: pytest.MonkeyPatch) -> None:
client = _client() client = _client()
monkeypatch.setattr("plyngent.lmproto.deepseek.search.client.RETRY_DELAYS", (0.0,)) monkeypatch.setattr("plyngent.lmproto.deepseek.webtools.client.RETRY_DELAYS", (0.0,))
statuses = [429, 200] statuses = [429, 200]
async def fake_post(path: str, **kwargs: object) -> _Resp: async def fake_post(path: str, **kwargs: object) -> _Resp:
@@ -181,7 +181,7 @@ async def test_a_retryable_status_is_tried_again(monkeypatch: pytest.MonkeyPatch
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_a_balance_error_is_not_retried(monkeypatch: pytest.MonkeyPatch) -> None: async def test_a_balance_error_is_not_retried(monkeypatch: pytest.MonkeyPatch) -> None:
client = _client() client = _client()
monkeypatch.setattr("plyngent.lmproto.deepseek.search.client.RETRY_DELAYS", (0.0,)) monkeypatch.setattr("plyngent.lmproto.deepseek.webtools.client.RETRY_DELAYS", (0.0,))
calls: list[int] = [] calls: list[int] = []
async def fake_post(path: str, **kwargs: object) -> _Resp: async def fake_post(path: str, **kwargs: object) -> _Resp:
@@ -197,3 +197,50 @@ async def test_a_balance_error_is_not_retried(monkeypatch: pytest.MonkeyPatch) -
def test_the_request_model_omits_what_was_not_set() -> None: def test_the_request_model_omits_what_was_not_set() -> None:
encoded = msgspec.json.encode(SearchRequest(query="q")) encoded = msgspec.json.encode(SearchRequest(query="q"))
assert msgspec.json.decode(encoded, type=dict[str, object]) == {"query": "q"} assert msgspec.json.decode(encoded, type=dict[str, object]) == {"query": "q"}
@pytest.mark.asyncio
async def test_fetch_posts_the_url_and_returns_the_page(monkeypatch: pytest.MonkeyPatch) -> None:
client = _client()
sent: dict[str, object] = {}
async def fake_post(path: str, **kwargs: object) -> _Resp:
sent["path"] = path
sent.update(kwargs)
return _Resp(200, msgspec.json.encode({"title": "Example article", "content": "# Article\n\nBody."}))
monkeypatch.setattr(client.session, "post", fake_post)
page = await client.fetch("https://example.com/article")
assert page.title == "Example article"
assert page.content == "# Article\n\nBody."
assert sent["path"] == "/v1/tools/fetch"
assert _decode_request(sent["data"]) == {"url": "https://example.com/article"}
@pytest.mark.asyncio
async def test_fetch_rejects_a_url_the_endpoint_would_refuse(monkeypatch: pytest.MonkeyPatch) -> None:
client = _client()
calls: list[object] = []
async def fake_post(path: str, **kwargs: object) -> _Resp:
calls.append(path)
return _Resp(200, b"{}")
monkeypatch.setattr(client.session, "post", fake_post)
for url in ("file:///etc/passwd", "https://user:secret@example.com/", "https:///no-host", ""):
with pytest.raises(ValueError):
_ = await client.fetch(url)
assert calls == []
@pytest.mark.asyncio
async def test_fetch_surfaces_a_security_policy_rejection(monkeypatch: pytest.MonkeyPatch) -> None:
client = _client()
async def fake_post(path: str, **kwargs: object) -> _Resp:
return _Resp(403, b'{"error": {"message": "target rejected by security policy"}}')
monkeypatch.setattr(client.session, "post", fake_post)
with pytest.raises(RuntimeError, match="security policy"):
_ = await client.fetch("https://example.com/blocked")
+1 -1
View File
@@ -74,7 +74,7 @@ def test_read_only_classification() -> None:
"vcs_diff", "vcs_diff",
"vcs_log", "vcs_log",
"vcs_branch", "vcs_branch",
"fetch", "fetch_local",
"skill_list", "skill_list",
"skill_read", "skill_read",
"skill_search", "skill_search",
@@ -1,4 +1,4 @@
"""Tests for tools.net.fetch (policy, UA, methods, private grants).""" """Tests for tools.net.fetch_local (policy, UA, methods, private grants)."""
from __future__ import annotations from __future__ import annotations
@@ -10,7 +10,7 @@ import pytest
from plyngent.tools.context import read_only_context from plyngent.tools.context import read_only_context
from plyngent.tools.danger import classify_danger from plyngent.tools.danger import classify_danger
from plyngent.tools.net import fetch, grant_private_host from plyngent.tools.net import fetch_local, grant_private_host
from plyngent.tools.net.policy import ( from plyngent.tools.net.policy import (
DEFAULT_USER_AGENT, DEFAULT_USER_AGENT,
FetchPolicyError, FetchPolicyError,
@@ -30,7 +30,7 @@ if TYPE_CHECKING:
class _Handler(BaseHTTPRequestHandler): class _Handler(BaseHTTPRequestHandler):
"""Minimal echo server for fetch tests.""" """Minimal echo server for fetch_local tests."""
@override @override
def log_message(self, format: str, *args: object) -> None: def log_message(self, format: str, *args: object) -> None:
@@ -171,10 +171,10 @@ def test_soft_confirm_reason_matrix() -> None:
def test_classify_danger_fetch() -> None: def test_classify_danger_fetch() -> None:
assert classify_danger("fetch", {"url": "https://example.com/", "method": "GET"}) is None assert classify_danger("fetch_local", {"url": "https://example.com/", "method": "GET"}) is None
reason = classify_danger("fetch", {"url": "http://example.com/", "method": "GET"}) reason = classify_danger("fetch_local", {"url": "http://example.com/", "method": "GET"})
assert reason is not None and "cleartext" in reason assert reason is not None and "cleartext" in reason
reason2 = classify_danger("fetch", {"url": "https://example.com/api", "method": "DELETE"}) reason2 = classify_danger("fetch_local", {"url": "https://example.com/api", "method": "DELETE"})
assert reason2 is not None and "DELETE" in reason2 assert reason2 is not None and "DELETE" in reason2
@@ -184,36 +184,36 @@ async def test_fetch_get_post_put_delete(workspace: Path, http_server: str) -> N
# Grant loopback for this process/instance (fixture binds InstanceState). # Grant loopback for this process/instance (fixture binds InstanceState).
grant_private_host("127.0.0.1", int(base.rsplit(":", 1)[1])) grant_private_host("127.0.0.1", int(base.rsplit(":", 1)[1]))
out = await call_async(fetch, f"{base}/hello") out = await call_async(fetch_local, f"{base}/hello")
assert "status=200" in out assert "status=200" in out
assert "GET /hello" in out assert "GET /hello" in out
assert "security=cleartext-http" in out assert "security=cleartext-http" in out
out_post = await call_async(fetch, f"{base}/echo", method="POST", body="hi") out_post = await call_async(fetch_local, f"{base}/echo", method="POST", body="hi")
assert "status=201" in out_post assert "status=201" in out_post
assert "POST:hi" in out_post assert "POST:hi" in out_post
out_put = await call_async(fetch, f"{base}/echo", method="PUT", body="x") out_put = await call_async(fetch_local, f"{base}/echo", method="PUT", body="x")
assert "PUT:x" in out_put assert "PUT:x" in out_put
out_del = await call_async(fetch, f"{base}/x", method="DELETE") out_del = await call_async(fetch_local, f"{base}/x", method="DELETE")
assert "status=204" in out_del assert "status=204" in out_del
async def test_fetch_read_only_context_blocks_non_get(workspace: Path, http_server: str) -> None: async def test_fetch_read_only_context_blocks_non_get(workspace: Path, http_server: str) -> None:
"""In a read-only context fetch only allows GET; outside it POST works.""" """In a read-only context fetch_local only allows GET; outside it POST works."""
del workspace del workspace
port = int(http_server.rsplit(":", 1)[1]) port = int(http_server.rsplit(":", 1)[1])
grant_private_host("127.0.0.1", port) grant_private_host("127.0.0.1", port)
with read_only_context(): with read_only_context():
out = await call_async(fetch, f"{http_server}/hello") out = await call_async(fetch_local, f"{http_server}/hello")
assert "status=200" in out # GET is fine in a read-only context assert "status=200" in out # GET is fine in a read-only context
blocked = await call_async(fetch, f"{http_server}/echo", method="POST", body="hi") blocked = await call_async(fetch_local, f"{http_server}/echo", method="POST", body="hi")
assert blocked.startswith("error:") assert blocked.startswith("error:")
assert "GET" in blocked assert "GET" in blocked
out_post = await call_async(fetch, f"{http_server}/echo", method="POST", body="hi") out_post = await call_async(fetch_local, f"{http_server}/echo", method="POST", body="hi")
assert "POST:hi" in out_post assert "POST:hi" in out_post
@@ -222,18 +222,18 @@ async def test_fetch_user_agent_passthrough(workspace: Path, http_server: str) -
port = int(http_server.rsplit(":", 1)[1]) port = int(http_server.rsplit(":", 1)[1])
grant_private_host("127.0.0.1", port) grant_private_host("127.0.0.1", port)
out = await call_async(fetch, f"{http_server}/ua", user_agent="AgentUA/9") out = await call_async(fetch_local, f"{http_server}/ua", user_agent="AgentUA/9")
assert "AgentUA/9" in out assert "AgentUA/9" in out
out2 = await call_async( out2 = await call_async(
fetch, fetch_local,
f"{http_server}/ua", f"{http_server}/ua",
headers={"User-Agent": "HeaderUA/1"}, headers={"User-Agent": "HeaderUA/1"},
) )
assert "HeaderUA/1" in out2 assert "HeaderUA/1" in out2
out3 = await call_async( out3 = await call_async(
fetch, fetch_local,
f"{http_server}/ua", f"{http_server}/ua",
headers={"User-Agent": "HeaderUA/1"}, headers={"User-Agent": "HeaderUA/1"},
user_agent="ArgUA/2", user_agent="ArgUA/2",
@@ -247,11 +247,11 @@ async def test_fetch_truncation_and_binary(workspace: Path, http_server: str) ->
port = int(http_server.rsplit(":", 1)[1]) port = int(http_server.rsplit(":", 1)[1])
grant_private_host("127.0.0.1", port) grant_private_host("127.0.0.1", port)
out = await call_async(fetch, f"{http_server}/big", max_bytes=100) out = await call_async(fetch_local, f"{http_server}/big", max_bytes=100)
assert "truncated=true" in out assert "truncated=true" in out
assert "bytes=100" in out assert "bytes=100" in out
out_bin = await call_async(fetch, f"{http_server}/bin") out_bin = await call_async(fetch_local, f"{http_server}/bin")
assert "body_kind=binary" in out_bin assert "body_kind=binary" in out_bin
assert "binary body omitted" in out_bin assert "binary body omitted" in out_bin
@@ -261,7 +261,7 @@ async def test_fetch_char_truncation_embeds_token(workspace: Path, http_server:
port = int(http_server.rsplit(":", 1)[1]) port = int(http_server.rsplit(":", 1)[1])
grant_private_host("127.0.0.1", port) grant_private_host("127.0.0.1", port)
out = await call_async(fetch, f"{http_server}/big", max_chars=1000) out = await call_async(fetch_local, f"{http_server}/big", max_chars=1000)
assert "truncated=true" in out assert "truncated=true" in out
body = out.split("--- body ---", 1)[-1] body = out.split("--- body ---", 1)[-1]
assert "truncate_token=" in body assert "truncate_token=" in body
@@ -281,7 +281,7 @@ async def test_fetch_offset_skips_body(workspace: Path, http_server: str) -> Non
grant_private_host("127.0.0.1", port) grant_private_host("127.0.0.1", port)
# /big body is 5000 chars; offset 4900 leaves 100 chars, no token needed. # /big body is 5000 chars; offset 4900 leaves 100 chars, no token needed.
out = await call_async(fetch, f"{http_server}/big", max_chars=200, offset=4900) out = await call_async(fetch_local, f"{http_server}/big", max_chars=200, offset=4900)
body = out.split("--- body ---", 1)[-1] body = out.split("--- body ---", 1)[-1]
assert body.strip() == "x" * 100 assert body.strip() == "x" * 100
assert "truncate_token=" not in out assert "truncate_token=" not in out
@@ -290,21 +290,21 @@ async def test_fetch_offset_skips_body(workspace: Path, http_server: str) -> Non
async def test_fetch_private_denied_without_grant(workspace: Path, http_server: str) -> None: async def test_fetch_private_denied_without_grant(workspace: Path, http_server: str) -> None:
del workspace del workspace
# No grant, no policy hook → hard deny (even though server is up). # No grant, no policy hook → hard deny (even though server is up).
out = await call_async(fetch, f"{http_server}/hello") out = await call_async(fetch_local, f"{http_server}/hello")
assert out.startswith("error:") assert out.startswith("error:")
assert "private" in out or "loopback" in out assert "private" in out or "loopback" in out
async def test_fetch_forbidden_metadata(workspace: Path) -> None: async def test_fetch_forbidden_metadata(workspace: Path) -> None:
del workspace del workspace
out = await call_async(fetch, "http://169.254.169.254/latest/meta-data/") out = await call_async(fetch_local, "http://169.254.169.254/latest/meta-data/")
assert out.startswith("error:") assert out.startswith("error:")
assert "forbidden" in out assert "forbidden" in out
async def test_fetch_bad_scheme(workspace: Path) -> None: async def test_fetch_bad_scheme(workspace: Path) -> None:
del workspace del workspace
out = await call_async(fetch, "file:///etc/passwd") out = await call_async(fetch_local, "file:///etc/passwd")
assert out.startswith("error:") assert out.startswith("error:")
assert "scheme" in out assert "scheme" in out
@@ -313,6 +313,6 @@ async def test_fetch_redirect_loop_capped(workspace: Path, http_server: str) ->
del workspace del workspace
port = int(http_server.rsplit(":", 1)[1]) port = int(http_server.rsplit(":", 1)[1])
grant_private_host("127.0.0.1", port) grant_private_host("127.0.0.1", port)
out = await call_async(fetch, f"{http_server}/redirect-loop", max_redirects=3) out = await call_async(fetch_local, f"{http_server}/redirect-loop", max_redirects=3)
assert out.startswith("error:") assert out.startswith("error:")
assert "redirect" in out.lower() assert "redirect" in out.lower()
+73
View File
@@ -0,0 +1,73 @@
"""The remote ``fetch`` tool (DeepSeek's Fetch API, Markdown pages)."""
from __future__ import annotations
import pytest
from plyngent.agent.tools import ToolTag
from plyngent.lmproto.deepseek.webtools import FetchResponse
from plyngent.tools.net.fetch_remote import FETCH_TOOL_NAME, build_fetch_tool
class _Client:
"""Stands in for DeepseekWebToolsClient.fetch."""
def __init__(self, *, page: FetchResponse | None = None, error: str | None = None) -> None:
self.page = page or FetchResponse(title="T", content="body")
self.error = error
self.urls: list[str] = []
async def fetch(self, url: str) -> FetchResponse:
self.urls.append(url)
if self.error is not None:
raise RuntimeError(self.error)
return self.page
def _tool(client: _Client, *, max_chars: int = 32_000):
return build_fetch_tool(client, default_max_chars=max_chars)
def test_definition_shape() -> None:
definition = _tool(_Client())
assert definition.name == FETCH_TOOL_NAME == "fetch"
assert definition.tags == ToolTag.LOCAL | ToolTag.READ_ONLY
properties = definition.parameters["properties"]
assert set(properties) == {"url", "offset", "max_chars"}
assert definition.parameters["required"] == ["url"]
assert "fetch_local" in definition.description
@pytest.mark.asyncio
async def test_handler_renders_title_url_and_content() -> None:
client = _Client(page=FetchResponse(title=" Example article ", content="# Heading\n\nBody."))
result = await _tool(client).handler(url="https://example.com/article")
assert client.urls == ["https://example.com/article"]
assert result.splitlines() == [
"title=Example article",
"url=https://example.com/article",
"chars=16",
"truncated=false",
"--- content ---",
"# Heading",
"",
"Body.",
]
@pytest.mark.asyncio
async def test_offset_and_truncation_keep_the_remainder_resumable() -> None:
client = _Client(page=FetchResponse(title="", content="0123456789"))
result = await _tool(client, max_chars=3).handler(url="https://example.com/", offset=5)
assert "title=(untitled)" in result
assert "--- content ---\n567" in result
assert "truncated=true" in result
assert "truncate_token=" in result
@pytest.mark.asyncio
async def test_handler_reports_a_client_error() -> None:
result = await _tool(_Client(error="HTTP 404: no usable content")).handler(url="https://example.com/x")
assert result == "error: fetch failed: HTTP 404: no usable content"
+2 -2
View File
@@ -86,9 +86,9 @@ async def test_get_truncated_file_missing(workspace: Path) -> None:
async def test_get_truncated_serves_a_fetch_body_without_refetching(monkeypatch) -> None: async def test_get_truncated_serves_a_fetch_body_without_refetching(monkeypatch) -> None:
"""A fetch body's token reads the stored remainder; the request is not repeated.""" """A fetch body's token reads the stored remainder; the request is not repeated."""
from plyngent.tools.net.fetch import format_fetch_result from plyngent.tools.net.fetch_local import format_fetch_result
fetch_module = importlib.import_module("plyngent.tools.net.fetch") fetch_module = importlib.import_module("plyngent.tools.net.fetch_local")
async def _must_not_refetch(*_args: object, **_kwargs: object) -> str: async def _must_not_refetch(*_args: object, **_kwargs: object) -> str:
msg = "resuming a fetch body must not re-run the request" msg = "resuming a fetch body must not re-run the request"