diff --git a/src/plyngent/agent/directive_checkpoint.py b/src/plyngent/agent/directive_checkpoint.py index 5c0ad53..881081c 100644 --- a/src/plyngent/agent/directive_checkpoint.py +++ b/src/plyngent/agent/directive_checkpoint.py @@ -26,8 +26,8 @@ use `run_argv_batch` for ordered pipelines. - `regex_files`: `pattern` is a Python `re` regex (not a glob, so `*.py` fails) and \ `path` is a literal file/dir (no glob expansion). - Truncated results carry a `truncate_token=...`; continue with `get_truncated`. -- Prefer `fetch` for HTTP(S); `offset` (0-based) skips leading body characters; \ -private/LAN hosts need human policy allow (not YOLO). +- Prefer `fetch` for public pages (clean Markdown); `fetch_local` for localhost/LAN, \ +non-GET methods, headers, or raw status; `offset` (0-based) skips leading body characters. - PTY secrets only via `ask_into_pty`; denylists and confirms still apply. - `wait` pauses; pressing Enter disturbs early (optional reason). - Todo stack: open items mean unfinished work; the stack does not auto-clear — \ diff --git a/src/plyngent/agent/notices.py b/src/plyngent/agent/notices.py index 448b7f5..229928a 100644 --- a/src/plyngent/agent/notices.py +++ b/src/plyngent/agent/notices.py @@ -115,8 +115,8 @@ def truncated_turn_notice() -> Notice: _ASIDE_BODIES: dict[str, str] = { "read": ( "Side question: this exchange is not saved to the session transcript and does not " - "change the main conversation. Tools here are read-only (file/VCS reads, ask, fetch " - "GET): you cannot edit files or run state-changing commands, so describe a change " + "change the main conversation. Tools here are read-only (file/VCS reads, ask, fetch/" + "fetch_local GET): you cannot edit files or run state-changing commands, so describe a change " "instead of applying it." ), "no": ( diff --git a/src/plyngent/cli/app.py b/src/plyngent/cli/app.py index 46f21a2..bb11233 100644 --- a/src/plyngent/cli/app.py +++ b/src/plyngent/cli/app.py @@ -341,7 +341,7 @@ async def _run_chat( # noqa: C901, PLR0915 — chat orchestration state.instance_state.workspace.path_denylist = tuple(store.agent_config.path_denylist or ()) from plyngent.tools.net import set_ssrf_assume_public_cidrs - # Clash Fake-IP etc.: treat configured CIDRs as public for fetch SSRF. + # Clash Fake-IP etc.: treat configured CIDRs as public for fetch_local SSRF. _ = set_ssrf_assume_public_cidrs(store.networking_config.ssrf_assume_public_cidrs) if interactive: from plyngent.cli.limits import prompt_policy_command_confirm, prompt_policy_fetch_confirm diff --git a/src/plyngent/cli/display.py b/src/plyngent/cli/display.py index c554fe6..31822ed 100644 --- a/src/plyngent/cli/display.py +++ b/src/plyngent/cli/display.py @@ -452,13 +452,34 @@ def _http_status_fg(status: str) -> str | None: def _fetch_prefix(args_json: str) -> str: + """Remote ``fetch``: the URL is the whole call.""" + url = _json_str_arg(args_json, "url") or "?" + return _pretty_prefix(f"* Fetch {url} ") + + +def _fetch_detail(args_json: str, result: str) -> str: + """Remote ``fetch`` outcome: fetched size, or an error. + + A row written before the local tool was renamed still carries the local + ``status=...`` block; render it with the local formatter. + """ + if result.startswith("error:"): + return _pretty_segments((f"({result})", "red")) + fields = _status_fields(result) + if "status" in fields: + return _fetch_local_detail(args_json, result) + chars = fields.get("chars", "") + return _pretty_segments((f"({chars} chars)" if chars else "(done)", "green")) + + +def _fetch_local_prefix(args_json: str) -> str: method = _json_str_arg(args_json, "method") or "GET" url = _json_str_arg(args_json, "url") or "?" return _pretty_prefix(f"* Fetch {method} {url} ") -def _fetch_detail(_args_json: str, result: str) -> str: - """``fetch`` outcome: HTTP status, or error.""" +def _fetch_local_detail(_args_json: str, result: str) -> str: + """``fetch_local`` outcome: HTTP status, or error.""" if result.startswith("error:"): return _pretty_segments((f"({result})", "red")) status = _status_fields(result).get("status", "") @@ -888,6 +909,7 @@ _PRETTY_LINES: dict[str, PrettyLine] = { "run_argv": PrettyLine(prefix=_run_argv_prefix, detail=_run_argv_detail), "run_argv_batch": PrettyLine(prefix=_run_argv_batch_prefix, detail=_run_argv_batch_detail), "fetch": PrettyLine(prefix=_fetch_prefix, detail=_fetch_detail), + "fetch_local": PrettyLine(prefix=_fetch_local_prefix, detail=_fetch_local_detail), "web_search": PrettyLine(prefix=_web_search_prefix, detail=_web_search_detail), "edit_replace": _mutator_line("edit_replace"), "edit_lineno": _mutator_line("edit_lineno"), diff --git a/src/plyngent/cli/limits.py b/src/plyngent/cli/limits.py index 387f0e3..c944394 100644 --- a/src/plyngent/cli/limits.py +++ b/src/plyngent/cli/limits.py @@ -311,14 +311,14 @@ def prompt_policy_fetch_confirm( url: str, timeout_seconds: float, ) -> bool: - """Ask whether to allow fetch to a private/loopback host (timed). + """Ask whether to allow fetch_local to a private/loopback host (timed). Independent of YOLO: always prompts when interactive. Default is **deny**. Timeout, cancel, or non-interactive → False. On approve, the host:port is granted for the rest of this process (see net grants). """ reason = ( - f"fetch targets private/loopback host {host!r} port {port}\n" + f"fetch_local targets private/loopback host {host!r} port {port}\n" f"url: {url}\n" f"Allow this host:port for the rest of this process?\n" f"(timeout {timeout_seconds:g}s defaults to DENY; not skipped by YOLO)" @@ -332,7 +332,7 @@ def prompt_policy_fetch_confirm( backend.secho(format_tool_confirm_box("policy", reason), fg="yellow") backend.echo() grant = f"{host}:{port}" - prompt = f"[policy] allow fetch {grant!r}? [y/N] (timeout {timeout_seconds:g}s): " + prompt = f"[policy] allow fetch_local {grant!r}? [y/N] (timeout {timeout_seconds:g}s): " with contextlib.suppress(OSError): _ = sys.stderr.write(prompt) _ = sys.stderr.flush() diff --git a/src/plyngent/cli/state.py b/src/plyngent/cli/state.py index 7b44913..a274bf4 100644 --- a/src/plyngent/cli/state.py +++ b/src/plyngent/cli/state.py @@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Literal from plyngent.agent import ChatAgent, ToolRegistry from plyngent.agent.loop import DEFAULT_MAX_ROUNDS from plyngent.agent.todo_stack import TodoStack +from plyngent.lmproto.deepseek.webtools import DeepseekWebToolsClient if TYPE_CHECKING: from datetime import datetime @@ -382,20 +383,43 @@ class ReplState: search_tool = self._web_search_tool() if search_tool is not None: registry.register(search_tool) + fetch_tool = self._remote_fetch_tool() + if fetch_tool is not None: + registry.register(fetch_tool) return registry + def _deepseek_web_client(self) -> DeepseekWebToolsClient | None: + """The Search/Fetch API client for the active provider (None otherwise).""" + from plyngent.config.routing import deepseek_api_base_url + from plyngent.lmproto.openai_compatible.config import OpenAIConfig + + provider = self.effective_provider + if provider.preset != "deepseek": + return None + return DeepseekWebToolsClient( + OpenAIConfig( + access_key_or_token=provider.access_key_or_token, + base_url=deepseek_api_base_url(provider), + timeout=normalize_http_timeout(provider.timeout), + ) + ) + + def _remote_fetch_tool(self) -> ToolDefinition | None: + """``fetch`` through DeepSeek's Fetch API (DeepSeek providers only).""" + from plyngent.tools.net.fetch_remote import build_fetch_tool + + client = self._deepseek_web_client() + if client is None: + return None + return build_fetch_tool(client) + def _web_search_tool(self) -> ToolDefinition | None: """Local ``web_search`` for a provider whose surface cannot host one. The sources come from ``provider.search_sources``, in fallback order (see :func:`~plyngent.config.routing.local_web_search`). """ - from plyngent.config.routing import ( - deepseek_api_base_url, - local_web_search, - ) - from plyngent.lmproto.deepseek.search import DeepseekSearchClient - from plyngent.lmproto.openai_compatible.config import OpenAIConfig + from plyngent.config.routing import local_web_search from plyngent.tools.net.bing import BingSearchBackend from plyngent.tools.net.search import ( DeepseekSearchBackend, @@ -408,17 +432,10 @@ class ReplState: backends: list[SearchBackend] = [] for source in provider.search_sources: if source == "deepseek": - backends.append( - DeepseekSearchBackend( - DeepseekSearchClient( - OpenAIConfig( - access_key_or_token=provider.access_key_or_token, - base_url=deepseek_api_base_url(provider), - timeout=normalize_http_timeout(provider.timeout), - ) - ) - ) - ) + client = self._deepseek_web_client() + if client is None: + continue + backends.append(DeepseekSearchBackend(client)) elif source == "bing": backends.append(BingSearchBackend()) if not backends: diff --git a/src/plyngent/config/models.py b/src/plyngent/config/models.py index 9eb3627..cec9669 100644 --- a/src/plyngent/config/models.py +++ b/src/plyngent/config/models.py @@ -71,18 +71,24 @@ with it to keep reading (chains through truncations). a `-c` one-liner, a script (`python x.py`), or a bare interactive shell alike — and each \ confirm blocks the turn, so it costs time. Invoke the target command directly (argv) instead \ of wrapping it in `bash -c` / `python -c` / `node`, and reach for a dedicated tool \ -(`read_file` / `edit_*` / `vcs_*` / `fetch` / …) when one already does the job. +(`read_file` / `edit_*` / `vcs_*` / `fetch` / `fetch_local` / …) when one already does the job. - Several `run_argv` calls in one step may run in parallel; use `run_argv_batch` \ for ordered pipelines (`pipe_out` / `mix_stderr` as needed). - Prefer `vcs_*` for status/diff/log/branch when enough. ### Network -- Prefer `fetch` (GET/POST/PUT/DELETE) for HTTP(S) docs/APIs over curl/wget via shell; \ -`offset` (0-based) skips leading body characters to page through a large body. -- Set `user_agent` (or a User-Agent header) when the remote expects a specific client; \ -otherwise a small default is used. Never rely on shell to spoof identity. -- Private/loopback/LAN hosts need an explicit human policy allow (not skipped by YOLO). -- Prefer provider hosted search (e.g. web_search) for open-ended research; use `fetch` for known URLs. +- `fetch` reads a public page as clean Markdown through the provider's Fetch API \ +(when available, e.g. DeepSeek); `fetch_local` is the raw HTTP client \ +(GET/POST/PUT/DELETE) from this host for localhost/LAN, APIs, custom headers, and \ +status/redirect inspection. +- `offset` (0-based) skips leading body characters to page through a large body; \ +truncated bodies resume with `get_truncated` and are not re-requested. +- Set `user_agent` (or a User-Agent header) on `fetch_local` when the remote expects a \ +specific client; otherwise a small default is used. Never rely on shell to spoof identity. +- Private/loopback/LAN hosts need an explicit human policy allow (not skipped by YOLO); \ +this applies to `fetch_local`, since `fetch` leaves from the provider's network. +- Prefer provider hosted search (e.g. web_search) for open-ended research; use `fetch` \ +or `fetch_local` for known URLs. - Respect denials, size limits, and HTTPS→HTTP redirect blocks; do not re-fetch the same URL repeatedly. ### Humans & safety @@ -232,7 +238,7 @@ class McpConfig(Struct, omit_defaults=True): class NetworkingConfig(Struct, omit_defaults=True): - """Host-side network policy for tools such as ``fetch``. + """Host-side network policy for tools such as ``fetch_local``. ``ssrf_assume_public_cidrs``: CIDR strings whose resolved addresses are treated as **public for SSRF policy** (skip private-host grants). Use for Clash/meta diff --git a/src/plyngent/lmproto/deepseek/search/__init__.py b/src/plyngent/lmproto/deepseek/webtools/__init__.py similarity index 57% rename from src/plyngent/lmproto/deepseek/search/__init__.py rename to src/plyngent/lmproto/deepseek/webtools/__init__.py index a4b4149..62c9714 100644 --- a/src/plyngent/lmproto/deepseek/search/__init__.py +++ b/src/plyngent/lmproto/deepseek/webtools/__init__.py @@ -1,8 +1,12 @@ -from .client import DeepseekSearchClient as DeepseekSearchClient +from .client import DeepseekWebToolsClient as DeepseekWebToolsClient from .model import DEFAULT_SEARCH_COUNT as DEFAULT_SEARCH_COUNT +from .model import FETCH_PATH as FETCH_PATH +from .model import MAX_FETCH_URL_CHARS as MAX_FETCH_URL_CHARS from .model import MAX_QUERY_CHARS as MAX_QUERY_CHARS from .model import MAX_SEARCH_COUNT as MAX_SEARCH_COUNT from .model import SEARCH_PATH as SEARCH_PATH +from .model import FetchRequest as FetchRequest +from .model import FetchResponse as FetchResponse from .model import SearchRequest as SearchRequest from .model import SearchResponse as SearchResponse from .model import SearchResult as SearchResult diff --git a/src/plyngent/lmproto/deepseek/search/client.py b/src/plyngent/lmproto/deepseek/webtools/client.py similarity index 65% rename from src/plyngent/lmproto/deepseek/search/client.py rename to src/plyngent/lmproto/deepseek/webtools/client.py index 63f7717..ef11e68 100644 --- a/src/plyngent/lmproto/deepseek/search/client.py +++ b/src/plyngent/lmproto/deepseek/webtools/client.py @@ -1,23 +1,27 @@ -"""DeepSeek's Search API: one request per query, no model round. +"""DeepSeek's Search and Fetch APIs: plain JSON, one bearer key. -The endpoint is a plain JSON API on the platform base URL, authenticated with -the same bearer key as the model APIs. It replaces the older server-side search -turn (``POST /messages`` with a hosted ``web_search`` tool), which spent a whole -completion round — and its input tokens — just to pick the query, and could only -return titles and URLs. A Search API response already carries a snippet and a -model-facing summary, so the local ``web_search`` tool can hand the model +``POST /v1/tools/search`` answers a query in one request, without spending a +model round — the endpoint replaces the older server-side search turn, which +cost a whole completion round (and its input tokens) just to pick the query and +could only return titles and URLs. A Search response already carries a snippet +and a model-facing summary, so the local ``web_search`` tool can hand the model something readable on the first call. +``POST /v1/tools/fetch`` reads one public page and returns its title and content +as Markdown, for the remote ``fetch`` tool. Both endpoints share the session, +authentication, error shape, retry policy and rate limits. + ``count`` does not affect billing (one successful request costs the same for 1 or 50 results); only successful requests are billable, so the retries below are safe. The account-level limit is 5 QPS and is shared across the Search and Fetch -APIs independently of model concurrency. +APIs independently of model concurrency; Fetch also has a per-domain limit. """ from __future__ import annotations import asyncio from typing import TYPE_CHECKING +from urllib.parse import urlsplit import msgspec import niquests @@ -31,9 +35,13 @@ from plyngent.lmproto.search import SearchHit from .model import ( DEFAULT_SEARCH_COUNT, + FETCH_PATH, + MAX_FETCH_URL_CHARS, MAX_QUERY_CHARS, MAX_SEARCH_COUNT, SEARCH_PATH, + FetchRequest, + FetchResponse, SearchRequest, SearchResponse, SearchResult, @@ -51,12 +59,13 @@ RETRY_DELAYS: tuple[float, ...] = (0.5, 1.0) _SNIPPET_FALLBACK_CHARS = 200 -class DeepseekSearchClient: - """``POST /v1/tools/search``: one query in, ranked hits out.""" +class DeepseekWebToolsClient: + """``POST /v1/tools/search`` and ``/v1/tools/fetch`` on one session.""" session: niquests.AsyncSession encoder: msgspec.json.Encoder decoder: msgspec.json.Decoder[SearchResponse] + fetch_decoder: msgspec.json.Decoder[FetchResponse] def __init__(self, config: OpenAIConfig) -> None: self.session = niquests.AsyncSession( @@ -66,6 +75,7 @@ class DeepseekSearchClient: ) self.encoder = msgspec.json.Encoder() self.decoder = msgspec.json.Decoder(SearchResponse) + self.fetch_decoder = msgspec.json.Decoder(FetchResponse) async def search( self, @@ -97,6 +107,19 @@ class DeepseekSearchClient: parsed = self.decoder.decode(body) return [hit for hit in map(_to_hit, parsed.results) if hit.url] + async def fetch(self, url: str) -> FetchResponse: + """Read *url* and return its title and Markdown content. + + The page is fetched from DeepSeek's network, not from this host: local + SSRF checks do not apply, and neither do private-host grants — a LAN or + loopback target is simply unreachable there (the endpoint answers 403 or + 404). The URL is still validated here so a malformed request fails + before it costs a round trip. + """ + _validate_fetch_url(url) + body = await self._post(FETCH_PATH, self.encoder.encode(FetchRequest(url=url))) + return self.fetch_decoder.decode(body) + async def _post(self, path: str, data: bytes) -> bytes: """POST *data*, retrying the statuses that mean "wait and try again".""" for delay in RETRY_DELAYS: @@ -150,4 +173,22 @@ def _shorten(text: str) -> str: return text[: _SNIPPET_FALLBACK_CHARS - 1].rstrip() + "…" -__all__ = ["RETRY_DELAYS", "DeepseekSearchClient"] +def _validate_fetch_url(url: str) -> None: + """Reject a URL the endpoint would refuse, before paying for the request.""" + if not url or len(url) > MAX_FETCH_URL_CHARS: + msg = f"fetch url must be 1-{MAX_FETCH_URL_CHARS} characters" + raise ValueError(msg) + try: + parts = urlsplit(url) + except ValueError: + msg = "fetch url is malformed" + raise ValueError(msg) from None + if parts.scheme not in {"http", "https"} or not parts.hostname: + msg = "fetch url must be an absolute http(s) URL with a host" + raise ValueError(msg) + if parts.username or parts.password: + msg = "fetch url must not carry a username or password" + raise ValueError(msg) + + +__all__ = ["RETRY_DELAYS", "DeepseekWebToolsClient"] diff --git a/src/plyngent/lmproto/deepseek/search/model.py b/src/plyngent/lmproto/deepseek/webtools/model.py similarity index 70% rename from src/plyngent/lmproto/deepseek/search/model.py rename to src/plyngent/lmproto/deepseek/webtools/model.py index d14fe71..b8e4dec 100644 --- a/src/plyngent/lmproto/deepseek/search/model.py +++ b/src/plyngent/lmproto/deepseek/webtools/model.py @@ -1,13 +1,15 @@ -"""Wire models for DeepSeek's Search API (``POST /v1/tools/search``).""" +"""Wire models for DeepSeek's Search and Fetch APIs.""" from __future__ import annotations import msgspec SEARCH_PATH = "/v1/tools/search" +FETCH_PATH = "/v1/tools/fetch" DEFAULT_SEARCH_COUNT = 10 MAX_SEARCH_COUNT = 50 MAX_QUERY_CHARS = 128 +MAX_FETCH_URL_CHARS = 8192 class SearchRequest(msgspec.Struct, omit_defaults=True): @@ -38,11 +40,28 @@ class SearchResponse(msgspec.Struct): results: list[SearchResult] +class FetchRequest(msgspec.Struct): + """Body of one fetch request: a single absolute http(s) URL.""" + + url: str + + +class FetchResponse(msgspec.Struct): + """A fetched page: its title and its content rendered as Markdown.""" + + title: str = "" + content: str = "" + + __all__ = [ "DEFAULT_SEARCH_COUNT", + "FETCH_PATH", + "MAX_FETCH_URL_CHARS", "MAX_QUERY_CHARS", "MAX_SEARCH_COUNT", "SEARCH_PATH", + "FetchRequest", + "FetchResponse", "SearchRequest", "SearchResponse", "SearchResult", diff --git a/src/plyngent/tools/__init__.py b/src/plyngent/tools/__init__.py index 4ec3210..45b8153 100644 --- a/src/plyngent/tools/__init__.py +++ b/src/plyngent/tools/__init__.py @@ -46,7 +46,7 @@ from .file import write_file as write_file from .net import NET_TOOLS as NET_TOOLS from .net import clear_private_grants as clear_private_grants from .net import clear_ssrf_assume_public_cidrs as clear_ssrf_assume_public_cidrs -from .net import fetch as fetch +from .net import fetch_local as fetch_local from .net import grant_private_host as grant_private_host from .net import set_fetch_policy_confirm_hook as set_fetch_policy_confirm_hook from .net import set_ssrf_assume_public_cidrs as set_ssrf_assume_public_cidrs diff --git a/src/plyngent/tools/context.py b/src/plyngent/tools/context.py index 05e67d1..489c48c 100644 --- a/src/plyngent/tools/context.py +++ b/src/plyngent/tools/context.py @@ -155,7 +155,7 @@ _read_only: ContextVar[bool] = ContextVar("plyngent_read_only_context", default= def read_only_context() -> Generator[None]: """Mark the current context as read-only for tool execution. - Tools that may mutate depending on arguments (e.g. ``fetch`` with a + Tools that may mutate depending on arguments (e.g. ``fetch_local`` with a non-GET method) check :func:`is_read_only_context` and refuse to mutate while it is set. """ diff --git a/src/plyngent/tools/danger.py b/src/plyngent/tools/danger.py index 1f23117..e70947a 100644 --- a/src/plyngent/tools/danger.py +++ b/src/plyngent/tools/danger.py @@ -192,8 +192,8 @@ def _open_pty_reason(args: Mapping[str, object]) -> str | None: def _fetch_reason(args: Mapping[str, object]) -> str | None: - """Soft-confirm cleartext HTTP and mutating fetch methods (not private-host policy).""" - from plyngent.tools.net.fetch import fetch_soft_reason + """Soft-confirm cleartext HTTP and mutating fetch_local methods (not private-host policy).""" + from plyngent.tools.net.fetch_local import fetch_soft_reason method_obj = args.get("method", "GET") url_obj = args.get("url", "") @@ -211,7 +211,7 @@ def classify_danger(name: str, args: Mapping[str, object]) -> str | None: # noq soft confirms for mutating tools and risky command launches — interpreters (however invoked) and detached runners such as ``nohup``, resolved past environment wrappers and `` run``. - Private/loopback fetch targets use a separate policy grant (not YOLO). + Private/loopback fetch_local targets use a separate policy grant (not YOLO). """ if name == "delete_path": return _delete_path_reason(args) @@ -227,6 +227,6 @@ def classify_danger(name: str, args: Mapping[str, object]) -> str | None: # noq return _run_argv_batch_reason(args) if name == "open_pty": return _open_pty_reason(args) - if name == "fetch": + if name == "fetch_local": return _fetch_reason(args) return None diff --git a/src/plyngent/tools/file/get_truncated.py b/src/plyngent/tools/file/get_truncated.py index 9eaaaba..8a0b52c 100644 --- a/src/plyngent/tools/file/get_truncated.py +++ b/src/plyngent/tools/file/get_truncated.py @@ -60,14 +60,15 @@ async def get_truncated(token: str, *, max_chars: int = DEFAULT_TOOL_RESULT_MAX_ """Fetch the next chunk of a truncated result using its ``truncate_token``. Pass the token shown at the end of a truncated ``read_file``, ``fetch``, - ``run_argv``, or earlier ``get_truncated`` result to keep reading without - re-requesting the whole source or raising limits. Truncated output carries a - fresh token, so chunks chain indefinitely. File chunks resume in the same - view as the original read: numbered reads continue as numbered lines - (editable via ``edit_lineno``), plain reads as raw text with a 1-based - ``L{begin}-{end}`` range. Memory chunks (generic tool output, and ``fetch`` - bodies) are served from a short-lived in-memory store that is forgotten when - the agent exits, so resuming a fetch never repeats the request. + ``fetch_local``, ``run_argv``, or earlier ``get_truncated`` result to keep + reading without re-requesting the whole source or raising limits. Truncated + output carries a fresh token, so chunks chain indefinitely. File chunks + resume in the same view as the original read: numbered reads continue as + numbered lines (editable via ``edit_lineno``), plain reads as raw text with + a 1-based ``L{begin}-{end}`` range. Memory chunks (generic tool output, and + ``fetch``/``fetch_local`` bodies) are served from a short-lived in-memory + store that is forgotten when the agent exits, so resuming a fetch never + repeats the request. """ parsed = decode_truncate_token(token) if parsed is None: diff --git a/src/plyngent/tools/net/__init__.py b/src/plyngent/tools/net/__init__.py index 6fbf082..7ecfd4f 100644 --- a/src/plyngent/tools/net/__init__.py +++ b/src/plyngent/tools/net/__init__.py @@ -1,4 +1,4 @@ -from .fetch import fetch as fetch +from .fetch_local import fetch_local as fetch_local from .grants import clear_private_grants as clear_private_grants from .grants import grant_private_host as grant_private_host from .grants import set_fetch_policy_confirm_hook as set_fetch_policy_confirm_hook @@ -6,5 +6,5 @@ from .policy import clear_ssrf_assume_public_cidrs as clear_ssrf_assume_public_c from .policy import set_ssrf_assume_public_cidrs as set_ssrf_assume_public_cidrs NET_TOOLS = [ - fetch, + fetch_local, ] diff --git a/src/plyngent/tools/net/bing.py b/src/plyngent/tools/net/bing.py index f250f7d..9ad19b3 100644 --- a/src/plyngent/tools/net/bing.py +++ b/src/plyngent/tools/net/bing.py @@ -3,7 +3,7 @@ Bing answers a query with a plain RSS document (``?format=rss``): no key, no page scraping, and each ```` already carries a title, a link and a description (the snippet DeepSeek's own index does not hand out). The request -goes through the fetch client the ``fetch`` tool uses, so the SSRF and +goes through the fetch client the ``fetch_local`` tool uses, so the SSRF and private-host grant policy applies to it as well. """ diff --git a/src/plyngent/tools/net/fetch.py b/src/plyngent/tools/net/fetch_local.py similarity index 95% rename from src/plyngent/tools/net/fetch.py rename to src/plyngent/tools/net/fetch_local.py index d00ec2b..f234683 100644 --- a/src/plyngent/tools/net/fetch.py +++ b/src/plyngent/tools/net/fetch_local.py @@ -1,4 +1,4 @@ -"""Model-facing ``fetch`` tool (HTTP GET/POST/PUT/DELETE).""" +"""Model-facing ``fetch_local`` tool (HTTP GET/POST/PUT/DELETE from this host).""" from __future__ import annotations @@ -62,7 +62,7 @@ def format_fetch_result( @tool(tags=ToolTag.LOCAL | ToolTag.INSTANCE_STATE | ToolTag.YOLO | ToolTag.TRUSTABLE | ToolTag.READ_ONLY) -async def fetch( +async def fetch_local( url: str, *, method: str = "GET", @@ -97,7 +97,7 @@ async def fetch( try: verb = normalize_method(method) if is_read_only_context() and verb != "GET": - return "error: fetch method must be GET in a read-only context (POST/PUT/DELETE mutate)" + return "error: fetch_local method must be GET in a read-only context (POST/PUT/DELETE mutate)" parsed = parse_fetch_url(url) hdrs = normalize_request_headers(headers, user_agent=user_agent) body_bytes: bytes | None = None @@ -119,7 +119,7 @@ async def fetch( except FetchPolicyError as exc: return f"error: {exc}" except Exception as exc: # noqa: BLE001 — tool surface returns error text - return f"error: fetch failed: {exc}" + return f"error: fetch_local failed: {exc}" return format_fetch_result( status=result.status, diff --git a/src/plyngent/tools/net/fetch_remote.py b/src/plyngent/tools/net/fetch_remote.py new file mode 100644 index 0000000..0a770fd --- /dev/null +++ b/src/plyngent/tools/net/fetch_remote.py @@ -0,0 +1,88 @@ +"""Model-facing ``fetch``: read a public page through DeepSeek's Fetch API. + +The page is rendered to Markdown server-side, so it is the friendly counterpart +of ``fetch_local``: use it for public articles/docs, and keep ``fetch_local`` +for localhost/LAN targets, non-GET requests, custom headers, and raw status or +redirect inspection. The request leaves from DeepSeek's network, not this host, +so local SSRF checks and private-host grants do not apply here. +""" + +from __future__ import annotations + +import inspect +from typing import TYPE_CHECKING + +from plyngent.agent.tools import ToolDefinition, ToolTag, schema_from_callable +from plyngent.tools.net.policy import DEFAULT_MAX_CHARS +from plyngent.tools.truncate_token import truncate_to_memory + +if TYPE_CHECKING: + from plyngent.lmproto.deepseek.webtools import DeepseekWebToolsClient + +FETCH_TOOL_NAME = "fetch" + + +def format_page_result( + *, + title: str, + url: str, + content: str, + max_chars: int, + offset: int = 0, +) -> str: + """Render a fetched page the way the other structured tool results read.""" + start = max(0, offset) + body, token = truncate_to_memory(content[start:], max_chars) + parts = [ + f"title={' '.join(title.split()) or '(untitled)'}", + f"url={url}", + f"chars={len(content)}", + f"truncated={'true' if token is not None else 'false'}", + "--- content ---", + body, + ] + return "\n".join(parts) + + +def build_fetch_tool( + client: DeepseekWebToolsClient, + *, + default_max_chars: int = DEFAULT_MAX_CHARS, +) -> ToolDefinition: + """A ``fetch`` definition backed by DeepSeek's Fetch API.""" + + async def fetch(url: str, offset: int = 0, max_chars: int = default_max_chars) -> str: + """Read a public web page as Markdown through DeepSeek's Fetch API. + + Use for public articles and docs when clean Markdown is enough. The page + is fetched from DeepSeek's network, so localhost/LAN targets are + unreachable here — use ``fetch_local`` for those, for non-GET methods, or + when you need the raw HTTP status/headers. A successful fetch is billed + per request (failures are not). + + ``offset`` (0-based) skips leading characters of the content. A body cut + by ``max_chars`` keeps its remainder in memory, resumable via its + ``truncate_token`` without another request (and without another charge). + """ + try: + page = await client.fetch(url) + except Exception as exc: # noqa: BLE001 — tool surface returns error text + return f"error: fetch failed: {exc}" + return format_page_result( + title=page.title, + url=url, + content=page.content, + max_chars=max_chars, + offset=offset, + ) + + return ToolDefinition( + name=FETCH_TOOL_NAME, + description=inspect.getdoc(fetch) or "", + parameters=schema_from_callable(fetch), + handler=fetch, + tags=ToolTag.LOCAL | ToolTag.READ_ONLY, + ) + + +__all__ = ["FETCH_TOOL_NAME", "build_fetch_tool", "format_page_result"] diff --git a/src/plyngent/tools/net/grants.py b/src/plyngent/tools/net/grants.py index 57f10e2..0961ed8 100644 --- a/src/plyngent/tools/net/grants.py +++ b/src/plyngent/tools/net/grants.py @@ -77,7 +77,7 @@ async def ensure_host_allowed( try: inst = instance if instance is not None else require_bound_instance() except WorkspaceError as exc: - msg = f"instance state is not bound for fetch policy: {exc}" + msg = f"instance state is not bound for fetch_local policy: {exc}" raise FetchPolicyError(msg) from exc parsed = parse_fetch_url(url) @@ -85,7 +85,7 @@ async def ensure_host_allowed( if assessment.classification is HostClass.PUBLIC: return if assessment.classification is HostClass.FORBIDDEN: - msg = f"fetch blocked forbidden host {parsed.host!r} (addresses: {', '.join(assessment.addresses)})" + msg = f"fetch_local blocked forbidden host {parsed.host!r} (addresses: {', '.join(assessment.addresses)})" raise FetchPolicyError(msg) # PRIVATE @@ -100,18 +100,18 @@ async def ensure_host_allowed( hook = get_fetch_policy_confirm_hook(inst) if hook is None: msg = ( - f"fetch blocked private/loopback host {parsed.host}:{parsed.port} " + f"fetch_local blocked private/loopback host {parsed.host}:{parsed.port} " f"({assessment.reason}; no policy confirm hook / non-interactive deny)" ) raise FetchPolicyError(msg) try: allowed = bool(hook(parsed.host, parsed.port, parsed.url, timeout)) except Exception as exc: - msg = f"fetch blocked private/loopback host {parsed.host}:{parsed.port} (confirm failed: {exc})" + msg = f"fetch_local blocked private/loopback host {parsed.host}:{parsed.port} (confirm failed: {exc})" raise FetchPolicyError(msg) from exc if not allowed: msg = ( - f"fetch blocked private/loopback host {parsed.host}:{parsed.port} " + f"fetch_local blocked private/loopback host {parsed.host}:{parsed.port} " f"(user declined or timed out after {timeout:g}s; not skipped by YOLO)" ) raise FetchPolicyError(msg) diff --git a/src/plyngent/tools/net/policy.py b/src/plyngent/tools/net/policy.py index 0c9e4fb..10c8977 100644 --- a/src/plyngent/tools/net/policy.py +++ b/src/plyngent/tools/net/policy.py @@ -1,4 +1,4 @@ -"""URL / host policy for the fetch tool (SSRF, private grants, cleartext).""" +"""URL / host policy for the fetch_local tool (SSRF, private grants, cleartext).""" from __future__ import annotations diff --git a/src/plyngent/tools/net/search.py b/src/plyngent/tools/net/search.py index 6cc6052..7a5747a 100644 --- a/src/plyngent/tools/net/search.py +++ b/src/plyngent/tools/net/search.py @@ -9,7 +9,8 @@ the handler asks the configured search sources behind the scenes — one fails or finds nothing. DeepSeek's own source is the Search API (``POST /v1/tools/search``), which answers in one request instead of a model round. A hit is a title, a URL and (when the source has one) a snippet: read a -page with ``fetch`` before relying on its content. +page with ``fetch`` (or ``fetch_local`` for local/LAN targets) before relying on +its content. """ from __future__ import annotations @@ -22,7 +23,7 @@ from plyngent.agent.tools import ToolDefinition, ToolTag, schema_from_callable if TYPE_CHECKING: from collections.abc import Sequence - from plyngent.lmproto.deepseek.search import DeepseekSearchClient + from plyngent.lmproto.deepseek.webtools import DeepseekWebToolsClient from plyngent.lmproto.search import SearchHit WEB_SEARCH_TOOL_NAME = "web_search" @@ -38,9 +39,9 @@ class SearchBackend(Protocol): class DeepseekSearchBackend: """DeepSeek's own index: one Search API request per query.""" - _client: DeepseekSearchClient + _client: DeepseekWebToolsClient - def __init__(self, client: DeepseekSearchClient) -> None: + def __init__(self, client: DeepseekWebToolsClient) -> None: self._client = client async def search(self, query: str) -> Sequence[SearchHit]: @@ -82,7 +83,8 @@ def build_web_search_tool( """Search the web and return the pages that matched. One entry per hit (``title — url``, plus the snippet when the source has - one): read a page with ``fetch`` before relying on its content. + one): read a page with ``fetch`` (or ``fetch_local`` for local targets) + before relying on its content. """ found, failures = await search_in_order(backends, query) if not found: diff --git a/src/plyngent/tools/truncate_token.py b/src/plyngent/tools/truncate_token.py index 30fbea9..057a9cb 100644 --- a/src/plyngent/tools/truncate_token.py +++ b/src/plyngent/tools/truncate_token.py @@ -2,8 +2,8 @@ Every truncation site (the agent loop's generic tool-result cap, ``run_argv`` / ``run_argv_batch`` per-stream caps, ``read_file``, -``fetch``, and request-time compact shrinks) appends the same compact marker -when output is cut short:: +``fetch`` / ``fetch_local``, and request-time compact shrinks) appends the same +compact marker when output is cut short:: [Truncated (32000 chars max.; 500 omitted). truncate_token=...] @@ -14,9 +14,9 @@ source is exhausted (then no marker is emitted). A ``file`` remainder is re-read from disk by ``location`` + ``offset``, so it survives for as long as the file does. Everything else — arbitrary tool output -(run_argv stdout, todo renders, …) and ``fetch`` bodies — has no source worth -re-running (a re-fetch costs a round trip and re-hits the remote), so its -remainder lives in a short-lived in-memory store (``kind="memory"``) that is +(run_argv stdout, todo renders, …) and ``fetch`` / ``fetch_local`` bodies — have +no source worth re-running (a re-fetch costs a round trip and re-hits the remote), +so its remainder lives in a short-lived in-memory store (``kind="memory"``) that is forgotten when the agent process exits; the model must re-run the tool after that. """ diff --git a/tests/test_cli/test_display.py b/tests/test_cli/test_display.py index 46d8084..bcfd640 100644 --- a/tests/test_cli/test_display.py +++ b/tests/test_cli/test_display.py @@ -544,7 +544,7 @@ async def test_pretty_fetch(capsys: pytest.CaptureFixture[str]) -> None: "status=200\nmethod=GET\nfinal_url=https://example.com\ncontent_type=application/json\n" "body_kind=json\nbytes=1229\ntruncated=false\nredirects=0\nsecurity=public\n--- body ---\n{...}\n" ) - await render_events(_aiter([_pretty_call("fetch", '{"url": "https://example.com"}'), _result(result)])) + await render_events(_aiter([_pretty_call("fetch_local", '{"url": "https://example.com"}'), _result(result)])) out = capsys.readouterr().out assert "* Fetch GET https://example.com (200)" in out assert "[tool]" not in out @@ -555,7 +555,7 @@ async def test_pretty_fetch_error_status(capsys: pytest.CaptureFixture[str]) -> "status=404\nmethod=GET\nfinal_url=https://example.com/nope\ncontent_type=text/html\n" "body_kind=html\nbytes=45\ntruncated=false\nredirects=0\nsecurity=public\n--- body ---\nNot found\n" ) - await render_events(_aiter([_pretty_call("fetch", '{"url": "https://example.com/nope"}'), _result(result)])) + await render_events(_aiter([_pretty_call("fetch_local", '{"url": "https://example.com/nope"}'), _result(result)])) out = capsys.readouterr().out assert "* Fetch GET https://example.com/nope (404)" in out @@ -565,17 +565,37 @@ async def test_pretty_fetch_truncated(capsys: pytest.CaptureFixture[str]) -> Non "status=200\nmethod=GET\nfinal_url=https://example.com\ncontent_type=text/plain\n" "body_kind=text\nbytes=50000\ntruncated=true\nredirects=0\nsecurity=public\n--- body ---\n..." ) - await render_events(_aiter([_pretty_call("fetch", '{"url": "https://example.com"}'), _result(result)])) + await render_events(_aiter([_pretty_call("fetch_local", '{"url": "https://example.com"}'), _result(result)])) out = capsys.readouterr().out assert "* Fetch GET https://example.com (200)" in out async def test_pretty_fetch_error(capsys: pytest.CaptureFixture[str]) -> None: await render_events( - _aiter([_pretty_call("fetch", '{"url": "https://example.com"}'), _result("error: fetch failed: boom")]) + _aiter( + [_pretty_call("fetch_local", '{"url": "https://example.com"}'), _result("error: fetch_local failed: boom")] + ) ) out = capsys.readouterr().out - assert "* Fetch GET https://example.com (error: fetch failed: boom)" in out + assert "* Fetch GET https://example.com (error: fetch_local failed: boom)" in out + + +async def test_pretty_remote_fetch(capsys: pytest.CaptureFixture[str]) -> None: + result = "title=Example article\nurl=https://example.com\nchars=1234\ntruncated=false\n--- content ---\n..." + await render_events(_aiter([_pretty_call("fetch", '{"url": "https://example.com"}'), _result(result)])) + out = capsys.readouterr().out + assert "* Fetch https://example.com (1234 chars)" in out + + +async def test_pretty_fetch_reads_a_legacy_local_row(capsys: pytest.CaptureFixture[str]) -> None: + # Rows written before the local tool was renamed carry the local status block. + result = ( + "status=200\nmethod=GET\nfinal_url=https://example.com\ncontent_type=text/plain\n" + "body_kind=text\nbytes=12\ntruncated=false\nredirects=0\nsecurity=public\n--- body ---\nhi\n" + ) + await render_events(_aiter([_pretty_call("fetch", '{"url": "https://example.com"}'), _result(result)])) + out = capsys.readouterr().out + assert "* Fetch https://example.com (200)" in out async def test_pretty_web_search(capsys: pytest.CaptureFixture[str]) -> None: diff --git a/tests/test_lmproto/test_deepseek_search.py b/tests/test_lmproto/test_deepseek_webtools.py similarity index 74% rename from tests/test_lmproto/test_deepseek_search.py rename to tests/test_lmproto/test_deepseek_webtools.py index 2a038e3..5e42d04 100644 --- a/tests/test_lmproto/test_deepseek_search.py +++ b/tests/test_lmproto/test_deepseek_webtools.py @@ -1,12 +1,12 @@ -"""DeepSeek's Search API client (``POST /v1/tools/search``).""" +"""DeepSeek's Search and Fetch APIs (``/v1/tools/search``, ``/v1/tools/fetch``).""" from __future__ import annotations import msgspec import pytest -from plyngent.lmproto.deepseek.search import ( - DeepseekSearchClient, +from plyngent.lmproto.deepseek.webtools import ( + DeepseekWebToolsClient, SearchRequest, ) from plyngent.lmproto.openai_compatible.config import OpenAIConfig @@ -23,8 +23,8 @@ def _body(*results: dict[str, object]) -> bytes: return msgspec.json.encode({"results": list(results)}) -def _client() -> DeepseekSearchClient: - return DeepseekSearchClient( +def _client() -> DeepseekWebToolsClient: + return DeepseekWebToolsClient( OpenAIConfig(access_key_or_token="sk-test", base_url="https://api.deepseek.com"), ) @@ -166,7 +166,7 @@ async def test_the_endpoint_error_detail_is_raised(monkeypatch: pytest.MonkeyPat @pytest.mark.asyncio async def test_a_retryable_status_is_tried_again(monkeypatch: pytest.MonkeyPatch) -> None: client = _client() - monkeypatch.setattr("plyngent.lmproto.deepseek.search.client.RETRY_DELAYS", (0.0,)) + monkeypatch.setattr("plyngent.lmproto.deepseek.webtools.client.RETRY_DELAYS", (0.0,)) statuses = [429, 200] async def fake_post(path: str, **kwargs: object) -> _Resp: @@ -181,7 +181,7 @@ async def test_a_retryable_status_is_tried_again(monkeypatch: pytest.MonkeyPatch @pytest.mark.asyncio async def test_a_balance_error_is_not_retried(monkeypatch: pytest.MonkeyPatch) -> None: client = _client() - monkeypatch.setattr("plyngent.lmproto.deepseek.search.client.RETRY_DELAYS", (0.0,)) + monkeypatch.setattr("plyngent.lmproto.deepseek.webtools.client.RETRY_DELAYS", (0.0,)) calls: list[int] = [] async def fake_post(path: str, **kwargs: object) -> _Resp: @@ -197,3 +197,50 @@ async def test_a_balance_error_is_not_retried(monkeypatch: pytest.MonkeyPatch) - def test_the_request_model_omits_what_was_not_set() -> None: encoded = msgspec.json.encode(SearchRequest(query="q")) assert msgspec.json.decode(encoded, type=dict[str, object]) == {"query": "q"} + + +@pytest.mark.asyncio +async def test_fetch_posts_the_url_and_returns_the_page(monkeypatch: pytest.MonkeyPatch) -> None: + client = _client() + sent: dict[str, object] = {} + + async def fake_post(path: str, **kwargs: object) -> _Resp: + sent["path"] = path + sent.update(kwargs) + return _Resp(200, msgspec.json.encode({"title": "Example article", "content": "# Article\n\nBody."})) + + monkeypatch.setattr(client.session, "post", fake_post) + page = await client.fetch("https://example.com/article") + + assert page.title == "Example article" + assert page.content == "# Article\n\nBody." + assert sent["path"] == "/v1/tools/fetch" + assert _decode_request(sent["data"]) == {"url": "https://example.com/article"} + + +@pytest.mark.asyncio +async def test_fetch_rejects_a_url_the_endpoint_would_refuse(monkeypatch: pytest.MonkeyPatch) -> None: + client = _client() + calls: list[object] = [] + + async def fake_post(path: str, **kwargs: object) -> _Resp: + calls.append(path) + return _Resp(200, b"{}") + + monkeypatch.setattr(client.session, "post", fake_post) + for url in ("file:///etc/passwd", "https://user:secret@example.com/", "https:///no-host", ""): + with pytest.raises(ValueError): + _ = await client.fetch(url) + assert calls == [] + + +@pytest.mark.asyncio +async def test_fetch_surfaces_a_security_policy_rejection(monkeypatch: pytest.MonkeyPatch) -> None: + client = _client() + + async def fake_post(path: str, **kwargs: object) -> _Resp: + return _Resp(403, b'{"error": {"message": "target rejected by security policy"}}') + + monkeypatch.setattr(client.session, "post", fake_post) + with pytest.raises(RuntimeError, match="security policy"): + _ = await client.fetch("https://example.com/blocked") diff --git a/tests/test_tools/test_catalog.py b/tests/test_tools/test_catalog.py index 06579cd..4ea3617 100644 --- a/tests/test_tools/test_catalog.py +++ b/tests/test_tools/test_catalog.py @@ -74,7 +74,7 @@ def test_read_only_classification() -> None: "vcs_diff", "vcs_log", "vcs_branch", - "fetch", + "fetch_local", "skill_list", "skill_read", "skill_search", diff --git a/tests/test_tools/test_fetch.py b/tests/test_tools/test_fetch_local.py similarity index 84% rename from tests/test_tools/test_fetch.py rename to tests/test_tools/test_fetch_local.py index 632b204..6294918 100644 --- a/tests/test_tools/test_fetch.py +++ b/tests/test_tools/test_fetch_local.py @@ -1,4 +1,4 @@ -"""Tests for tools.net.fetch (policy, UA, methods, private grants).""" +"""Tests for tools.net.fetch_local (policy, UA, methods, private grants).""" from __future__ import annotations @@ -10,7 +10,7 @@ import pytest from plyngent.tools.context import read_only_context from plyngent.tools.danger import classify_danger -from plyngent.tools.net import fetch, grant_private_host +from plyngent.tools.net import fetch_local, grant_private_host from plyngent.tools.net.policy import ( DEFAULT_USER_AGENT, FetchPolicyError, @@ -30,7 +30,7 @@ if TYPE_CHECKING: class _Handler(BaseHTTPRequestHandler): - """Minimal echo server for fetch tests.""" + """Minimal echo server for fetch_local tests.""" @override def log_message(self, format: str, *args: object) -> None: @@ -171,10 +171,10 @@ def test_soft_confirm_reason_matrix() -> None: def test_classify_danger_fetch() -> None: - assert classify_danger("fetch", {"url": "https://example.com/", "method": "GET"}) is None - reason = classify_danger("fetch", {"url": "http://example.com/", "method": "GET"}) + assert classify_danger("fetch_local", {"url": "https://example.com/", "method": "GET"}) is None + reason = classify_danger("fetch_local", {"url": "http://example.com/", "method": "GET"}) assert reason is not None and "cleartext" in reason - reason2 = classify_danger("fetch", {"url": "https://example.com/api", "method": "DELETE"}) + reason2 = classify_danger("fetch_local", {"url": "https://example.com/api", "method": "DELETE"}) assert reason2 is not None and "DELETE" in reason2 @@ -184,36 +184,36 @@ async def test_fetch_get_post_put_delete(workspace: Path, http_server: str) -> N # Grant loopback for this process/instance (fixture binds InstanceState). grant_private_host("127.0.0.1", int(base.rsplit(":", 1)[1])) - out = await call_async(fetch, f"{base}/hello") + out = await call_async(fetch_local, f"{base}/hello") assert "status=200" in out assert "GET /hello" in out assert "security=cleartext-http" in out - out_post = await call_async(fetch, f"{base}/echo", method="POST", body="hi") + out_post = await call_async(fetch_local, f"{base}/echo", method="POST", body="hi") assert "status=201" in out_post assert "POST:hi" in out_post - out_put = await call_async(fetch, f"{base}/echo", method="PUT", body="x") + out_put = await call_async(fetch_local, f"{base}/echo", method="PUT", body="x") assert "PUT:x" in out_put - out_del = await call_async(fetch, f"{base}/x", method="DELETE") + out_del = await call_async(fetch_local, f"{base}/x", method="DELETE") assert "status=204" in out_del async def test_fetch_read_only_context_blocks_non_get(workspace: Path, http_server: str) -> None: - """In a read-only context fetch only allows GET; outside it POST works.""" + """In a read-only context fetch_local only allows GET; outside it POST works.""" del workspace port = int(http_server.rsplit(":", 1)[1]) grant_private_host("127.0.0.1", port) with read_only_context(): - out = await call_async(fetch, f"{http_server}/hello") + out = await call_async(fetch_local, f"{http_server}/hello") assert "status=200" in out # GET is fine in a read-only context - blocked = await call_async(fetch, f"{http_server}/echo", method="POST", body="hi") + blocked = await call_async(fetch_local, f"{http_server}/echo", method="POST", body="hi") assert blocked.startswith("error:") assert "GET" in blocked - out_post = await call_async(fetch, f"{http_server}/echo", method="POST", body="hi") + out_post = await call_async(fetch_local, f"{http_server}/echo", method="POST", body="hi") assert "POST:hi" in out_post @@ -222,18 +222,18 @@ async def test_fetch_user_agent_passthrough(workspace: Path, http_server: str) - port = int(http_server.rsplit(":", 1)[1]) grant_private_host("127.0.0.1", port) - out = await call_async(fetch, f"{http_server}/ua", user_agent="AgentUA/9") + out = await call_async(fetch_local, f"{http_server}/ua", user_agent="AgentUA/9") assert "AgentUA/9" in out out2 = await call_async( - fetch, + fetch_local, f"{http_server}/ua", headers={"User-Agent": "HeaderUA/1"}, ) assert "HeaderUA/1" in out2 out3 = await call_async( - fetch, + fetch_local, f"{http_server}/ua", headers={"User-Agent": "HeaderUA/1"}, user_agent="ArgUA/2", @@ -247,11 +247,11 @@ async def test_fetch_truncation_and_binary(workspace: Path, http_server: str) -> port = int(http_server.rsplit(":", 1)[1]) grant_private_host("127.0.0.1", port) - out = await call_async(fetch, f"{http_server}/big", max_bytes=100) + out = await call_async(fetch_local, f"{http_server}/big", max_bytes=100) assert "truncated=true" in out assert "bytes=100" in out - out_bin = await call_async(fetch, f"{http_server}/bin") + out_bin = await call_async(fetch_local, f"{http_server}/bin") assert "body_kind=binary" in out_bin assert "binary body omitted" in out_bin @@ -261,7 +261,7 @@ async def test_fetch_char_truncation_embeds_token(workspace: Path, http_server: port = int(http_server.rsplit(":", 1)[1]) grant_private_host("127.0.0.1", port) - out = await call_async(fetch, f"{http_server}/big", max_chars=1000) + out = await call_async(fetch_local, f"{http_server}/big", max_chars=1000) assert "truncated=true" in out body = out.split("--- body ---", 1)[-1] assert "truncate_token=" in body @@ -281,7 +281,7 @@ async def test_fetch_offset_skips_body(workspace: Path, http_server: str) -> Non grant_private_host("127.0.0.1", port) # /big body is 5000 chars; offset 4900 leaves 100 chars, no token needed. - out = await call_async(fetch, f"{http_server}/big", max_chars=200, offset=4900) + out = await call_async(fetch_local, f"{http_server}/big", max_chars=200, offset=4900) body = out.split("--- body ---", 1)[-1] assert body.strip() == "x" * 100 assert "truncate_token=" not in out @@ -290,21 +290,21 @@ async def test_fetch_offset_skips_body(workspace: Path, http_server: str) -> Non async def test_fetch_private_denied_without_grant(workspace: Path, http_server: str) -> None: del workspace # No grant, no policy hook → hard deny (even though server is up). - out = await call_async(fetch, f"{http_server}/hello") + out = await call_async(fetch_local, f"{http_server}/hello") assert out.startswith("error:") assert "private" in out or "loopback" in out async def test_fetch_forbidden_metadata(workspace: Path) -> None: del workspace - out = await call_async(fetch, "http://169.254.169.254/latest/meta-data/") + out = await call_async(fetch_local, "http://169.254.169.254/latest/meta-data/") assert out.startswith("error:") assert "forbidden" in out async def test_fetch_bad_scheme(workspace: Path) -> None: del workspace - out = await call_async(fetch, "file:///etc/passwd") + out = await call_async(fetch_local, "file:///etc/passwd") assert out.startswith("error:") assert "scheme" in out @@ -313,6 +313,6 @@ async def test_fetch_redirect_loop_capped(workspace: Path, http_server: str) -> del workspace port = int(http_server.rsplit(":", 1)[1]) grant_private_host("127.0.0.1", port) - out = await call_async(fetch, f"{http_server}/redirect-loop", max_redirects=3) + out = await call_async(fetch_local, f"{http_server}/redirect-loop", max_redirects=3) assert out.startswith("error:") assert "redirect" in out.lower() diff --git a/tests/test_tools/test_fetch_remote.py b/tests/test_tools/test_fetch_remote.py new file mode 100644 index 0000000..882420d --- /dev/null +++ b/tests/test_tools/test_fetch_remote.py @@ -0,0 +1,73 @@ +"""The remote ``fetch`` tool (DeepSeek's Fetch API, Markdown pages).""" + +from __future__ import annotations + +import pytest + +from plyngent.agent.tools import ToolTag +from plyngent.lmproto.deepseek.webtools import FetchResponse +from plyngent.tools.net.fetch_remote import FETCH_TOOL_NAME, build_fetch_tool + + +class _Client: + """Stands in for DeepseekWebToolsClient.fetch.""" + + def __init__(self, *, page: FetchResponse | None = None, error: str | None = None) -> None: + self.page = page or FetchResponse(title="T", content="body") + self.error = error + self.urls: list[str] = [] + + async def fetch(self, url: str) -> FetchResponse: + self.urls.append(url) + if self.error is not None: + raise RuntimeError(self.error) + return self.page + + +def _tool(client: _Client, *, max_chars: int = 32_000): + return build_fetch_tool(client, default_max_chars=max_chars) + + +def test_definition_shape() -> None: + definition = _tool(_Client()) + assert definition.name == FETCH_TOOL_NAME == "fetch" + assert definition.tags == ToolTag.LOCAL | ToolTag.READ_ONLY + properties = definition.parameters["properties"] + assert set(properties) == {"url", "offset", "max_chars"} + assert definition.parameters["required"] == ["url"] + assert "fetch_local" in definition.description + + +@pytest.mark.asyncio +async def test_handler_renders_title_url_and_content() -> None: + client = _Client(page=FetchResponse(title=" Example article ", content="# Heading\n\nBody.")) + result = await _tool(client).handler(url="https://example.com/article") + + assert client.urls == ["https://example.com/article"] + assert result.splitlines() == [ + "title=Example article", + "url=https://example.com/article", + "chars=16", + "truncated=false", + "--- content ---", + "# Heading", + "", + "Body.", + ] + + +@pytest.mark.asyncio +async def test_offset_and_truncation_keep_the_remainder_resumable() -> None: + client = _Client(page=FetchResponse(title="", content="0123456789")) + result = await _tool(client, max_chars=3).handler(url="https://example.com/", offset=5) + + assert "title=(untitled)" in result + assert "--- content ---\n567" in result + assert "truncated=true" in result + assert "truncate_token=" in result + + +@pytest.mark.asyncio +async def test_handler_reports_a_client_error() -> None: + result = await _tool(_Client(error="HTTP 404: no usable content")).handler(url="https://example.com/x") + assert result == "error: fetch failed: HTTP 404: no usable content" diff --git a/tests/test_tools/test_get_truncated.py b/tests/test_tools/test_get_truncated.py index 6823319..02a7311 100644 --- a/tests/test_tools/test_get_truncated.py +++ b/tests/test_tools/test_get_truncated.py @@ -86,9 +86,9 @@ async def test_get_truncated_file_missing(workspace: Path) -> None: async def test_get_truncated_serves_a_fetch_body_without_refetching(monkeypatch) -> None: """A fetch body's token reads the stored remainder; the request is not repeated.""" - from plyngent.tools.net.fetch import format_fetch_result + from plyngent.tools.net.fetch_local import format_fetch_result - fetch_module = importlib.import_module("plyngent.tools.net.fetch") + fetch_module = importlib.import_module("plyngent.tools.net.fetch_local") async def _must_not_refetch(*_args: object, **_kwargs: object) -> str: msg = "resuming a fetch body must not re-run the request"