mirror of
https://github.com/NCBM/plyngent.git
synced 2026-11-07 23:47:54 +08:00
core/tools+cli: split fetch into remote fetch and fetch_local
This commit is contained in:
@@ -26,8 +26,8 @@ use `run_argv_batch` for ordered pipelines.
|
|||||||
- `regex_files`: `pattern` is a Python `re` regex (not a glob, so `*.py` fails) and \
|
- `regex_files`: `pattern` is a Python `re` regex (not a glob, so `*.py` fails) and \
|
||||||
`path` is a literal file/dir (no glob expansion).
|
`path` is a literal file/dir (no glob expansion).
|
||||||
- Truncated results carry a `truncate_token=...`; continue with `get_truncated`.
|
- Truncated results carry a `truncate_token=...`; continue with `get_truncated`.
|
||||||
- Prefer `fetch` for HTTP(S); `offset` (0-based) skips leading body characters; \
|
- Prefer `fetch` for public pages (clean Markdown); `fetch_local` for localhost/LAN, \
|
||||||
private/LAN hosts need human policy allow (not YOLO).
|
non-GET methods, headers, or raw status; `offset` (0-based) skips leading body characters.
|
||||||
- PTY secrets only via `ask_into_pty`; denylists and confirms still apply.
|
- PTY secrets only via `ask_into_pty`; denylists and confirms still apply.
|
||||||
- `wait` pauses; pressing Enter disturbs early (optional reason).
|
- `wait` pauses; pressing Enter disturbs early (optional reason).
|
||||||
- Todo stack: open items mean unfinished work; the stack does not auto-clear — \
|
- Todo stack: open items mean unfinished work; the stack does not auto-clear — \
|
||||||
|
|||||||
@@ -115,8 +115,8 @@ def truncated_turn_notice() -> Notice:
|
|||||||
_ASIDE_BODIES: dict[str, str] = {
|
_ASIDE_BODIES: dict[str, str] = {
|
||||||
"read": (
|
"read": (
|
||||||
"Side question: this exchange is not saved to the session transcript and does not "
|
"Side question: this exchange is not saved to the session transcript and does not "
|
||||||
"change the main conversation. Tools here are read-only (file/VCS reads, ask, fetch "
|
"change the main conversation. Tools here are read-only (file/VCS reads, ask, fetch/"
|
||||||
"GET): you cannot edit files or run state-changing commands, so describe a change "
|
"fetch_local GET): you cannot edit files or run state-changing commands, so describe a change "
|
||||||
"instead of applying it."
|
"instead of applying it."
|
||||||
),
|
),
|
||||||
"no": (
|
"no": (
|
||||||
|
|||||||
@@ -341,7 +341,7 @@ async def _run_chat( # noqa: C901, PLR0915 — chat orchestration
|
|||||||
state.instance_state.workspace.path_denylist = tuple(store.agent_config.path_denylist or ())
|
state.instance_state.workspace.path_denylist = tuple(store.agent_config.path_denylist or ())
|
||||||
from plyngent.tools.net import set_ssrf_assume_public_cidrs
|
from plyngent.tools.net import set_ssrf_assume_public_cidrs
|
||||||
|
|
||||||
# Clash Fake-IP etc.: treat configured CIDRs as public for fetch SSRF.
|
# Clash Fake-IP etc.: treat configured CIDRs as public for fetch_local SSRF.
|
||||||
_ = set_ssrf_assume_public_cidrs(store.networking_config.ssrf_assume_public_cidrs)
|
_ = set_ssrf_assume_public_cidrs(store.networking_config.ssrf_assume_public_cidrs)
|
||||||
if interactive:
|
if interactive:
|
||||||
from plyngent.cli.limits import prompt_policy_command_confirm, prompt_policy_fetch_confirm
|
from plyngent.cli.limits import prompt_policy_command_confirm, prompt_policy_fetch_confirm
|
||||||
|
|||||||
@@ -452,13 +452,34 @@ def _http_status_fg(status: str) -> str | None:
|
|||||||
|
|
||||||
|
|
||||||
def _fetch_prefix(args_json: str) -> str:
|
def _fetch_prefix(args_json: str) -> str:
|
||||||
|
"""Remote ``fetch``: the URL is the whole call."""
|
||||||
|
url = _json_str_arg(args_json, "url") or "?"
|
||||||
|
return _pretty_prefix(f"* Fetch {url} ")
|
||||||
|
|
||||||
|
|
||||||
|
def _fetch_detail(args_json: str, result: str) -> str:
|
||||||
|
"""Remote ``fetch`` outcome: fetched size, or an error.
|
||||||
|
|
||||||
|
A row written before the local tool was renamed still carries the local
|
||||||
|
``status=...`` block; render it with the local formatter.
|
||||||
|
"""
|
||||||
|
if result.startswith("error:"):
|
||||||
|
return _pretty_segments((f"({result})", "red"))
|
||||||
|
fields = _status_fields(result)
|
||||||
|
if "status" in fields:
|
||||||
|
return _fetch_local_detail(args_json, result)
|
||||||
|
chars = fields.get("chars", "")
|
||||||
|
return _pretty_segments((f"({chars} chars)" if chars else "(done)", "green"))
|
||||||
|
|
||||||
|
|
||||||
|
def _fetch_local_prefix(args_json: str) -> str:
|
||||||
method = _json_str_arg(args_json, "method") or "GET"
|
method = _json_str_arg(args_json, "method") or "GET"
|
||||||
url = _json_str_arg(args_json, "url") or "?"
|
url = _json_str_arg(args_json, "url") or "?"
|
||||||
return _pretty_prefix(f"* Fetch {method} {url} ")
|
return _pretty_prefix(f"* Fetch {method} {url} ")
|
||||||
|
|
||||||
|
|
||||||
def _fetch_detail(_args_json: str, result: str) -> str:
|
def _fetch_local_detail(_args_json: str, result: str) -> str:
|
||||||
"""``fetch`` outcome: HTTP status, or error."""
|
"""``fetch_local`` outcome: HTTP status, or error."""
|
||||||
if result.startswith("error:"):
|
if result.startswith("error:"):
|
||||||
return _pretty_segments((f"({result})", "red"))
|
return _pretty_segments((f"({result})", "red"))
|
||||||
status = _status_fields(result).get("status", "")
|
status = _status_fields(result).get("status", "")
|
||||||
@@ -888,6 +909,7 @@ _PRETTY_LINES: dict[str, PrettyLine] = {
|
|||||||
"run_argv": PrettyLine(prefix=_run_argv_prefix, detail=_run_argv_detail),
|
"run_argv": PrettyLine(prefix=_run_argv_prefix, detail=_run_argv_detail),
|
||||||
"run_argv_batch": PrettyLine(prefix=_run_argv_batch_prefix, detail=_run_argv_batch_detail),
|
"run_argv_batch": PrettyLine(prefix=_run_argv_batch_prefix, detail=_run_argv_batch_detail),
|
||||||
"fetch": PrettyLine(prefix=_fetch_prefix, detail=_fetch_detail),
|
"fetch": PrettyLine(prefix=_fetch_prefix, detail=_fetch_detail),
|
||||||
|
"fetch_local": PrettyLine(prefix=_fetch_local_prefix, detail=_fetch_local_detail),
|
||||||
"web_search": PrettyLine(prefix=_web_search_prefix, detail=_web_search_detail),
|
"web_search": PrettyLine(prefix=_web_search_prefix, detail=_web_search_detail),
|
||||||
"edit_replace": _mutator_line("edit_replace"),
|
"edit_replace": _mutator_line("edit_replace"),
|
||||||
"edit_lineno": _mutator_line("edit_lineno"),
|
"edit_lineno": _mutator_line("edit_lineno"),
|
||||||
|
|||||||
@@ -311,14 +311,14 @@ def prompt_policy_fetch_confirm(
|
|||||||
url: str,
|
url: str,
|
||||||
timeout_seconds: float,
|
timeout_seconds: float,
|
||||||
) -> bool:
|
) -> bool:
|
||||||
"""Ask whether to allow fetch to a private/loopback host (timed).
|
"""Ask whether to allow fetch_local to a private/loopback host (timed).
|
||||||
|
|
||||||
Independent of YOLO: always prompts when interactive. Default is **deny**.
|
Independent of YOLO: always prompts when interactive. Default is **deny**.
|
||||||
Timeout, cancel, or non-interactive → False. On approve, the host:port is
|
Timeout, cancel, or non-interactive → False. On approve, the host:port is
|
||||||
granted for the rest of this process (see net grants).
|
granted for the rest of this process (see net grants).
|
||||||
"""
|
"""
|
||||||
reason = (
|
reason = (
|
||||||
f"fetch targets private/loopback host {host!r} port {port}\n"
|
f"fetch_local targets private/loopback host {host!r} port {port}\n"
|
||||||
f"url: {url}\n"
|
f"url: {url}\n"
|
||||||
f"Allow this host:port for the rest of this process?\n"
|
f"Allow this host:port for the rest of this process?\n"
|
||||||
f"(timeout {timeout_seconds:g}s defaults to DENY; not skipped by YOLO)"
|
f"(timeout {timeout_seconds:g}s defaults to DENY; not skipped by YOLO)"
|
||||||
@@ -332,7 +332,7 @@ def prompt_policy_fetch_confirm(
|
|||||||
backend.secho(format_tool_confirm_box("policy", reason), fg="yellow")
|
backend.secho(format_tool_confirm_box("policy", reason), fg="yellow")
|
||||||
backend.echo()
|
backend.echo()
|
||||||
grant = f"{host}:{port}"
|
grant = f"{host}:{port}"
|
||||||
prompt = f"[policy] allow fetch {grant!r}? [y/N] (timeout {timeout_seconds:g}s): "
|
prompt = f"[policy] allow fetch_local {grant!r}? [y/N] (timeout {timeout_seconds:g}s): "
|
||||||
with contextlib.suppress(OSError):
|
with contextlib.suppress(OSError):
|
||||||
_ = sys.stderr.write(prompt)
|
_ = sys.stderr.write(prompt)
|
||||||
_ = sys.stderr.flush()
|
_ = sys.stderr.flush()
|
||||||
|
|||||||
+34
-17
@@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Literal
|
|||||||
from plyngent.agent import ChatAgent, ToolRegistry
|
from plyngent.agent import ChatAgent, ToolRegistry
|
||||||
from plyngent.agent.loop import DEFAULT_MAX_ROUNDS
|
from plyngent.agent.loop import DEFAULT_MAX_ROUNDS
|
||||||
from plyngent.agent.todo_stack import TodoStack
|
from plyngent.agent.todo_stack import TodoStack
|
||||||
|
from plyngent.lmproto.deepseek.webtools import DeepseekWebToolsClient
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
@@ -382,20 +383,43 @@ class ReplState:
|
|||||||
search_tool = self._web_search_tool()
|
search_tool = self._web_search_tool()
|
||||||
if search_tool is not None:
|
if search_tool is not None:
|
||||||
registry.register(search_tool)
|
registry.register(search_tool)
|
||||||
|
fetch_tool = self._remote_fetch_tool()
|
||||||
|
if fetch_tool is not None:
|
||||||
|
registry.register(fetch_tool)
|
||||||
return registry
|
return registry
|
||||||
|
|
||||||
|
def _deepseek_web_client(self) -> DeepseekWebToolsClient | None:
|
||||||
|
"""The Search/Fetch API client for the active provider (None otherwise)."""
|
||||||
|
from plyngent.config.routing import deepseek_api_base_url
|
||||||
|
from plyngent.lmproto.openai_compatible.config import OpenAIConfig
|
||||||
|
|
||||||
|
provider = self.effective_provider
|
||||||
|
if provider.preset != "deepseek":
|
||||||
|
return None
|
||||||
|
return DeepseekWebToolsClient(
|
||||||
|
OpenAIConfig(
|
||||||
|
access_key_or_token=provider.access_key_or_token,
|
||||||
|
base_url=deepseek_api_base_url(provider),
|
||||||
|
timeout=normalize_http_timeout(provider.timeout),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
def _remote_fetch_tool(self) -> ToolDefinition | None:
|
||||||
|
"""``fetch`` through DeepSeek's Fetch API (DeepSeek providers only)."""
|
||||||
|
from plyngent.tools.net.fetch_remote import build_fetch_tool
|
||||||
|
|
||||||
|
client = self._deepseek_web_client()
|
||||||
|
if client is None:
|
||||||
|
return None
|
||||||
|
return build_fetch_tool(client)
|
||||||
|
|
||||||
def _web_search_tool(self) -> ToolDefinition | None:
|
def _web_search_tool(self) -> ToolDefinition | None:
|
||||||
"""Local ``web_search`` for a provider whose surface cannot host one.
|
"""Local ``web_search`` for a provider whose surface cannot host one.
|
||||||
|
|
||||||
The sources come from ``provider.search_sources``, in fallback order
|
The sources come from ``provider.search_sources``, in fallback order
|
||||||
(see :func:`~plyngent.config.routing.local_web_search`).
|
(see :func:`~plyngent.config.routing.local_web_search`).
|
||||||
"""
|
"""
|
||||||
from plyngent.config.routing import (
|
from plyngent.config.routing import local_web_search
|
||||||
deepseek_api_base_url,
|
|
||||||
local_web_search,
|
|
||||||
)
|
|
||||||
from plyngent.lmproto.deepseek.search import DeepseekSearchClient
|
|
||||||
from plyngent.lmproto.openai_compatible.config import OpenAIConfig
|
|
||||||
from plyngent.tools.net.bing import BingSearchBackend
|
from plyngent.tools.net.bing import BingSearchBackend
|
||||||
from plyngent.tools.net.search import (
|
from plyngent.tools.net.search import (
|
||||||
DeepseekSearchBackend,
|
DeepseekSearchBackend,
|
||||||
@@ -408,17 +432,10 @@ class ReplState:
|
|||||||
backends: list[SearchBackend] = []
|
backends: list[SearchBackend] = []
|
||||||
for source in provider.search_sources:
|
for source in provider.search_sources:
|
||||||
if source == "deepseek":
|
if source == "deepseek":
|
||||||
backends.append(
|
client = self._deepseek_web_client()
|
||||||
DeepseekSearchBackend(
|
if client is None:
|
||||||
DeepseekSearchClient(
|
continue
|
||||||
OpenAIConfig(
|
backends.append(DeepseekSearchBackend(client))
|
||||||
access_key_or_token=provider.access_key_or_token,
|
|
||||||
base_url=deepseek_api_base_url(provider),
|
|
||||||
timeout=normalize_http_timeout(provider.timeout),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
elif source == "bing":
|
elif source == "bing":
|
||||||
backends.append(BingSearchBackend())
|
backends.append(BingSearchBackend())
|
||||||
if not backends:
|
if not backends:
|
||||||
|
|||||||
@@ -71,18 +71,24 @@ with it to keep reading (chains through truncations).
|
|||||||
a `-c` one-liner, a script (`python x.py`), or a bare interactive shell alike — and each \
|
a `-c` one-liner, a script (`python x.py`), or a bare interactive shell alike — and each \
|
||||||
confirm blocks the turn, so it costs time. Invoke the target command directly (argv) instead \
|
confirm blocks the turn, so it costs time. Invoke the target command directly (argv) instead \
|
||||||
of wrapping it in `bash -c` / `python -c` / `node`, and reach for a dedicated tool \
|
of wrapping it in `bash -c` / `python -c` / `node`, and reach for a dedicated tool \
|
||||||
(`read_file` / `edit_*` / `vcs_*` / `fetch` / …) when one already does the job.
|
(`read_file` / `edit_*` / `vcs_*` / `fetch` / `fetch_local` / …) when one already does the job.
|
||||||
- Several `run_argv` calls in one step may run in parallel; use `run_argv_batch` \
|
- Several `run_argv` calls in one step may run in parallel; use `run_argv_batch` \
|
||||||
for ordered pipelines (`pipe_out` / `mix_stderr` as needed).
|
for ordered pipelines (`pipe_out` / `mix_stderr` as needed).
|
||||||
- Prefer `vcs_*` for status/diff/log/branch when enough.
|
- Prefer `vcs_*` for status/diff/log/branch when enough.
|
||||||
|
|
||||||
### Network
|
### Network
|
||||||
- Prefer `fetch` (GET/POST/PUT/DELETE) for HTTP(S) docs/APIs over curl/wget via shell; \
|
- `fetch` reads a public page as clean Markdown through the provider's Fetch API \
|
||||||
`offset` (0-based) skips leading body characters to page through a large body.
|
(when available, e.g. DeepSeek); `fetch_local` is the raw HTTP client \
|
||||||
- Set `user_agent` (or a User-Agent header) when the remote expects a specific client; \
|
(GET/POST/PUT/DELETE) from this host for localhost/LAN, APIs, custom headers, and \
|
||||||
otherwise a small default is used. Never rely on shell to spoof identity.
|
status/redirect inspection.
|
||||||
- Private/loopback/LAN hosts need an explicit human policy allow (not skipped by YOLO).
|
- `offset` (0-based) skips leading body characters to page through a large body; \
|
||||||
- Prefer provider hosted search (e.g. web_search) for open-ended research; use `fetch` for known URLs.
|
truncated bodies resume with `get_truncated` and are not re-requested.
|
||||||
|
- Set `user_agent` (or a User-Agent header) on `fetch_local` when the remote expects a \
|
||||||
|
specific client; otherwise a small default is used. Never rely on shell to spoof identity.
|
||||||
|
- Private/loopback/LAN hosts need an explicit human policy allow (not skipped by YOLO); \
|
||||||
|
this applies to `fetch_local`, since `fetch` leaves from the provider's network.
|
||||||
|
- Prefer provider hosted search (e.g. web_search) for open-ended research; use `fetch` \
|
||||||
|
or `fetch_local` for known URLs.
|
||||||
- Respect denials, size limits, and HTTPS→HTTP redirect blocks; do not re-fetch the same URL repeatedly.
|
- Respect denials, size limits, and HTTPS→HTTP redirect blocks; do not re-fetch the same URL repeatedly.
|
||||||
|
|
||||||
### Humans & safety
|
### Humans & safety
|
||||||
@@ -232,7 +238,7 @@ class McpConfig(Struct, omit_defaults=True):
|
|||||||
|
|
||||||
|
|
||||||
class NetworkingConfig(Struct, omit_defaults=True):
|
class NetworkingConfig(Struct, omit_defaults=True):
|
||||||
"""Host-side network policy for tools such as ``fetch``.
|
"""Host-side network policy for tools such as ``fetch_local``.
|
||||||
|
|
||||||
``ssrf_assume_public_cidrs``: CIDR strings whose resolved addresses are treated
|
``ssrf_assume_public_cidrs``: CIDR strings whose resolved addresses are treated
|
||||||
as **public for SSRF policy** (skip private-host grants). Use for Clash/meta
|
as **public for SSRF policy** (skip private-host grants). Use for Clash/meta
|
||||||
|
|||||||
+5
-1
@@ -1,8 +1,12 @@
|
|||||||
from .client import DeepseekSearchClient as DeepseekSearchClient
|
from .client import DeepseekWebToolsClient as DeepseekWebToolsClient
|
||||||
from .model import DEFAULT_SEARCH_COUNT as DEFAULT_SEARCH_COUNT
|
from .model import DEFAULT_SEARCH_COUNT as DEFAULT_SEARCH_COUNT
|
||||||
|
from .model import FETCH_PATH as FETCH_PATH
|
||||||
|
from .model import MAX_FETCH_URL_CHARS as MAX_FETCH_URL_CHARS
|
||||||
from .model import MAX_QUERY_CHARS as MAX_QUERY_CHARS
|
from .model import MAX_QUERY_CHARS as MAX_QUERY_CHARS
|
||||||
from .model import MAX_SEARCH_COUNT as MAX_SEARCH_COUNT
|
from .model import MAX_SEARCH_COUNT as MAX_SEARCH_COUNT
|
||||||
from .model import SEARCH_PATH as SEARCH_PATH
|
from .model import SEARCH_PATH as SEARCH_PATH
|
||||||
|
from .model import FetchRequest as FetchRequest
|
||||||
|
from .model import FetchResponse as FetchResponse
|
||||||
from .model import SearchRequest as SearchRequest
|
from .model import SearchRequest as SearchRequest
|
||||||
from .model import SearchResponse as SearchResponse
|
from .model import SearchResponse as SearchResponse
|
||||||
from .model import SearchResult as SearchResult
|
from .model import SearchResult as SearchResult
|
||||||
+52
-11
@@ -1,23 +1,27 @@
|
|||||||
"""DeepSeek's Search API: one request per query, no model round.
|
"""DeepSeek's Search and Fetch APIs: plain JSON, one bearer key.
|
||||||
|
|
||||||
The endpoint is a plain JSON API on the platform base URL, authenticated with
|
``POST /v1/tools/search`` answers a query in one request, without spending a
|
||||||
the same bearer key as the model APIs. It replaces the older server-side search
|
model round — the endpoint replaces the older server-side search turn, which
|
||||||
turn (``POST /messages`` with a hosted ``web_search`` tool), which spent a whole
|
cost a whole completion round (and its input tokens) just to pick the query and
|
||||||
completion round — and its input tokens — just to pick the query, and could only
|
could only return titles and URLs. A Search response already carries a snippet
|
||||||
return titles and URLs. A Search API response already carries a snippet and a
|
and a model-facing summary, so the local ``web_search`` tool can hand the model
|
||||||
model-facing summary, so the local ``web_search`` tool can hand the model
|
|
||||||
something readable on the first call.
|
something readable on the first call.
|
||||||
|
|
||||||
|
``POST /v1/tools/fetch`` reads one public page and returns its title and content
|
||||||
|
as Markdown, for the remote ``fetch`` tool. Both endpoints share the session,
|
||||||
|
authentication, error shape, retry policy and rate limits.
|
||||||
|
|
||||||
``count`` does not affect billing (one successful request costs the same for 1
|
``count`` does not affect billing (one successful request costs the same for 1
|
||||||
or 50 results); only successful requests are billable, so the retries below are
|
or 50 results); only successful requests are billable, so the retries below are
|
||||||
safe. The account-level limit is 5 QPS and is shared across the Search and Fetch
|
safe. The account-level limit is 5 QPS and is shared across the Search and Fetch
|
||||||
APIs independently of model concurrency.
|
APIs independently of model concurrency; Fetch also has a per-domain limit.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
|
from urllib.parse import urlsplit
|
||||||
|
|
||||||
import msgspec
|
import msgspec
|
||||||
import niquests
|
import niquests
|
||||||
@@ -31,9 +35,13 @@ from plyngent.lmproto.search import SearchHit
|
|||||||
|
|
||||||
from .model import (
|
from .model import (
|
||||||
DEFAULT_SEARCH_COUNT,
|
DEFAULT_SEARCH_COUNT,
|
||||||
|
FETCH_PATH,
|
||||||
|
MAX_FETCH_URL_CHARS,
|
||||||
MAX_QUERY_CHARS,
|
MAX_QUERY_CHARS,
|
||||||
MAX_SEARCH_COUNT,
|
MAX_SEARCH_COUNT,
|
||||||
SEARCH_PATH,
|
SEARCH_PATH,
|
||||||
|
FetchRequest,
|
||||||
|
FetchResponse,
|
||||||
SearchRequest,
|
SearchRequest,
|
||||||
SearchResponse,
|
SearchResponse,
|
||||||
SearchResult,
|
SearchResult,
|
||||||
@@ -51,12 +59,13 @@ RETRY_DELAYS: tuple[float, ...] = (0.5, 1.0)
|
|||||||
_SNIPPET_FALLBACK_CHARS = 200
|
_SNIPPET_FALLBACK_CHARS = 200
|
||||||
|
|
||||||
|
|
||||||
class DeepseekSearchClient:
|
class DeepseekWebToolsClient:
|
||||||
"""``POST /v1/tools/search``: one query in, ranked hits out."""
|
"""``POST /v1/tools/search`` and ``/v1/tools/fetch`` on one session."""
|
||||||
|
|
||||||
session: niquests.AsyncSession
|
session: niquests.AsyncSession
|
||||||
encoder: msgspec.json.Encoder
|
encoder: msgspec.json.Encoder
|
||||||
decoder: msgspec.json.Decoder[SearchResponse]
|
decoder: msgspec.json.Decoder[SearchResponse]
|
||||||
|
fetch_decoder: msgspec.json.Decoder[FetchResponse]
|
||||||
|
|
||||||
def __init__(self, config: OpenAIConfig) -> None:
|
def __init__(self, config: OpenAIConfig) -> None:
|
||||||
self.session = niquests.AsyncSession(
|
self.session = niquests.AsyncSession(
|
||||||
@@ -66,6 +75,7 @@ class DeepseekSearchClient:
|
|||||||
)
|
)
|
||||||
self.encoder = msgspec.json.Encoder()
|
self.encoder = msgspec.json.Encoder()
|
||||||
self.decoder = msgspec.json.Decoder(SearchResponse)
|
self.decoder = msgspec.json.Decoder(SearchResponse)
|
||||||
|
self.fetch_decoder = msgspec.json.Decoder(FetchResponse)
|
||||||
|
|
||||||
async def search(
|
async def search(
|
||||||
self,
|
self,
|
||||||
@@ -97,6 +107,19 @@ class DeepseekSearchClient:
|
|||||||
parsed = self.decoder.decode(body)
|
parsed = self.decoder.decode(body)
|
||||||
return [hit for hit in map(_to_hit, parsed.results) if hit.url]
|
return [hit for hit in map(_to_hit, parsed.results) if hit.url]
|
||||||
|
|
||||||
|
async def fetch(self, url: str) -> FetchResponse:
|
||||||
|
"""Read *url* and return its title and Markdown content.
|
||||||
|
|
||||||
|
The page is fetched from DeepSeek's network, not from this host: local
|
||||||
|
SSRF checks do not apply, and neither do private-host grants — a LAN or
|
||||||
|
loopback target is simply unreachable there (the endpoint answers 403 or
|
||||||
|
404). The URL is still validated here so a malformed request fails
|
||||||
|
before it costs a round trip.
|
||||||
|
"""
|
||||||
|
_validate_fetch_url(url)
|
||||||
|
body = await self._post(FETCH_PATH, self.encoder.encode(FetchRequest(url=url)))
|
||||||
|
return self.fetch_decoder.decode(body)
|
||||||
|
|
||||||
async def _post(self, path: str, data: bytes) -> bytes:
|
async def _post(self, path: str, data: bytes) -> bytes:
|
||||||
"""POST *data*, retrying the statuses that mean "wait and try again"."""
|
"""POST *data*, retrying the statuses that mean "wait and try again"."""
|
||||||
for delay in RETRY_DELAYS:
|
for delay in RETRY_DELAYS:
|
||||||
@@ -150,4 +173,22 @@ def _shorten(text: str) -> str:
|
|||||||
return text[: _SNIPPET_FALLBACK_CHARS - 1].rstrip() + "…"
|
return text[: _SNIPPET_FALLBACK_CHARS - 1].rstrip() + "…"
|
||||||
|
|
||||||
|
|
||||||
__all__ = ["RETRY_DELAYS", "DeepseekSearchClient"]
|
def _validate_fetch_url(url: str) -> None:
|
||||||
|
"""Reject a URL the endpoint would refuse, before paying for the request."""
|
||||||
|
if not url or len(url) > MAX_FETCH_URL_CHARS:
|
||||||
|
msg = f"fetch url must be 1-{MAX_FETCH_URL_CHARS} characters"
|
||||||
|
raise ValueError(msg)
|
||||||
|
try:
|
||||||
|
parts = urlsplit(url)
|
||||||
|
except ValueError:
|
||||||
|
msg = "fetch url is malformed"
|
||||||
|
raise ValueError(msg) from None
|
||||||
|
if parts.scheme not in {"http", "https"} or not parts.hostname:
|
||||||
|
msg = "fetch url must be an absolute http(s) URL with a host"
|
||||||
|
raise ValueError(msg)
|
||||||
|
if parts.username or parts.password:
|
||||||
|
msg = "fetch url must not carry a username or password"
|
||||||
|
raise ValueError(msg)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["RETRY_DELAYS", "DeepseekWebToolsClient"]
|
||||||
+20
-1
@@ -1,13 +1,15 @@
|
|||||||
"""Wire models for DeepSeek's Search API (``POST /v1/tools/search``)."""
|
"""Wire models for DeepSeek's Search and Fetch APIs."""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import msgspec
|
import msgspec
|
||||||
|
|
||||||
SEARCH_PATH = "/v1/tools/search"
|
SEARCH_PATH = "/v1/tools/search"
|
||||||
|
FETCH_PATH = "/v1/tools/fetch"
|
||||||
DEFAULT_SEARCH_COUNT = 10
|
DEFAULT_SEARCH_COUNT = 10
|
||||||
MAX_SEARCH_COUNT = 50
|
MAX_SEARCH_COUNT = 50
|
||||||
MAX_QUERY_CHARS = 128
|
MAX_QUERY_CHARS = 128
|
||||||
|
MAX_FETCH_URL_CHARS = 8192
|
||||||
|
|
||||||
|
|
||||||
class SearchRequest(msgspec.Struct, omit_defaults=True):
|
class SearchRequest(msgspec.Struct, omit_defaults=True):
|
||||||
@@ -38,11 +40,28 @@ class SearchResponse(msgspec.Struct):
|
|||||||
results: list[SearchResult]
|
results: list[SearchResult]
|
||||||
|
|
||||||
|
|
||||||
|
class FetchRequest(msgspec.Struct):
|
||||||
|
"""Body of one fetch request: a single absolute http(s) URL."""
|
||||||
|
|
||||||
|
url: str
|
||||||
|
|
||||||
|
|
||||||
|
class FetchResponse(msgspec.Struct):
|
||||||
|
"""A fetched page: its title and its content rendered as Markdown."""
|
||||||
|
|
||||||
|
title: str = ""
|
||||||
|
content: str = ""
|
||||||
|
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"DEFAULT_SEARCH_COUNT",
|
"DEFAULT_SEARCH_COUNT",
|
||||||
|
"FETCH_PATH",
|
||||||
|
"MAX_FETCH_URL_CHARS",
|
||||||
"MAX_QUERY_CHARS",
|
"MAX_QUERY_CHARS",
|
||||||
"MAX_SEARCH_COUNT",
|
"MAX_SEARCH_COUNT",
|
||||||
"SEARCH_PATH",
|
"SEARCH_PATH",
|
||||||
|
"FetchRequest",
|
||||||
|
"FetchResponse",
|
||||||
"SearchRequest",
|
"SearchRequest",
|
||||||
"SearchResponse",
|
"SearchResponse",
|
||||||
"SearchResult",
|
"SearchResult",
|
||||||
@@ -46,7 +46,7 @@ from .file import write_file as write_file
|
|||||||
from .net import NET_TOOLS as NET_TOOLS
|
from .net import NET_TOOLS as NET_TOOLS
|
||||||
from .net import clear_private_grants as clear_private_grants
|
from .net import clear_private_grants as clear_private_grants
|
||||||
from .net import clear_ssrf_assume_public_cidrs as clear_ssrf_assume_public_cidrs
|
from .net import clear_ssrf_assume_public_cidrs as clear_ssrf_assume_public_cidrs
|
||||||
from .net import fetch as fetch
|
from .net import fetch_local as fetch_local
|
||||||
from .net import grant_private_host as grant_private_host
|
from .net import grant_private_host as grant_private_host
|
||||||
from .net import set_fetch_policy_confirm_hook as set_fetch_policy_confirm_hook
|
from .net import set_fetch_policy_confirm_hook as set_fetch_policy_confirm_hook
|
||||||
from .net import set_ssrf_assume_public_cidrs as set_ssrf_assume_public_cidrs
|
from .net import set_ssrf_assume_public_cidrs as set_ssrf_assume_public_cidrs
|
||||||
|
|||||||
@@ -155,7 +155,7 @@ _read_only: ContextVar[bool] = ContextVar("plyngent_read_only_context", default=
|
|||||||
def read_only_context() -> Generator[None]:
|
def read_only_context() -> Generator[None]:
|
||||||
"""Mark the current context as read-only for tool execution.
|
"""Mark the current context as read-only for tool execution.
|
||||||
|
|
||||||
Tools that may mutate depending on arguments (e.g. ``fetch`` with a
|
Tools that may mutate depending on arguments (e.g. ``fetch_local`` with a
|
||||||
non-GET method) check :func:`is_read_only_context` and refuse to mutate
|
non-GET method) check :func:`is_read_only_context` and refuse to mutate
|
||||||
while it is set.
|
while it is set.
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -192,8 +192,8 @@ def _open_pty_reason(args: Mapping[str, object]) -> str | None:
|
|||||||
|
|
||||||
|
|
||||||
def _fetch_reason(args: Mapping[str, object]) -> str | None:
|
def _fetch_reason(args: Mapping[str, object]) -> str | None:
|
||||||
"""Soft-confirm cleartext HTTP and mutating fetch methods (not private-host policy)."""
|
"""Soft-confirm cleartext HTTP and mutating fetch_local methods (not private-host policy)."""
|
||||||
from plyngent.tools.net.fetch import fetch_soft_reason
|
from plyngent.tools.net.fetch_local import fetch_soft_reason
|
||||||
|
|
||||||
method_obj = args.get("method", "GET")
|
method_obj = args.get("method", "GET")
|
||||||
url_obj = args.get("url", "")
|
url_obj = args.get("url", "")
|
||||||
@@ -211,7 +211,7 @@ def classify_danger(name: str, args: Mapping[str, object]) -> str | None: # noq
|
|||||||
soft confirms for mutating tools and risky command launches — interpreters
|
soft confirms for mutating tools and risky command launches — interpreters
|
||||||
(however invoked) and detached runners such as ``nohup``, resolved past
|
(however invoked) and detached runners such as ``nohup``, resolved past
|
||||||
environment wrappers and ``<launcher> run``.
|
environment wrappers and ``<launcher> run``.
|
||||||
Private/loopback fetch targets use a separate policy grant (not YOLO).
|
Private/loopback fetch_local targets use a separate policy grant (not YOLO).
|
||||||
"""
|
"""
|
||||||
if name == "delete_path":
|
if name == "delete_path":
|
||||||
return _delete_path_reason(args)
|
return _delete_path_reason(args)
|
||||||
@@ -227,6 +227,6 @@ def classify_danger(name: str, args: Mapping[str, object]) -> str | None: # noq
|
|||||||
return _run_argv_batch_reason(args)
|
return _run_argv_batch_reason(args)
|
||||||
if name == "open_pty":
|
if name == "open_pty":
|
||||||
return _open_pty_reason(args)
|
return _open_pty_reason(args)
|
||||||
if name == "fetch":
|
if name == "fetch_local":
|
||||||
return _fetch_reason(args)
|
return _fetch_reason(args)
|
||||||
return None
|
return None
|
||||||
|
|||||||
@@ -60,14 +60,15 @@ async def get_truncated(token: str, *, max_chars: int = DEFAULT_TOOL_RESULT_MAX_
|
|||||||
"""Fetch the next chunk of a truncated result using its ``truncate_token``.
|
"""Fetch the next chunk of a truncated result using its ``truncate_token``.
|
||||||
|
|
||||||
Pass the token shown at the end of a truncated ``read_file``, ``fetch``,
|
Pass the token shown at the end of a truncated ``read_file``, ``fetch``,
|
||||||
``run_argv``, or earlier ``get_truncated`` result to keep reading without
|
``fetch_local``, ``run_argv``, or earlier ``get_truncated`` result to keep
|
||||||
re-requesting the whole source or raising limits. Truncated output carries a
|
reading without re-requesting the whole source or raising limits. Truncated
|
||||||
fresh token, so chunks chain indefinitely. File chunks resume in the same
|
output carries a fresh token, so chunks chain indefinitely. File chunks
|
||||||
view as the original read: numbered reads continue as numbered lines
|
resume in the same view as the original read: numbered reads continue as
|
||||||
(editable via ``edit_lineno``), plain reads as raw text with a 1-based
|
numbered lines (editable via ``edit_lineno``), plain reads as raw text with
|
||||||
``L{begin}-{end}`` range. Memory chunks (generic tool output, and ``fetch``
|
a 1-based ``L{begin}-{end}`` range. Memory chunks (generic tool output, and
|
||||||
bodies) are served from a short-lived in-memory store that is forgotten when
|
``fetch``/``fetch_local`` bodies) are served from a short-lived in-memory
|
||||||
the agent exits, so resuming a fetch never repeats the request.
|
store that is forgotten when the agent exits, so resuming a fetch never
|
||||||
|
repeats the request.
|
||||||
"""
|
"""
|
||||||
parsed = decode_truncate_token(token)
|
parsed = decode_truncate_token(token)
|
||||||
if parsed is None:
|
if parsed is None:
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
from .fetch import fetch as fetch
|
from .fetch_local import fetch_local as fetch_local
|
||||||
from .grants import clear_private_grants as clear_private_grants
|
from .grants import clear_private_grants as clear_private_grants
|
||||||
from .grants import grant_private_host as grant_private_host
|
from .grants import grant_private_host as grant_private_host
|
||||||
from .grants import set_fetch_policy_confirm_hook as set_fetch_policy_confirm_hook
|
from .grants import set_fetch_policy_confirm_hook as set_fetch_policy_confirm_hook
|
||||||
@@ -6,5 +6,5 @@ from .policy import clear_ssrf_assume_public_cidrs as clear_ssrf_assume_public_c
|
|||||||
from .policy import set_ssrf_assume_public_cidrs as set_ssrf_assume_public_cidrs
|
from .policy import set_ssrf_assume_public_cidrs as set_ssrf_assume_public_cidrs
|
||||||
|
|
||||||
NET_TOOLS = [
|
NET_TOOLS = [
|
||||||
fetch,
|
fetch_local,
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
Bing answers a query with a plain RSS document (``?format=rss``): no key, no
|
Bing answers a query with a plain RSS document (``?format=rss``): no key, no
|
||||||
page scraping, and each ``<item>`` already carries a title, a link and a
|
page scraping, and each ``<item>`` already carries a title, a link and a
|
||||||
description (the snippet DeepSeek's own index does not hand out). The request
|
description (the snippet DeepSeek's own index does not hand out). The request
|
||||||
goes through the fetch client the ``fetch`` tool uses, so the SSRF and
|
goes through the fetch client the ``fetch_local`` tool uses, so the SSRF and
|
||||||
private-host grant policy applies to it as well.
|
private-host grant policy applies to it as well.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
"""Model-facing ``fetch`` tool (HTTP GET/POST/PUT/DELETE)."""
|
"""Model-facing ``fetch_local`` tool (HTTP GET/POST/PUT/DELETE from this host)."""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
@@ -62,7 +62,7 @@ def format_fetch_result(
|
|||||||
|
|
||||||
|
|
||||||
@tool(tags=ToolTag.LOCAL | ToolTag.INSTANCE_STATE | ToolTag.YOLO | ToolTag.TRUSTABLE | ToolTag.READ_ONLY)
|
@tool(tags=ToolTag.LOCAL | ToolTag.INSTANCE_STATE | ToolTag.YOLO | ToolTag.TRUSTABLE | ToolTag.READ_ONLY)
|
||||||
async def fetch(
|
async def fetch_local(
|
||||||
url: str,
|
url: str,
|
||||||
*,
|
*,
|
||||||
method: str = "GET",
|
method: str = "GET",
|
||||||
@@ -97,7 +97,7 @@ async def fetch(
|
|||||||
try:
|
try:
|
||||||
verb = normalize_method(method)
|
verb = normalize_method(method)
|
||||||
if is_read_only_context() and verb != "GET":
|
if is_read_only_context() and verb != "GET":
|
||||||
return "error: fetch method must be GET in a read-only context (POST/PUT/DELETE mutate)"
|
return "error: fetch_local method must be GET in a read-only context (POST/PUT/DELETE mutate)"
|
||||||
parsed = parse_fetch_url(url)
|
parsed = parse_fetch_url(url)
|
||||||
hdrs = normalize_request_headers(headers, user_agent=user_agent)
|
hdrs = normalize_request_headers(headers, user_agent=user_agent)
|
||||||
body_bytes: bytes | None = None
|
body_bytes: bytes | None = None
|
||||||
@@ -119,7 +119,7 @@ async def fetch(
|
|||||||
except FetchPolicyError as exc:
|
except FetchPolicyError as exc:
|
||||||
return f"error: {exc}"
|
return f"error: {exc}"
|
||||||
except Exception as exc: # noqa: BLE001 — tool surface returns error text
|
except Exception as exc: # noqa: BLE001 — tool surface returns error text
|
||||||
return f"error: fetch failed: {exc}"
|
return f"error: fetch_local failed: {exc}"
|
||||||
|
|
||||||
return format_fetch_result(
|
return format_fetch_result(
|
||||||
status=result.status,
|
status=result.status,
|
||||||
@@ -0,0 +1,88 @@
|
|||||||
|
"""Model-facing ``fetch``: read a public page through DeepSeek's Fetch API.
|
||||||
|
|
||||||
|
The page is rendered to Markdown server-side, so it is the friendly counterpart
|
||||||
|
of ``fetch_local``: use it for public articles/docs, and keep ``fetch_local``
|
||||||
|
for localhost/LAN targets, non-GET requests, custom headers, and raw status or
|
||||||
|
redirect inspection. The request leaves from DeepSeek's network, not this host,
|
||||||
|
so local SSRF checks and private-host grants do not apply here.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import inspect
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
from plyngent.agent.tools import ToolDefinition, ToolTag, schema_from_callable
|
||||||
|
from plyngent.tools.net.policy import DEFAULT_MAX_CHARS
|
||||||
|
from plyngent.tools.truncate_token import truncate_to_memory
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from plyngent.lmproto.deepseek.webtools import DeepseekWebToolsClient
|
||||||
|
|
||||||
|
FETCH_TOOL_NAME = "fetch"
|
||||||
|
|
||||||
|
|
||||||
|
def format_page_result(
|
||||||
|
*,
|
||||||
|
title: str,
|
||||||
|
url: str,
|
||||||
|
content: str,
|
||||||
|
max_chars: int,
|
||||||
|
offset: int = 0,
|
||||||
|
) -> str:
|
||||||
|
"""Render a fetched page the way the other structured tool results read."""
|
||||||
|
start = max(0, offset)
|
||||||
|
body, token = truncate_to_memory(content[start:], max_chars)
|
||||||
|
parts = [
|
||||||
|
f"title={' '.join(title.split()) or '(untitled)'}",
|
||||||
|
f"url={url}",
|
||||||
|
f"chars={len(content)}",
|
||||||
|
f"truncated={'true' if token is not None else 'false'}",
|
||||||
|
"--- content ---",
|
||||||
|
body,
|
||||||
|
]
|
||||||
|
return "\n".join(parts)
|
||||||
|
|
||||||
|
|
||||||
|
def build_fetch_tool(
|
||||||
|
client: DeepseekWebToolsClient,
|
||||||
|
*,
|
||||||
|
default_max_chars: int = DEFAULT_MAX_CHARS,
|
||||||
|
) -> ToolDefinition:
|
||||||
|
"""A ``fetch`` definition backed by DeepSeek's Fetch API."""
|
||||||
|
|
||||||
|
async def fetch(url: str, offset: int = 0, max_chars: int = default_max_chars) -> str:
|
||||||
|
"""Read a public web page as Markdown through DeepSeek's Fetch API.
|
||||||
|
|
||||||
|
Use for public articles and docs when clean Markdown is enough. The page
|
||||||
|
is fetched from DeepSeek's network, so localhost/LAN targets are
|
||||||
|
unreachable here — use ``fetch_local`` for those, for non-GET methods, or
|
||||||
|
when you need the raw HTTP status/headers. A successful fetch is billed
|
||||||
|
per request (failures are not).
|
||||||
|
|
||||||
|
``offset`` (0-based) skips leading characters of the content. A body cut
|
||||||
|
by ``max_chars`` keeps its remainder in memory, resumable via its
|
||||||
|
``truncate_token`` without another request (and without another charge).
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
page = await client.fetch(url)
|
||||||
|
except Exception as exc: # noqa: BLE001 — tool surface returns error text
|
||||||
|
return f"error: fetch failed: {exc}"
|
||||||
|
return format_page_result(
|
||||||
|
title=page.title,
|
||||||
|
url=url,
|
||||||
|
content=page.content,
|
||||||
|
max_chars=max_chars,
|
||||||
|
offset=offset,
|
||||||
|
)
|
||||||
|
|
||||||
|
return ToolDefinition(
|
||||||
|
name=FETCH_TOOL_NAME,
|
||||||
|
description=inspect.getdoc(fetch) or "",
|
||||||
|
parameters=schema_from_callable(fetch),
|
||||||
|
handler=fetch,
|
||||||
|
tags=ToolTag.LOCAL | ToolTag.READ_ONLY,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["FETCH_TOOL_NAME", "build_fetch_tool", "format_page_result"]
|
||||||
@@ -77,7 +77,7 @@ async def ensure_host_allowed(
|
|||||||
try:
|
try:
|
||||||
inst = instance if instance is not None else require_bound_instance()
|
inst = instance if instance is not None else require_bound_instance()
|
||||||
except WorkspaceError as exc:
|
except WorkspaceError as exc:
|
||||||
msg = f"instance state is not bound for fetch policy: {exc}"
|
msg = f"instance state is not bound for fetch_local policy: {exc}"
|
||||||
raise FetchPolicyError(msg) from exc
|
raise FetchPolicyError(msg) from exc
|
||||||
|
|
||||||
parsed = parse_fetch_url(url)
|
parsed = parse_fetch_url(url)
|
||||||
@@ -85,7 +85,7 @@ async def ensure_host_allowed(
|
|||||||
if assessment.classification is HostClass.PUBLIC:
|
if assessment.classification is HostClass.PUBLIC:
|
||||||
return
|
return
|
||||||
if assessment.classification is HostClass.FORBIDDEN:
|
if assessment.classification is HostClass.FORBIDDEN:
|
||||||
msg = f"fetch blocked forbidden host {parsed.host!r} (addresses: {', '.join(assessment.addresses)})"
|
msg = f"fetch_local blocked forbidden host {parsed.host!r} (addresses: {', '.join(assessment.addresses)})"
|
||||||
raise FetchPolicyError(msg)
|
raise FetchPolicyError(msg)
|
||||||
|
|
||||||
# PRIVATE
|
# PRIVATE
|
||||||
@@ -100,18 +100,18 @@ async def ensure_host_allowed(
|
|||||||
hook = get_fetch_policy_confirm_hook(inst)
|
hook = get_fetch_policy_confirm_hook(inst)
|
||||||
if hook is None:
|
if hook is None:
|
||||||
msg = (
|
msg = (
|
||||||
f"fetch blocked private/loopback host {parsed.host}:{parsed.port} "
|
f"fetch_local blocked private/loopback host {parsed.host}:{parsed.port} "
|
||||||
f"({assessment.reason}; no policy confirm hook / non-interactive deny)"
|
f"({assessment.reason}; no policy confirm hook / non-interactive deny)"
|
||||||
)
|
)
|
||||||
raise FetchPolicyError(msg)
|
raise FetchPolicyError(msg)
|
||||||
try:
|
try:
|
||||||
allowed = bool(hook(parsed.host, parsed.port, parsed.url, timeout))
|
allowed = bool(hook(parsed.host, parsed.port, parsed.url, timeout))
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
msg = f"fetch blocked private/loopback host {parsed.host}:{parsed.port} (confirm failed: {exc})"
|
msg = f"fetch_local blocked private/loopback host {parsed.host}:{parsed.port} (confirm failed: {exc})"
|
||||||
raise FetchPolicyError(msg) from exc
|
raise FetchPolicyError(msg) from exc
|
||||||
if not allowed:
|
if not allowed:
|
||||||
msg = (
|
msg = (
|
||||||
f"fetch blocked private/loopback host {parsed.host}:{parsed.port} "
|
f"fetch_local blocked private/loopback host {parsed.host}:{parsed.port} "
|
||||||
f"(user declined or timed out after {timeout:g}s; not skipped by YOLO)"
|
f"(user declined or timed out after {timeout:g}s; not skipped by YOLO)"
|
||||||
)
|
)
|
||||||
raise FetchPolicyError(msg)
|
raise FetchPolicyError(msg)
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
"""URL / host policy for the fetch tool (SSRF, private grants, cleartext)."""
|
"""URL / host policy for the fetch_local tool (SSRF, private grants, cleartext)."""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
|||||||
@@ -9,7 +9,8 @@ the handler asks the configured search sources behind the scenes —
|
|||||||
one fails or finds nothing. DeepSeek's own source is the Search API
|
one fails or finds nothing. DeepSeek's own source is the Search API
|
||||||
(``POST /v1/tools/search``), which answers in one request instead of a model
|
(``POST /v1/tools/search``), which answers in one request instead of a model
|
||||||
round. A hit is a title, a URL and (when the source has one) a snippet: read a
|
round. A hit is a title, a URL and (when the source has one) a snippet: read a
|
||||||
page with ``fetch`` before relying on its content.
|
page with ``fetch`` (or ``fetch_local`` for local/LAN targets) before relying on
|
||||||
|
its content.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -22,7 +23,7 @@ from plyngent.agent.tools import ToolDefinition, ToolTag, schema_from_callable
|
|||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from collections.abc import Sequence
|
from collections.abc import Sequence
|
||||||
|
|
||||||
from plyngent.lmproto.deepseek.search import DeepseekSearchClient
|
from plyngent.lmproto.deepseek.webtools import DeepseekWebToolsClient
|
||||||
from plyngent.lmproto.search import SearchHit
|
from plyngent.lmproto.search import SearchHit
|
||||||
|
|
||||||
WEB_SEARCH_TOOL_NAME = "web_search"
|
WEB_SEARCH_TOOL_NAME = "web_search"
|
||||||
@@ -38,9 +39,9 @@ class SearchBackend(Protocol):
|
|||||||
class DeepseekSearchBackend:
|
class DeepseekSearchBackend:
|
||||||
"""DeepSeek's own index: one Search API request per query."""
|
"""DeepSeek's own index: one Search API request per query."""
|
||||||
|
|
||||||
_client: DeepseekSearchClient
|
_client: DeepseekWebToolsClient
|
||||||
|
|
||||||
def __init__(self, client: DeepseekSearchClient) -> None:
|
def __init__(self, client: DeepseekWebToolsClient) -> None:
|
||||||
self._client = client
|
self._client = client
|
||||||
|
|
||||||
async def search(self, query: str) -> Sequence[SearchHit]:
|
async def search(self, query: str) -> Sequence[SearchHit]:
|
||||||
@@ -82,7 +83,8 @@ def build_web_search_tool(
|
|||||||
"""Search the web and return the pages that matched.
|
"""Search the web and return the pages that matched.
|
||||||
|
|
||||||
One entry per hit (``title — url``, plus the snippet when the source has
|
One entry per hit (``title — url``, plus the snippet when the source has
|
||||||
one): read a page with ``fetch`` before relying on its content.
|
one): read a page with ``fetch`` (or ``fetch_local`` for local targets)
|
||||||
|
before relying on its content.
|
||||||
"""
|
"""
|
||||||
found, failures = await search_in_order(backends, query)
|
found, failures = await search_in_order(backends, query)
|
||||||
if not found:
|
if not found:
|
||||||
|
|||||||
@@ -2,8 +2,8 @@
|
|||||||
|
|
||||||
Every truncation site (the agent loop's generic tool-result cap,
|
Every truncation site (the agent loop's generic tool-result cap,
|
||||||
``run_argv`` / ``run_argv_batch`` per-stream caps, ``read_file``,
|
``run_argv`` / ``run_argv_batch`` per-stream caps, ``read_file``,
|
||||||
``fetch``, and request-time compact shrinks) appends the same compact marker
|
``fetch`` / ``fetch_local``, and request-time compact shrinks) appends the same
|
||||||
when output is cut short::
|
compact marker when output is cut short::
|
||||||
|
|
||||||
[Truncated (32000 chars max.; 500 omitted). truncate_token=...]
|
[Truncated (32000 chars max.; 500 omitted). truncate_token=...]
|
||||||
|
|
||||||
@@ -14,9 +14,9 @@ source is exhausted (then no marker is emitted).
|
|||||||
|
|
||||||
A ``file`` remainder is re-read from disk by ``location`` + ``offset``, so it
|
A ``file`` remainder is re-read from disk by ``location`` + ``offset``, so it
|
||||||
survives for as long as the file does. Everything else — arbitrary tool output
|
survives for as long as the file does. Everything else — arbitrary tool output
|
||||||
(run_argv stdout, todo renders, …) and ``fetch`` bodies — has no source worth
|
(run_argv stdout, todo renders, …) and ``fetch`` / ``fetch_local`` bodies — have
|
||||||
re-running (a re-fetch costs a round trip and re-hits the remote), so its
|
no source worth re-running (a re-fetch costs a round trip and re-hits the remote),
|
||||||
remainder lives in a short-lived in-memory store (``kind="memory"``) that is
|
so its remainder lives in a short-lived in-memory store (``kind="memory"``) that is
|
||||||
forgotten when the agent process exits; the model must re-run the tool after
|
forgotten when the agent process exits; the model must re-run the tool after
|
||||||
that.
|
that.
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -544,7 +544,7 @@ async def test_pretty_fetch(capsys: pytest.CaptureFixture[str]) -> None:
|
|||||||
"status=200\nmethod=GET\nfinal_url=https://example.com\ncontent_type=application/json\n"
|
"status=200\nmethod=GET\nfinal_url=https://example.com\ncontent_type=application/json\n"
|
||||||
"body_kind=json\nbytes=1229\ntruncated=false\nredirects=0\nsecurity=public\n--- body ---\n{...}\n"
|
"body_kind=json\nbytes=1229\ntruncated=false\nredirects=0\nsecurity=public\n--- body ---\n{...}\n"
|
||||||
)
|
)
|
||||||
await render_events(_aiter([_pretty_call("fetch", '{"url": "https://example.com"}'), _result(result)]))
|
await render_events(_aiter([_pretty_call("fetch_local", '{"url": "https://example.com"}'), _result(result)]))
|
||||||
out = capsys.readouterr().out
|
out = capsys.readouterr().out
|
||||||
assert "* Fetch GET https://example.com (200)" in out
|
assert "* Fetch GET https://example.com (200)" in out
|
||||||
assert "[tool]" not in out
|
assert "[tool]" not in out
|
||||||
@@ -555,7 +555,7 @@ async def test_pretty_fetch_error_status(capsys: pytest.CaptureFixture[str]) ->
|
|||||||
"status=404\nmethod=GET\nfinal_url=https://example.com/nope\ncontent_type=text/html\n"
|
"status=404\nmethod=GET\nfinal_url=https://example.com/nope\ncontent_type=text/html\n"
|
||||||
"body_kind=html\nbytes=45\ntruncated=false\nredirects=0\nsecurity=public\n--- body ---\nNot found\n"
|
"body_kind=html\nbytes=45\ntruncated=false\nredirects=0\nsecurity=public\n--- body ---\nNot found\n"
|
||||||
)
|
)
|
||||||
await render_events(_aiter([_pretty_call("fetch", '{"url": "https://example.com/nope"}'), _result(result)]))
|
await render_events(_aiter([_pretty_call("fetch_local", '{"url": "https://example.com/nope"}'), _result(result)]))
|
||||||
out = capsys.readouterr().out
|
out = capsys.readouterr().out
|
||||||
assert "* Fetch GET https://example.com/nope (404)" in out
|
assert "* Fetch GET https://example.com/nope (404)" in out
|
||||||
|
|
||||||
@@ -565,17 +565,37 @@ async def test_pretty_fetch_truncated(capsys: pytest.CaptureFixture[str]) -> Non
|
|||||||
"status=200\nmethod=GET\nfinal_url=https://example.com\ncontent_type=text/plain\n"
|
"status=200\nmethod=GET\nfinal_url=https://example.com\ncontent_type=text/plain\n"
|
||||||
"body_kind=text\nbytes=50000\ntruncated=true\nredirects=0\nsecurity=public\n--- body ---\n..."
|
"body_kind=text\nbytes=50000\ntruncated=true\nredirects=0\nsecurity=public\n--- body ---\n..."
|
||||||
)
|
)
|
||||||
await render_events(_aiter([_pretty_call("fetch", '{"url": "https://example.com"}'), _result(result)]))
|
await render_events(_aiter([_pretty_call("fetch_local", '{"url": "https://example.com"}'), _result(result)]))
|
||||||
out = capsys.readouterr().out
|
out = capsys.readouterr().out
|
||||||
assert "* Fetch GET https://example.com (200)" in out
|
assert "* Fetch GET https://example.com (200)" in out
|
||||||
|
|
||||||
|
|
||||||
async def test_pretty_fetch_error(capsys: pytest.CaptureFixture[str]) -> None:
|
async def test_pretty_fetch_error(capsys: pytest.CaptureFixture[str]) -> None:
|
||||||
await render_events(
|
await render_events(
|
||||||
_aiter([_pretty_call("fetch", '{"url": "https://example.com"}'), _result("error: fetch failed: boom")])
|
_aiter(
|
||||||
|
[_pretty_call("fetch_local", '{"url": "https://example.com"}'), _result("error: fetch_local failed: boom")]
|
||||||
|
)
|
||||||
)
|
)
|
||||||
out = capsys.readouterr().out
|
out = capsys.readouterr().out
|
||||||
assert "* Fetch GET https://example.com (error: fetch failed: boom)" in out
|
assert "* Fetch GET https://example.com (error: fetch_local failed: boom)" in out
|
||||||
|
|
||||||
|
|
||||||
|
async def test_pretty_remote_fetch(capsys: pytest.CaptureFixture[str]) -> None:
|
||||||
|
result = "title=Example article\nurl=https://example.com\nchars=1234\ntruncated=false\n--- content ---\n..."
|
||||||
|
await render_events(_aiter([_pretty_call("fetch", '{"url": "https://example.com"}'), _result(result)]))
|
||||||
|
out = capsys.readouterr().out
|
||||||
|
assert "* Fetch https://example.com (1234 chars)" in out
|
||||||
|
|
||||||
|
|
||||||
|
async def test_pretty_fetch_reads_a_legacy_local_row(capsys: pytest.CaptureFixture[str]) -> None:
|
||||||
|
# Rows written before the local tool was renamed carry the local status block.
|
||||||
|
result = (
|
||||||
|
"status=200\nmethod=GET\nfinal_url=https://example.com\ncontent_type=text/plain\n"
|
||||||
|
"body_kind=text\nbytes=12\ntruncated=false\nredirects=0\nsecurity=public\n--- body ---\nhi\n"
|
||||||
|
)
|
||||||
|
await render_events(_aiter([_pretty_call("fetch", '{"url": "https://example.com"}'), _result(result)]))
|
||||||
|
out = capsys.readouterr().out
|
||||||
|
assert "* Fetch https://example.com (200)" in out
|
||||||
|
|
||||||
|
|
||||||
async def test_pretty_web_search(capsys: pytest.CaptureFixture[str]) -> None:
|
async def test_pretty_web_search(capsys: pytest.CaptureFixture[str]) -> None:
|
||||||
|
|||||||
+54
-7
@@ -1,12 +1,12 @@
|
|||||||
"""DeepSeek's Search API client (``POST /v1/tools/search``)."""
|
"""DeepSeek's Search and Fetch APIs (``/v1/tools/search``, ``/v1/tools/fetch``)."""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import msgspec
|
import msgspec
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from plyngent.lmproto.deepseek.search import (
|
from plyngent.lmproto.deepseek.webtools import (
|
||||||
DeepseekSearchClient,
|
DeepseekWebToolsClient,
|
||||||
SearchRequest,
|
SearchRequest,
|
||||||
)
|
)
|
||||||
from plyngent.lmproto.openai_compatible.config import OpenAIConfig
|
from plyngent.lmproto.openai_compatible.config import OpenAIConfig
|
||||||
@@ -23,8 +23,8 @@ def _body(*results: dict[str, object]) -> bytes:
|
|||||||
return msgspec.json.encode({"results": list(results)})
|
return msgspec.json.encode({"results": list(results)})
|
||||||
|
|
||||||
|
|
||||||
def _client() -> DeepseekSearchClient:
|
def _client() -> DeepseekWebToolsClient:
|
||||||
return DeepseekSearchClient(
|
return DeepseekWebToolsClient(
|
||||||
OpenAIConfig(access_key_or_token="sk-test", base_url="https://api.deepseek.com"),
|
OpenAIConfig(access_key_or_token="sk-test", base_url="https://api.deepseek.com"),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -166,7 +166,7 @@ async def test_the_endpoint_error_detail_is_raised(monkeypatch: pytest.MonkeyPat
|
|||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_a_retryable_status_is_tried_again(monkeypatch: pytest.MonkeyPatch) -> None:
|
async def test_a_retryable_status_is_tried_again(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
client = _client()
|
client = _client()
|
||||||
monkeypatch.setattr("plyngent.lmproto.deepseek.search.client.RETRY_DELAYS", (0.0,))
|
monkeypatch.setattr("plyngent.lmproto.deepseek.webtools.client.RETRY_DELAYS", (0.0,))
|
||||||
statuses = [429, 200]
|
statuses = [429, 200]
|
||||||
|
|
||||||
async def fake_post(path: str, **kwargs: object) -> _Resp:
|
async def fake_post(path: str, **kwargs: object) -> _Resp:
|
||||||
@@ -181,7 +181,7 @@ async def test_a_retryable_status_is_tried_again(monkeypatch: pytest.MonkeyPatch
|
|||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_a_balance_error_is_not_retried(monkeypatch: pytest.MonkeyPatch) -> None:
|
async def test_a_balance_error_is_not_retried(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
client = _client()
|
client = _client()
|
||||||
monkeypatch.setattr("plyngent.lmproto.deepseek.search.client.RETRY_DELAYS", (0.0,))
|
monkeypatch.setattr("plyngent.lmproto.deepseek.webtools.client.RETRY_DELAYS", (0.0,))
|
||||||
calls: list[int] = []
|
calls: list[int] = []
|
||||||
|
|
||||||
async def fake_post(path: str, **kwargs: object) -> _Resp:
|
async def fake_post(path: str, **kwargs: object) -> _Resp:
|
||||||
@@ -197,3 +197,50 @@ async def test_a_balance_error_is_not_retried(monkeypatch: pytest.MonkeyPatch) -
|
|||||||
def test_the_request_model_omits_what_was_not_set() -> None:
|
def test_the_request_model_omits_what_was_not_set() -> None:
|
||||||
encoded = msgspec.json.encode(SearchRequest(query="q"))
|
encoded = msgspec.json.encode(SearchRequest(query="q"))
|
||||||
assert msgspec.json.decode(encoded, type=dict[str, object]) == {"query": "q"}
|
assert msgspec.json.decode(encoded, type=dict[str, object]) == {"query": "q"}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_fetch_posts_the_url_and_returns_the_page(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
client = _client()
|
||||||
|
sent: dict[str, object] = {}
|
||||||
|
|
||||||
|
async def fake_post(path: str, **kwargs: object) -> _Resp:
|
||||||
|
sent["path"] = path
|
||||||
|
sent.update(kwargs)
|
||||||
|
return _Resp(200, msgspec.json.encode({"title": "Example article", "content": "# Article\n\nBody."}))
|
||||||
|
|
||||||
|
monkeypatch.setattr(client.session, "post", fake_post)
|
||||||
|
page = await client.fetch("https://example.com/article")
|
||||||
|
|
||||||
|
assert page.title == "Example article"
|
||||||
|
assert page.content == "# Article\n\nBody."
|
||||||
|
assert sent["path"] == "/v1/tools/fetch"
|
||||||
|
assert _decode_request(sent["data"]) == {"url": "https://example.com/article"}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_fetch_rejects_a_url_the_endpoint_would_refuse(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
client = _client()
|
||||||
|
calls: list[object] = []
|
||||||
|
|
||||||
|
async def fake_post(path: str, **kwargs: object) -> _Resp:
|
||||||
|
calls.append(path)
|
||||||
|
return _Resp(200, b"{}")
|
||||||
|
|
||||||
|
monkeypatch.setattr(client.session, "post", fake_post)
|
||||||
|
for url in ("file:///etc/passwd", "https://user:secret@example.com/", "https:///no-host", ""):
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
_ = await client.fetch(url)
|
||||||
|
assert calls == []
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_fetch_surfaces_a_security_policy_rejection(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
client = _client()
|
||||||
|
|
||||||
|
async def fake_post(path: str, **kwargs: object) -> _Resp:
|
||||||
|
return _Resp(403, b'{"error": {"message": "target rejected by security policy"}}')
|
||||||
|
|
||||||
|
monkeypatch.setattr(client.session, "post", fake_post)
|
||||||
|
with pytest.raises(RuntimeError, match="security policy"):
|
||||||
|
_ = await client.fetch("https://example.com/blocked")
|
||||||
@@ -74,7 +74,7 @@ def test_read_only_classification() -> None:
|
|||||||
"vcs_diff",
|
"vcs_diff",
|
||||||
"vcs_log",
|
"vcs_log",
|
||||||
"vcs_branch",
|
"vcs_branch",
|
||||||
"fetch",
|
"fetch_local",
|
||||||
"skill_list",
|
"skill_list",
|
||||||
"skill_read",
|
"skill_read",
|
||||||
"skill_search",
|
"skill_search",
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
"""Tests for tools.net.fetch (policy, UA, methods, private grants)."""
|
"""Tests for tools.net.fetch_local (policy, UA, methods, private grants)."""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
@@ -10,7 +10,7 @@ import pytest
|
|||||||
|
|
||||||
from plyngent.tools.context import read_only_context
|
from plyngent.tools.context import read_only_context
|
||||||
from plyngent.tools.danger import classify_danger
|
from plyngent.tools.danger import classify_danger
|
||||||
from plyngent.tools.net import fetch, grant_private_host
|
from plyngent.tools.net import fetch_local, grant_private_host
|
||||||
from plyngent.tools.net.policy import (
|
from plyngent.tools.net.policy import (
|
||||||
DEFAULT_USER_AGENT,
|
DEFAULT_USER_AGENT,
|
||||||
FetchPolicyError,
|
FetchPolicyError,
|
||||||
@@ -30,7 +30,7 @@ if TYPE_CHECKING:
|
|||||||
|
|
||||||
|
|
||||||
class _Handler(BaseHTTPRequestHandler):
|
class _Handler(BaseHTTPRequestHandler):
|
||||||
"""Minimal echo server for fetch tests."""
|
"""Minimal echo server for fetch_local tests."""
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def log_message(self, format: str, *args: object) -> None:
|
def log_message(self, format: str, *args: object) -> None:
|
||||||
@@ -171,10 +171,10 @@ def test_soft_confirm_reason_matrix() -> None:
|
|||||||
|
|
||||||
|
|
||||||
def test_classify_danger_fetch() -> None:
|
def test_classify_danger_fetch() -> None:
|
||||||
assert classify_danger("fetch", {"url": "https://example.com/", "method": "GET"}) is None
|
assert classify_danger("fetch_local", {"url": "https://example.com/", "method": "GET"}) is None
|
||||||
reason = classify_danger("fetch", {"url": "http://example.com/", "method": "GET"})
|
reason = classify_danger("fetch_local", {"url": "http://example.com/", "method": "GET"})
|
||||||
assert reason is not None and "cleartext" in reason
|
assert reason is not None and "cleartext" in reason
|
||||||
reason2 = classify_danger("fetch", {"url": "https://example.com/api", "method": "DELETE"})
|
reason2 = classify_danger("fetch_local", {"url": "https://example.com/api", "method": "DELETE"})
|
||||||
assert reason2 is not None and "DELETE" in reason2
|
assert reason2 is not None and "DELETE" in reason2
|
||||||
|
|
||||||
|
|
||||||
@@ -184,36 +184,36 @@ async def test_fetch_get_post_put_delete(workspace: Path, http_server: str) -> N
|
|||||||
# Grant loopback for this process/instance (fixture binds InstanceState).
|
# Grant loopback for this process/instance (fixture binds InstanceState).
|
||||||
grant_private_host("127.0.0.1", int(base.rsplit(":", 1)[1]))
|
grant_private_host("127.0.0.1", int(base.rsplit(":", 1)[1]))
|
||||||
|
|
||||||
out = await call_async(fetch, f"{base}/hello")
|
out = await call_async(fetch_local, f"{base}/hello")
|
||||||
assert "status=200" in out
|
assert "status=200" in out
|
||||||
assert "GET /hello" in out
|
assert "GET /hello" in out
|
||||||
assert "security=cleartext-http" in out
|
assert "security=cleartext-http" in out
|
||||||
|
|
||||||
out_post = await call_async(fetch, f"{base}/echo", method="POST", body="hi")
|
out_post = await call_async(fetch_local, f"{base}/echo", method="POST", body="hi")
|
||||||
assert "status=201" in out_post
|
assert "status=201" in out_post
|
||||||
assert "POST:hi" in out_post
|
assert "POST:hi" in out_post
|
||||||
|
|
||||||
out_put = await call_async(fetch, f"{base}/echo", method="PUT", body="x")
|
out_put = await call_async(fetch_local, f"{base}/echo", method="PUT", body="x")
|
||||||
assert "PUT:x" in out_put
|
assert "PUT:x" in out_put
|
||||||
|
|
||||||
out_del = await call_async(fetch, f"{base}/x", method="DELETE")
|
out_del = await call_async(fetch_local, f"{base}/x", method="DELETE")
|
||||||
assert "status=204" in out_del
|
assert "status=204" in out_del
|
||||||
|
|
||||||
|
|
||||||
async def test_fetch_read_only_context_blocks_non_get(workspace: Path, http_server: str) -> None:
|
async def test_fetch_read_only_context_blocks_non_get(workspace: Path, http_server: str) -> None:
|
||||||
"""In a read-only context fetch only allows GET; outside it POST works."""
|
"""In a read-only context fetch_local only allows GET; outside it POST works."""
|
||||||
del workspace
|
del workspace
|
||||||
port = int(http_server.rsplit(":", 1)[1])
|
port = int(http_server.rsplit(":", 1)[1])
|
||||||
grant_private_host("127.0.0.1", port)
|
grant_private_host("127.0.0.1", port)
|
||||||
|
|
||||||
with read_only_context():
|
with read_only_context():
|
||||||
out = await call_async(fetch, f"{http_server}/hello")
|
out = await call_async(fetch_local, f"{http_server}/hello")
|
||||||
assert "status=200" in out # GET is fine in a read-only context
|
assert "status=200" in out # GET is fine in a read-only context
|
||||||
blocked = await call_async(fetch, f"{http_server}/echo", method="POST", body="hi")
|
blocked = await call_async(fetch_local, f"{http_server}/echo", method="POST", body="hi")
|
||||||
assert blocked.startswith("error:")
|
assert blocked.startswith("error:")
|
||||||
assert "GET" in blocked
|
assert "GET" in blocked
|
||||||
|
|
||||||
out_post = await call_async(fetch, f"{http_server}/echo", method="POST", body="hi")
|
out_post = await call_async(fetch_local, f"{http_server}/echo", method="POST", body="hi")
|
||||||
assert "POST:hi" in out_post
|
assert "POST:hi" in out_post
|
||||||
|
|
||||||
|
|
||||||
@@ -222,18 +222,18 @@ async def test_fetch_user_agent_passthrough(workspace: Path, http_server: str) -
|
|||||||
port = int(http_server.rsplit(":", 1)[1])
|
port = int(http_server.rsplit(":", 1)[1])
|
||||||
grant_private_host("127.0.0.1", port)
|
grant_private_host("127.0.0.1", port)
|
||||||
|
|
||||||
out = await call_async(fetch, f"{http_server}/ua", user_agent="AgentUA/9")
|
out = await call_async(fetch_local, f"{http_server}/ua", user_agent="AgentUA/9")
|
||||||
assert "AgentUA/9" in out
|
assert "AgentUA/9" in out
|
||||||
|
|
||||||
out2 = await call_async(
|
out2 = await call_async(
|
||||||
fetch,
|
fetch_local,
|
||||||
f"{http_server}/ua",
|
f"{http_server}/ua",
|
||||||
headers={"User-Agent": "HeaderUA/1"},
|
headers={"User-Agent": "HeaderUA/1"},
|
||||||
)
|
)
|
||||||
assert "HeaderUA/1" in out2
|
assert "HeaderUA/1" in out2
|
||||||
|
|
||||||
out3 = await call_async(
|
out3 = await call_async(
|
||||||
fetch,
|
fetch_local,
|
||||||
f"{http_server}/ua",
|
f"{http_server}/ua",
|
||||||
headers={"User-Agent": "HeaderUA/1"},
|
headers={"User-Agent": "HeaderUA/1"},
|
||||||
user_agent="ArgUA/2",
|
user_agent="ArgUA/2",
|
||||||
@@ -247,11 +247,11 @@ async def test_fetch_truncation_and_binary(workspace: Path, http_server: str) ->
|
|||||||
port = int(http_server.rsplit(":", 1)[1])
|
port = int(http_server.rsplit(":", 1)[1])
|
||||||
grant_private_host("127.0.0.1", port)
|
grant_private_host("127.0.0.1", port)
|
||||||
|
|
||||||
out = await call_async(fetch, f"{http_server}/big", max_bytes=100)
|
out = await call_async(fetch_local, f"{http_server}/big", max_bytes=100)
|
||||||
assert "truncated=true" in out
|
assert "truncated=true" in out
|
||||||
assert "bytes=100" in out
|
assert "bytes=100" in out
|
||||||
|
|
||||||
out_bin = await call_async(fetch, f"{http_server}/bin")
|
out_bin = await call_async(fetch_local, f"{http_server}/bin")
|
||||||
assert "body_kind=binary" in out_bin
|
assert "body_kind=binary" in out_bin
|
||||||
assert "binary body omitted" in out_bin
|
assert "binary body omitted" in out_bin
|
||||||
|
|
||||||
@@ -261,7 +261,7 @@ async def test_fetch_char_truncation_embeds_token(workspace: Path, http_server:
|
|||||||
port = int(http_server.rsplit(":", 1)[1])
|
port = int(http_server.rsplit(":", 1)[1])
|
||||||
grant_private_host("127.0.0.1", port)
|
grant_private_host("127.0.0.1", port)
|
||||||
|
|
||||||
out = await call_async(fetch, f"{http_server}/big", max_chars=1000)
|
out = await call_async(fetch_local, f"{http_server}/big", max_chars=1000)
|
||||||
assert "truncated=true" in out
|
assert "truncated=true" in out
|
||||||
body = out.split("--- body ---", 1)[-1]
|
body = out.split("--- body ---", 1)[-1]
|
||||||
assert "truncate_token=" in body
|
assert "truncate_token=" in body
|
||||||
@@ -281,7 +281,7 @@ async def test_fetch_offset_skips_body(workspace: Path, http_server: str) -> Non
|
|||||||
grant_private_host("127.0.0.1", port)
|
grant_private_host("127.0.0.1", port)
|
||||||
|
|
||||||
# /big body is 5000 chars; offset 4900 leaves 100 chars, no token needed.
|
# /big body is 5000 chars; offset 4900 leaves 100 chars, no token needed.
|
||||||
out = await call_async(fetch, f"{http_server}/big", max_chars=200, offset=4900)
|
out = await call_async(fetch_local, f"{http_server}/big", max_chars=200, offset=4900)
|
||||||
body = out.split("--- body ---", 1)[-1]
|
body = out.split("--- body ---", 1)[-1]
|
||||||
assert body.strip() == "x" * 100
|
assert body.strip() == "x" * 100
|
||||||
assert "truncate_token=" not in out
|
assert "truncate_token=" not in out
|
||||||
@@ -290,21 +290,21 @@ async def test_fetch_offset_skips_body(workspace: Path, http_server: str) -> Non
|
|||||||
async def test_fetch_private_denied_without_grant(workspace: Path, http_server: str) -> None:
|
async def test_fetch_private_denied_without_grant(workspace: Path, http_server: str) -> None:
|
||||||
del workspace
|
del workspace
|
||||||
# No grant, no policy hook → hard deny (even though server is up).
|
# No grant, no policy hook → hard deny (even though server is up).
|
||||||
out = await call_async(fetch, f"{http_server}/hello")
|
out = await call_async(fetch_local, f"{http_server}/hello")
|
||||||
assert out.startswith("error:")
|
assert out.startswith("error:")
|
||||||
assert "private" in out or "loopback" in out
|
assert "private" in out or "loopback" in out
|
||||||
|
|
||||||
|
|
||||||
async def test_fetch_forbidden_metadata(workspace: Path) -> None:
|
async def test_fetch_forbidden_metadata(workspace: Path) -> None:
|
||||||
del workspace
|
del workspace
|
||||||
out = await call_async(fetch, "http://169.254.169.254/latest/meta-data/")
|
out = await call_async(fetch_local, "http://169.254.169.254/latest/meta-data/")
|
||||||
assert out.startswith("error:")
|
assert out.startswith("error:")
|
||||||
assert "forbidden" in out
|
assert "forbidden" in out
|
||||||
|
|
||||||
|
|
||||||
async def test_fetch_bad_scheme(workspace: Path) -> None:
|
async def test_fetch_bad_scheme(workspace: Path) -> None:
|
||||||
del workspace
|
del workspace
|
||||||
out = await call_async(fetch, "file:///etc/passwd")
|
out = await call_async(fetch_local, "file:///etc/passwd")
|
||||||
assert out.startswith("error:")
|
assert out.startswith("error:")
|
||||||
assert "scheme" in out
|
assert "scheme" in out
|
||||||
|
|
||||||
@@ -313,6 +313,6 @@ async def test_fetch_redirect_loop_capped(workspace: Path, http_server: str) ->
|
|||||||
del workspace
|
del workspace
|
||||||
port = int(http_server.rsplit(":", 1)[1])
|
port = int(http_server.rsplit(":", 1)[1])
|
||||||
grant_private_host("127.0.0.1", port)
|
grant_private_host("127.0.0.1", port)
|
||||||
out = await call_async(fetch, f"{http_server}/redirect-loop", max_redirects=3)
|
out = await call_async(fetch_local, f"{http_server}/redirect-loop", max_redirects=3)
|
||||||
assert out.startswith("error:")
|
assert out.startswith("error:")
|
||||||
assert "redirect" in out.lower()
|
assert "redirect" in out.lower()
|
||||||
@@ -0,0 +1,73 @@
|
|||||||
|
"""The remote ``fetch`` tool (DeepSeek's Fetch API, Markdown pages)."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from plyngent.agent.tools import ToolTag
|
||||||
|
from plyngent.lmproto.deepseek.webtools import FetchResponse
|
||||||
|
from plyngent.tools.net.fetch_remote import FETCH_TOOL_NAME, build_fetch_tool
|
||||||
|
|
||||||
|
|
||||||
|
class _Client:
|
||||||
|
"""Stands in for DeepseekWebToolsClient.fetch."""
|
||||||
|
|
||||||
|
def __init__(self, *, page: FetchResponse | None = None, error: str | None = None) -> None:
|
||||||
|
self.page = page or FetchResponse(title="T", content="body")
|
||||||
|
self.error = error
|
||||||
|
self.urls: list[str] = []
|
||||||
|
|
||||||
|
async def fetch(self, url: str) -> FetchResponse:
|
||||||
|
self.urls.append(url)
|
||||||
|
if self.error is not None:
|
||||||
|
raise RuntimeError(self.error)
|
||||||
|
return self.page
|
||||||
|
|
||||||
|
|
||||||
|
def _tool(client: _Client, *, max_chars: int = 32_000):
|
||||||
|
return build_fetch_tool(client, default_max_chars=max_chars)
|
||||||
|
|
||||||
|
|
||||||
|
def test_definition_shape() -> None:
|
||||||
|
definition = _tool(_Client())
|
||||||
|
assert definition.name == FETCH_TOOL_NAME == "fetch"
|
||||||
|
assert definition.tags == ToolTag.LOCAL | ToolTag.READ_ONLY
|
||||||
|
properties = definition.parameters["properties"]
|
||||||
|
assert set(properties) == {"url", "offset", "max_chars"}
|
||||||
|
assert definition.parameters["required"] == ["url"]
|
||||||
|
assert "fetch_local" in definition.description
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_handler_renders_title_url_and_content() -> None:
|
||||||
|
client = _Client(page=FetchResponse(title=" Example article ", content="# Heading\n\nBody."))
|
||||||
|
result = await _tool(client).handler(url="https://example.com/article")
|
||||||
|
|
||||||
|
assert client.urls == ["https://example.com/article"]
|
||||||
|
assert result.splitlines() == [
|
||||||
|
"title=Example article",
|
||||||
|
"url=https://example.com/article",
|
||||||
|
"chars=16",
|
||||||
|
"truncated=false",
|
||||||
|
"--- content ---",
|
||||||
|
"# Heading",
|
||||||
|
"",
|
||||||
|
"Body.",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_offset_and_truncation_keep_the_remainder_resumable() -> None:
|
||||||
|
client = _Client(page=FetchResponse(title="", content="0123456789"))
|
||||||
|
result = await _tool(client, max_chars=3).handler(url="https://example.com/", offset=5)
|
||||||
|
|
||||||
|
assert "title=(untitled)" in result
|
||||||
|
assert "--- content ---\n567" in result
|
||||||
|
assert "truncated=true" in result
|
||||||
|
assert "truncate_token=" in result
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_handler_reports_a_client_error() -> None:
|
||||||
|
result = await _tool(_Client(error="HTTP 404: no usable content")).handler(url="https://example.com/x")
|
||||||
|
assert result == "error: fetch failed: HTTP 404: no usable content"
|
||||||
@@ -86,9 +86,9 @@ async def test_get_truncated_file_missing(workspace: Path) -> None:
|
|||||||
|
|
||||||
async def test_get_truncated_serves_a_fetch_body_without_refetching(monkeypatch) -> None:
|
async def test_get_truncated_serves_a_fetch_body_without_refetching(monkeypatch) -> None:
|
||||||
"""A fetch body's token reads the stored remainder; the request is not repeated."""
|
"""A fetch body's token reads the stored remainder; the request is not repeated."""
|
||||||
from plyngent.tools.net.fetch import format_fetch_result
|
from plyngent.tools.net.fetch_local import format_fetch_result
|
||||||
|
|
||||||
fetch_module = importlib.import_module("plyngent.tools.net.fetch")
|
fetch_module = importlib.import_module("plyngent.tools.net.fetch_local")
|
||||||
|
|
||||||
async def _must_not_refetch(*_args: object, **_kwargs: object) -> str:
|
async def _must_not_refetch(*_args: object, **_kwargs: object) -> str:
|
||||||
msg = "resuming a fetch body must not re-run the request"
|
msg = "resuming a fetch body must not re-run the request"
|
||||||
|
|||||||
Reference in New Issue
Block a user