NCBM/plyngent
1
0
Fork 0
mirror of https://github.com/NCBM/plyngent.git synced 2026-11-07 15:37:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

core/tools+cli: split fetch into remote fetch and fetch_local

Browse Source
This commit is contained in:
worldmozara
2026-10-17 17:06:43 +08:00
parent 37e130a7dc
commit cfe36b024b
28 changed files with 465 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_tools/test_catalog.py
+1 -1
View File
@@ -74,7 +74,7 @@ def test_read_only_classification() -> None:
"vcs_diff",
"vcs_log",
"vcs_branch",
"fetch",
"fetch_local",
"skill_list",
"skill_read",
"skill_search",
tests/test_tools/test_fetch.py → tests/test_tools/test_fetch_local.py
+25 -25
View File
@@ -1,4 +1,4 @@
"""Tests for tools.net.fetch (policy, UA, methods, private grants)."""
"""Tests for tools.net.fetch_local (policy, UA, methods, private grants)."""
from __future__ import annotations
@@ -10,7 +10,7 @@ import pytest
from plyngent.tools.context import read_only_context
from plyngent.tools.danger import classify_danger
from plyngent.tools.net import fetch, grant_private_host
from plyngent.tools.net import fetch_local, grant_private_host
from plyngent.tools.net.policy import (
DEFAULT_USER_AGENT,
FetchPolicyError,
@@ -30,7 +30,7 @@ if TYPE_CHECKING:
class _Handler(BaseHTTPRequestHandler):
"""Minimal echo server for fetch tests."""
"""Minimal echo server for fetch_local tests."""
@override
def log_message(self, format: str, *args: object) -> None:
@@ -171,10 +171,10 @@ def test_soft_confirm_reason_matrix() -> None:
def test_classify_danger_fetch() -> None:
assert classify_danger("fetch", {"url": "https://example.com/", "method": "GET"}) is None
reason = classify_danger("fetch", {"url": "http://example.com/", "method": "GET"})
assert classify_danger("fetch_local", {"url": "https://example.com/", "method": "GET"}) is None
reason = classify_danger("fetch_local", {"url": "http://example.com/", "method": "GET"})
assert reason is not None and "cleartext" in reason
reason2 = classify_danger("fetch", {"url": "https://example.com/api", "method": "DELETE"})
reason2 = classify_danger("fetch_local", {"url": "https://example.com/api", "method": "DELETE"})
assert reason2 is not None and "DELETE" in reason2
@@ -184,36 +184,36 @@ async def test_fetch_get_post_put_delete(workspace: Path, http_server: str) -> N
# Grant loopback for this process/instance (fixture binds InstanceState).
grant_private_host("127.0.0.1", int(base.rsplit(":", 1)[1]))
out = await call_async(fetch, f"{base}/hello")
out = await call_async(fetch_local, f"{base}/hello")
assert "status=200" in out
assert "GET /hello" in out
assert "security=cleartext-http" in out
out_post = await call_async(fetch, f"{base}/echo", method="POST", body="hi")
out_post = await call_async(fetch_local, f"{base}/echo", method="POST", body="hi")
assert "status=201" in out_post
assert "POST:hi" in out_post
out_put = await call_async(fetch, f"{base}/echo", method="PUT", body="x")
out_put = await call_async(fetch_local, f"{base}/echo", method="PUT", body="x")
assert "PUT:x" in out_put
out_del = await call_async(fetch, f"{base}/x", method="DELETE")
out_del = await call_async(fetch_local, f"{base}/x", method="DELETE")
assert "status=204" in out_del
async def test_fetch_read_only_context_blocks_non_get(workspace: Path, http_server: str) -> None:
"""In a read-only context fetch only allows GET; outside it POST works."""
"""In a read-only context fetch_local only allows GET; outside it POST works."""
del workspace
port = int(http_server.rsplit(":", 1)[1])
grant_private_host("127.0.0.1", port)
with read_only_context():
out = await call_async(fetch, f"{http_server}/hello")
out = await call_async(fetch_local, f"{http_server}/hello")
assert "status=200" in out # GET is fine in a read-only context
blocked = await call_async(fetch, f"{http_server}/echo", method="POST", body="hi")
blocked = await call_async(fetch_local, f"{http_server}/echo", method="POST", body="hi")
assert blocked.startswith("error:")
assert "GET" in blocked
out_post = await call_async(fetch, f"{http_server}/echo", method="POST", body="hi")
out_post = await call_async(fetch_local, f"{http_server}/echo", method="POST", body="hi")
assert "POST:hi" in out_post
@@ -222,18 +222,18 @@ async def test_fetch_user_agent_passthrough(workspace: Path, http_server: str) -
port = int(http_server.rsplit(":", 1)[1])
grant_private_host("127.0.0.1", port)
out = await call_async(fetch, f"{http_server}/ua", user_agent="AgentUA/9")
out = await call_async(fetch_local, f"{http_server}/ua", user_agent="AgentUA/9")
assert "AgentUA/9" in out
out2 = await call_async(
fetch,
fetch_local,
f"{http_server}/ua",
headers={"User-Agent": "HeaderUA/1"},
)
assert "HeaderUA/1" in out2
out3 = await call_async(
fetch,
fetch_local,
f"{http_server}/ua",
headers={"User-Agent": "HeaderUA/1"},
user_agent="ArgUA/2",
@@ -247,11 +247,11 @@ async def test_fetch_truncation_and_binary(workspace: Path, http_server: str) ->
port = int(http_server.rsplit(":", 1)[1])
grant_private_host("127.0.0.1", port)
out = await call_async(fetch, f"{http_server}/big", max_bytes=100)
out = await call_async(fetch_local, f"{http_server}/big", max_bytes=100)
assert "truncated=true" in out
assert "bytes=100" in out
out_bin = await call_async(fetch, f"{http_server}/bin")
out_bin = await call_async(fetch_local, f"{http_server}/bin")
assert "body_kind=binary" in out_bin
assert "binary body omitted" in out_bin
@@ -261,7 +261,7 @@ async def test_fetch_char_truncation_embeds_token(workspace: Path, http_server:
port = int(http_server.rsplit(":", 1)[1])
grant_private_host("127.0.0.1", port)
out = await call_async(fetch, f"{http_server}/big", max_chars=1000)
out = await call_async(fetch_local, f"{http_server}/big", max_chars=1000)
assert "truncated=true" in out
body = out.split("--- body ---", 1)[-1]
assert "truncate_token=" in body
@@ -281,7 +281,7 @@ async def test_fetch_offset_skips_body(workspace: Path, http_server: str) -> Non
grant_private_host("127.0.0.1", port)
# /big body is 5000 chars; offset 4900 leaves 100 chars, no token needed.
out = await call_async(fetch, f"{http_server}/big", max_chars=200, offset=4900)
out = await call_async(fetch_local, f"{http_server}/big", max_chars=200, offset=4900)
body = out.split("--- body ---", 1)[-1]
assert body.strip() == "x" * 100
assert "truncate_token=" not in out
@@ -290,21 +290,21 @@ async def test_fetch_offset_skips_body(workspace: Path, http_server: str) -> Non
async def test_fetch_private_denied_without_grant(workspace: Path, http_server: str) -> None:
del workspace
# No grant, no policy hook → hard deny (even though server is up).
out = await call_async(fetch, f"{http_server}/hello")
out = await call_async(fetch_local, f"{http_server}/hello")
assert out.startswith("error:")
assert "private" in out or "loopback" in out
async def test_fetch_forbidden_metadata(workspace: Path) -> None:
del workspace
out = await call_async(fetch, "http://169.254.169.254/latest/meta-data/")
out = await call_async(fetch_local, "http://169.254.169.254/latest/meta-data/")
assert out.startswith("error:")
assert "forbidden" in out
async def test_fetch_bad_scheme(workspace: Path) -> None:
del workspace
out = await call_async(fetch, "file:///etc/passwd")
out = await call_async(fetch_local, "file:///etc/passwd")
assert out.startswith("error:")
assert "scheme" in out
@@ -313,6 +313,6 @@ async def test_fetch_redirect_loop_capped(workspace: Path, http_server: str) ->
del workspace
port = int(http_server.rsplit(":", 1)[1])
grant_private_host("127.0.0.1", port)
out = await call_async(fetch, f"{http_server}/redirect-loop", max_redirects=3)
out = await call_async(fetch_local, f"{http_server}/redirect-loop", max_redirects=3)
assert out.startswith("error:")
assert "redirect" in out.lower()
tests/test_tools/test_fetch_remote.py
+73
View File
@@ -0,0 +1,73 @@
"""The remote ``fetch`` tool (DeepSeek's Fetch API, Markdown pages)."""
from __future__ import annotations
import pytest
from plyngent.agent.tools import ToolTag
from plyngent.lmproto.deepseek.webtools import FetchResponse
from plyngent.tools.net.fetch_remote import FETCH_TOOL_NAME, build_fetch_tool
class _Client:
"""Stands in for DeepseekWebToolsClient.fetch."""
def __init__(self, *, page: FetchResponse | None = None, error: str | None = None) -> None:
self.page = page or FetchResponse(title="T", content="body")
self.error = error
self.urls: list[str] = []
async def fetch(self, url: str) -> FetchResponse:
self.urls.append(url)
if self.error is not None:
raise RuntimeError(self.error)
return self.page
def _tool(client: _Client, *, max_chars: int = 32_000):
return build_fetch_tool(client, default_max_chars=max_chars)
def test_definition_shape() -> None:
definition = _tool(_Client())
assert definition.name == FETCH_TOOL_NAME == "fetch"
assert definition.tags == ToolTag.LOCAL | ToolTag.READ_ONLY
properties = definition.parameters["properties"]
assert set(properties) == {"url", "offset", "max_chars"}
assert definition.parameters["required"] == ["url"]
assert "fetch_local" in definition.description
@pytest.mark.asyncio
async def test_handler_renders_title_url_and_content() -> None:
client = _Client(page=FetchResponse(title=" Example article ", content="# Heading\n\nBody."))
result = await _tool(client).handler(url="https://example.com/article")
assert client.urls == ["https://example.com/article"]
assert result.splitlines() == [
"title=Example article",
"url=https://example.com/article",
"chars=16",
"truncated=false",
"--- content ---",
"# Heading",
"",
"Body.",
]
@pytest.mark.asyncio
async def test_offset_and_truncation_keep_the_remainder_resumable() -> None:
client = _Client(page=FetchResponse(title="", content="0123456789"))
result = await _tool(client, max_chars=3).handler(url="https://example.com/", offset=5)
assert "title=(untitled)" in result
assert "--- content ---\n567" in result
assert "truncated=true" in result
assert "truncate_token=" in result
@pytest.mark.asyncio
async def test_handler_reports_a_client_error() -> None:
result = await _tool(_Client(error="HTTP 404: no usable content")).handler(url="https://example.com/x")
assert result == "error: fetch failed: HTTP 404: no usable content"
tests/test_tools/test_get_truncated.py
+2 -2
View File
@@ -86,9 +86,9 @@ async def test_get_truncated_file_missing(workspace: Path) -> None:
async def test_get_truncated_serves_a_fetch_body_without_refetching(monkeypatch) -> None:
"""A fetch body's token reads the stored remainder; the request is not repeated."""
from plyngent.tools.net.fetch import format_fetch_result
from plyngent.tools.net.fetch_local import format_fetch_result
fetch_module = importlib.import_module("plyngent.tools.net.fetch")
fetch_module = importlib.import_module("plyngent.tools.net.fetch_local")
async def _must_not_refetch(*_args: object, **_kwargs: object) -> str:
msg = "resuming a fetch body must not re-run the request"