core/tools: resume fetch bodies from memory instead of refetching

This commit is contained in:
2026-10-06 19:14:38 +08:00
parent be16839d9a
commit 25b8ef3e58
6 changed files with 76 additions and 53 deletions
+2 -5
View File
@@ -269,13 +269,10 @@ async def test_fetch_char_truncation_embeds_token(workspace: Path, http_server:
token_str = token_line.split("truncate_token=", 1)[1].rstrip("]")
token = decode_truncate_token(token_str)
assert token is not None
assert token.kind == "http"
# The body is kept in memory, so resuming never repeats the request.
assert token.kind == "memory"
assert token.limit == 1000
assert 0 < token.offset < 1000
# Continuation resumes exactly where the chunk stopped (no gap).
out2 = await call_async(fetch, f"{http_server}/big", max_chars=1000, offset=token.offset)
body2 = out2.split("--- body ---", 1)[-1].strip()
assert body2.startswith("x")
async def test_fetch_offset_skips_body(workspace: Path, http_server: str) -> None:
+31 -9
View File
@@ -84,18 +84,40 @@ async def test_get_truncated_file_missing(workspace: Path) -> None:
assert out.startswith("error:")
async def test_get_truncated_http_passthrough(monkeypatch) -> None:
calls: list[tuple[str, int, int]] = []
async def test_get_truncated_serves_a_fetch_body_without_refetching(monkeypatch) -> None:
"""A fetch body's token reads the stored remainder; the request is not repeated."""
from plyngent.tools.net.fetch import format_fetch_result
async def fake_fetch(url: str, *, offset: int, max_chars: int) -> str:
calls.append((url, offset, max_chars))
return f"body-from-{offset}"
fetch_module = importlib.import_module("plyngent.tools.net.fetch")
monkeypatch.setattr(get_truncated_module.fetch, "handler", fake_fetch)
token = encode_truncate_token(TruncateToken(kind="http", location="https://example.com/x", offset=500, limit=400))
async def _must_not_refetch(*_args: object, **_kwargs: object) -> str:
msg = "resuming a fetch body must not re-run the request"
raise AssertionError(msg)
monkeypatch.setattr(fetch_module, "http_fetch", _must_not_refetch)
first = format_fetch_result(
status=200,
final_url="https://example.com/x",
content_type="text/plain",
body_text="x" * 600,
body_bytes=600,
truncated=False,
redirects=0,
method="GET",
security="public",
warnings=[],
body_kind="text",
max_chars=200,
)
token = _extract_token(first)
assert token is not None
parsed = decode_truncate_token(token)
assert parsed is not None
assert parsed.kind == "memory"
out = await call_async(get_truncated, token, max_chars=200)
assert out == "body-from-500"
assert calls == [("https://example.com/x", 500, 400)]
content, _, _ = out.partition("\n[Truncated")
assert content == "x" * len(content)
assert 0 < len(content) <= 200
async def test_get_truncated_memory_chains() -> None:
+4 -4
View File
@@ -26,7 +26,7 @@ def test_decode_garbage_returns_none() -> None:
def test_decode_rejects_invalid_fields() -> None:
bad = TruncateToken(kind="file", location="x", offset=-1, limit=10)
assert decode_truncate_token(encode_truncate_token(bad)) is None
bad_limit = TruncateToken(kind="http", location="u", offset=0, limit=0)
bad_limit = TruncateToken(kind="file", location="u", offset=0, limit=0)
assert decode_truncate_token(encode_truncate_token(bad_limit)) is None
@@ -95,7 +95,7 @@ def test_truncate_with_token_end_reached_no_token() -> None:
out, token = truncate_with_token(
text,
40,
kind="http",
kind="file",
location="https://example.com/x",
offset=0,
limit=40,
@@ -111,14 +111,14 @@ def test_truncate_with_token_nonzero_start() -> None:
out, token = truncate_with_token(
text,
200,
kind="http",
kind="file",
location="https://example.com/x",
offset=150,
limit=200,
total_len=700,
)
assert token is not None
assert token.kind == "http"
assert token.kind == "file"
content, _, _ = out.partition("\n[Truncated")
assert token.offset == 150 + len(content)
assert "[Truncated (200 chars max.; 350 omitted). truncate_token=" in out