mirror of
https://github.com/NCBM/plyngent.git
synced 2026-11-07 15:37:55 +08:00
core/tools: resume fetch bodies from memory instead of refetching
This commit is contained in:
@@ -269,13 +269,10 @@ async def test_fetch_char_truncation_embeds_token(workspace: Path, http_server:
|
||||
token_str = token_line.split("truncate_token=", 1)[1].rstrip("]")
|
||||
token = decode_truncate_token(token_str)
|
||||
assert token is not None
|
||||
assert token.kind == "http"
|
||||
# The body is kept in memory, so resuming never repeats the request.
|
||||
assert token.kind == "memory"
|
||||
assert token.limit == 1000
|
||||
assert 0 < token.offset < 1000
|
||||
# Continuation resumes exactly where the chunk stopped (no gap).
|
||||
out2 = await call_async(fetch, f"{http_server}/big", max_chars=1000, offset=token.offset)
|
||||
body2 = out2.split("--- body ---", 1)[-1].strip()
|
||||
assert body2.startswith("x")
|
||||
|
||||
|
||||
async def test_fetch_offset_skips_body(workspace: Path, http_server: str) -> None:
|
||||
|
||||
@@ -84,18 +84,40 @@ async def test_get_truncated_file_missing(workspace: Path) -> None:
|
||||
assert out.startswith("error:")
|
||||
|
||||
|
||||
async def test_get_truncated_http_passthrough(monkeypatch) -> None:
|
||||
calls: list[tuple[str, int, int]] = []
|
||||
async def test_get_truncated_serves_a_fetch_body_without_refetching(monkeypatch) -> None:
|
||||
"""A fetch body's token reads the stored remainder; the request is not repeated."""
|
||||
from plyngent.tools.net.fetch import format_fetch_result
|
||||
|
||||
async def fake_fetch(url: str, *, offset: int, max_chars: int) -> str:
|
||||
calls.append((url, offset, max_chars))
|
||||
return f"body-from-{offset}"
|
||||
fetch_module = importlib.import_module("plyngent.tools.net.fetch")
|
||||
|
||||
monkeypatch.setattr(get_truncated_module.fetch, "handler", fake_fetch)
|
||||
token = encode_truncate_token(TruncateToken(kind="http", location="https://example.com/x", offset=500, limit=400))
|
||||
async def _must_not_refetch(*_args: object, **_kwargs: object) -> str:
|
||||
msg = "resuming a fetch body must not re-run the request"
|
||||
raise AssertionError(msg)
|
||||
|
||||
monkeypatch.setattr(fetch_module, "http_fetch", _must_not_refetch)
|
||||
first = format_fetch_result(
|
||||
status=200,
|
||||
final_url="https://example.com/x",
|
||||
content_type="text/plain",
|
||||
body_text="x" * 600,
|
||||
body_bytes=600,
|
||||
truncated=False,
|
||||
redirects=0,
|
||||
method="GET",
|
||||
security="public",
|
||||
warnings=[],
|
||||
body_kind="text",
|
||||
max_chars=200,
|
||||
)
|
||||
token = _extract_token(first)
|
||||
assert token is not None
|
||||
parsed = decode_truncate_token(token)
|
||||
assert parsed is not None
|
||||
assert parsed.kind == "memory"
|
||||
out = await call_async(get_truncated, token, max_chars=200)
|
||||
assert out == "body-from-500"
|
||||
assert calls == [("https://example.com/x", 500, 400)]
|
||||
content, _, _ = out.partition("\n[Truncated")
|
||||
assert content == "x" * len(content)
|
||||
assert 0 < len(content) <= 200
|
||||
|
||||
|
||||
async def test_get_truncated_memory_chains() -> None:
|
||||
|
||||
@@ -26,7 +26,7 @@ def test_decode_garbage_returns_none() -> None:
|
||||
def test_decode_rejects_invalid_fields() -> None:
|
||||
bad = TruncateToken(kind="file", location="x", offset=-1, limit=10)
|
||||
assert decode_truncate_token(encode_truncate_token(bad)) is None
|
||||
bad_limit = TruncateToken(kind="http", location="u", offset=0, limit=0)
|
||||
bad_limit = TruncateToken(kind="file", location="u", offset=0, limit=0)
|
||||
assert decode_truncate_token(encode_truncate_token(bad_limit)) is None
|
||||
|
||||
|
||||
@@ -95,7 +95,7 @@ def test_truncate_with_token_end_reached_no_token() -> None:
|
||||
out, token = truncate_with_token(
|
||||
text,
|
||||
40,
|
||||
kind="http",
|
||||
kind="file",
|
||||
location="https://example.com/x",
|
||||
offset=0,
|
||||
limit=40,
|
||||
@@ -111,14 +111,14 @@ def test_truncate_with_token_nonzero_start() -> None:
|
||||
out, token = truncate_with_token(
|
||||
text,
|
||||
200,
|
||||
kind="http",
|
||||
kind="file",
|
||||
location="https://example.com/x",
|
||||
offset=150,
|
||||
limit=200,
|
||||
total_len=700,
|
||||
)
|
||||
assert token is not None
|
||||
assert token.kind == "http"
|
||||
assert token.kind == "file"
|
||||
content, _, _ = out.partition("\n[Truncated")
|
||||
assert token.offset == 150 + len(content)
|
||||
assert "[Truncated (200 chars max.; 350 omitted). truncate_token=" in out
|
||||
|
||||
Reference in New Issue
Block a user