diff --git a/src/plyngent/agent/directive_checkpoint.py b/src/plyngent/agent/directive_checkpoint.py index 22c5e77..218f983 100644 --- a/src/plyngent/agent/directive_checkpoint.py +++ b/src/plyngent/agent/directive_checkpoint.py @@ -21,6 +21,7 @@ Tool playbook still applies (see system). Hard constraints: use `run_argv_batch` for ordered pipelines. - `edit_replace`: fix match / `max_replaces`; `edit_lineno` edits only lines read via \ `read_file(with_lineno=true)`, and line numbers reset after any file write. +- Truncated results carry a `TRUNCATE_TOKEN`; continue with `get_truncated`. - Prefer `fetch` for HTTP(S); private/LAN hosts need human policy allow (not YOLO). - PTY secrets only via `ask_into_pty`; denylists and confirms still apply. - `wait` pauses; pressing Enter disturbs early (optional reason). diff --git a/src/plyngent/config/models.py b/src/plyngent/config/models.py index ced3f89..36ff4c7 100644 --- a/src/plyngent/config/models.py +++ b/src/plyngent/config/models.py @@ -32,6 +32,8 @@ raise `max_replaces` or narrow `old_string`. - `edit_lineno` edits only lines you read via `read_file(path, with_lineno=true)`; \ re-read with line numbers after any `edit_*` / `write_file` / copy/move/delete, \ since line numbers go stale. +- Truncated `read_file` / `fetch` results carry a `TRUNCATE_TOKEN`; use \ +`get_truncated` with it to keep reading (chains through truncations). ### Commands - Prefer `run_argv` / `run_argv_batch` (argv lists, no shell) over `bash -c` or similar. diff --git a/src/plyngent/tools/file/__init__.py b/src/plyngent/tools/file/__init__.py index cc46280..5753146 100644 --- a/src/plyngent/tools/file/__init__.py +++ b/src/plyngent/tools/file/__init__.py @@ -5,6 +5,7 @@ from .edit_replace import edit_replace as edit_replace from .fs_ops import copy_path as copy_path from .fs_ops import delete_path as delete_path from .fs_ops import move_path as move_path +from .get_truncated import get_truncated as get_truncated from .glob_paths import glob_paths as glob_paths from .grep_files import grep_files as grep_files from .listdir import listdir as listdir @@ -25,4 +26,5 @@ FILE_TOOLS = [ move_path, delete_path, new_temporary_workspace, + get_truncated, ] diff --git a/src/plyngent/tools/file/get_truncated.py b/src/plyngent/tools/file/get_truncated.py new file mode 100644 index 0000000..0e10b1d --- /dev/null +++ b/src/plyngent/tools/file/get_truncated.py @@ -0,0 +1,40 @@ +"""``get_truncated`` tool: resume truncated tool results via a truncate token.""" + +from __future__ import annotations + +from plyngent.agent import ToolTag, tool +from plyngent.agent.budget import DEFAULT_TOOL_RESULT_MAX_CHARS +from plyngent.tools.file.read import read_file +from plyngent.tools.net.fetch import fetch +from plyngent.tools.truncate_token import decode_truncate_token, truncate_with_token + + +@tool(tags=ToolTag.LOCAL | ToolTag.INSTANCE_STATE) +async def get_truncated(token: str, *, max_chars: int = DEFAULT_TOOL_RESULT_MAX_CHARS) -> str: + """Fetch the next chunk of a truncated result using its ``TRUNCATE_TOKEN``. + + Pass the token shown at the end of a truncated ``read_file``, ``fetch``, or + earlier ``get_truncated`` result to keep reading without re-requesting the + whole source or raising limits. Truncated output carries a fresh token, so + chunks chain indefinitely. + """ + parsed = decode_truncate_token(token) + if parsed is None: + return "error: invalid truncate token" + if parsed.kind == "http": + return await fetch.handler(parsed.location, offset=parsed.offset, max_chars=parsed.limit) + text = await read_file.handler(parsed.location) + if text.startswith("error:"): + return text + start = parsed.offset + segment = text[start : start + max_chars] + chunk, _ = truncate_with_token( + segment, + max_chars, + kind="file", + location=parsed.location, + offset=start, + limit=max_chars, + total_len=len(text), + ) + return chunk diff --git a/tests/test_tools/test_fetch.py b/tests/test_tools/test_fetch.py index d8362e1..c29667e 100644 --- a/tests/test_tools/test_fetch.py +++ b/tests/test_tools/test_fetch.py @@ -252,8 +252,12 @@ async def test_fetch_char_truncation_embeds_token(workspace: Path, http_server: token = decode_truncate_token(token_str) assert token is not None assert token.kind == "http" - assert token.offset == 1000 # next chunk starts at max_chars assert token.limit == 1000 + assert 0 < token.offset < 1000 + # Continuation resumes exactly where the chunk stopped (no gap). + out2 = await call_async(fetch, f"{http_server}/big", max_chars=1000, offset=token.offset) + body2 = out2.split("--- body ---", 1)[-1].strip() + assert body2.startswith("x") async def test_fetch_offset_skips_body(workspace: Path, http_server: str) -> None: diff --git a/tests/test_tools/test_get_truncated.py b/tests/test_tools/test_get_truncated.py new file mode 100644 index 0000000..52ef9a8 --- /dev/null +++ b/tests/test_tools/test_get_truncated.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import importlib +from pathlib import Path + +from plyngent.tools.truncate_token import TruncateToken, decode_truncate_token, encode_truncate_token +from tests.test_tools.helpers import call_async + +get_truncated_module = importlib.import_module("plyngent.tools.file.get_truncated") +get_truncated = get_truncated_module.get_truncated + + +def _extract_token(out: str) -> str | None: + for line in out.splitlines(): + if "TRUNCATE_TOKEN:" in line: + return line.split("[TRUNCATE_TOKEN: ", 1)[1].rstrip("]") + return None + + +async def test_get_truncated_invalid_token() -> None: + out = await call_async(get_truncated, "garbage-token") + assert out.startswith("error:") + assert "invalid" in out + + +async def test_get_truncated_file_chains_without_gap(workspace: Path) -> None: + (workspace / "big.txt").write_text("a" * 1000, encoding="utf-8") + token = encode_truncate_token(TruncateToken(kind="file", location="big.txt", offset=0, limit=200)) + total_read = 0 + for _ in range(20): + out = await call_async(get_truncated, token, max_chars=200) + content, _, _ = out.partition("\n...[") + assert content == "a" * len(content) + total_read += len(content) + next_token = _extract_token(out) + if next_token is None: + break + parsed = decode_truncate_token(next_token) + assert parsed is not None + assert parsed.offset == total_read # no gap, no overlap + token = next_token + assert total_read == 1000 + + +async def test_get_truncated_file_missing(workspace: Path) -> None: + token = encode_truncate_token(TruncateToken(kind="file", location="nope.txt", offset=0, limit=200)) + out = await call_async(get_truncated, token, max_chars=200) + assert out.startswith("error:") + + +async def test_get_truncated_http_passthrough(monkeypatch) -> None: + calls: list[tuple[str, int, int]] = [] + + async def fake_fetch(url: str, *, offset: int, max_chars: int) -> str: + calls.append((url, offset, max_chars)) + return f"body-from-{offset}" + + monkeypatch.setattr(get_truncated_module.fetch, "handler", fake_fetch) + token = encode_truncate_token(TruncateToken(kind="http", location="https://example.com/x", offset=500, limit=400)) + out = await call_async(get_truncated, token, max_chars=200) + assert out == "body-from-500" + assert calls == [("https://example.com/x", 500, 400)] diff --git a/tests/test_tools/test_truncate_token.py b/tests/test_tools/test_truncate_token.py index 3a3b215..6c1c5a6 100644 --- a/tests/test_tools/test_truncate_token.py +++ b/tests/test_tools/test_truncate_token.py @@ -53,10 +53,11 @@ def test_truncate_with_token_cuts_and_advances() -> None: total_len=500, ) assert token is not None + content, _, _ = out.partition("\n...[") + assert content.startswith("x") assert "truncated 300 chars" in out assert f"[TRUNCATE_TOKEN: {encode_truncate_token(token)}]" in out - assert token.offset == 200 - assert token.limit == 200 + assert token.offset == len(content) # cursor lands exactly where content stops assert len(out) <= 200 # marker fits inside the char budget @@ -73,7 +74,7 @@ def test_truncate_with_token_more_after_nonoverflow() -> None: total_len=200, ) assert token is not None - assert token.offset == 40 + assert token.offset == 30 # advanced by returned length only assert "more content available" in out assert out.startswith("y" * 30) @@ -106,6 +107,7 @@ def test_truncate_with_token_nonzero_start() -> None: ) assert token is not None assert token.kind == "http" - assert token.offset == 350 # 150 + 200 + content, _, _ = out.partition("\n...[") + assert token.offset == 150 + len(content) assert "truncated 300 chars" in out assert len(out) <= 200