From d914590677a9a3f56cde7a70836abcd0adc73544 Mon Sep 17 00:00:00 2001 From: worldmozara Date: Sat, 15 Aug 2026 17:35:57 +0800 Subject: [PATCH] core/tools: numbered read continuation via get_truncated Two fixes for the truncated numbered-read flow: - read_file(with_lineno) tokens were rebased onto numbered-body chars (each line carries an N| prefix), so get_truncated over-advanced into raw text and silently skipped lines. Tokens now advance by raw chars of the complete lines shown and carry a numbered flag. - get_truncated resumes numbered tokens as numbered lines (via read_file with_lineno), marking them readable for edit_lineno, so read -> get_truncated -> edit_lineno works end-to-end without gaps. --- src/plyngent/tools/file/get_truncated.py | 70 +++++++++++++++++------- src/plyngent/tools/file/read.py | 24 +++++++- src/plyngent/tools/truncate_token.py | 4 +- tests/test_tools/test_get_truncated.py | 34 ++++++++++++ 4 files changed, 108 insertions(+), 24 deletions(-) diff --git a/src/plyngent/tools/file/get_truncated.py b/src/plyngent/tools/file/get_truncated.py index aab9203..6fafc6a 100644 --- a/src/plyngent/tools/file/get_truncated.py +++ b/src/plyngent/tools/file/get_truncated.py @@ -6,15 +6,56 @@ from typing import cast from plyngent.agent import ToolTag, tool from plyngent.agent.budget import DEFAULT_TOOL_RESULT_MAX_CHARS -from plyngent.tools.file.read import line_range_label, read_raw_text +from plyngent.tools.file.read import line_range_label, read_file, read_raw_text from plyngent.tools.net.fetch import fetch from plyngent.tools.truncate_token import ( + TruncateToken, decode_truncate_token, get_remainder, truncate_with_token, ) +def _memory_chunk(parsed: TruncateToken, max_chars: int) -> str: + """Serve one chunk from the in-memory remainder store (or an error).""" + remainder = get_remainder(parsed.location) + if remainder is None: + return "error: truncate token expired (no longer in memory); re-run the tool" + start = parsed.offset + segment = remainder[start : start + max_chars] + chunk, _ = truncate_with_token( + segment, + max_chars, + kind="memory", + location=parsed.location, + offset=start, + limit=max_chars, + total_len=len(remainder), + ) + return chunk + + +async def _numbered_continuation(text: str, location: str, char_offset: int, max_chars: int) -> str: + """Resume a numbered read at the line containing *char_offset*. + + Delegates to ``read_file(with_lineno=True)`` so the continuation carries + ``N|`` line numbers and marks the resumed lines readable for ``edit_lineno``. + """ + pos = 0 + line = 0 + for ln in text.splitlines(keepends=True): + if pos + len(ln) > char_offset: + break + pos += len(ln) + line += 1 + return await read_file.handler( + location, + offset=line, + with_lineno=True, + max_chars=max_chars, + ) + + @tool(tags=ToolTag.LOCAL | ToolTag.INSTANCE_STATE | ToolTag.READ_ONLY) async def get_truncated(token: str, *, max_chars: int = DEFAULT_TOOL_RESULT_MAX_CHARS) -> str: """Fetch the next chunk of a truncated result using its ``truncate_token``. @@ -22,10 +63,11 @@ async def get_truncated(token: str, *, max_chars: int = DEFAULT_TOOL_RESULT_MAX_ Pass the token shown at the end of a truncated ``read_file``, ``fetch``, ``run_command``, or earlier ``get_truncated`` result to keep reading without re-requesting the whole source or raising limits. Truncated output carries a - fresh token, so chunks chain indefinitely. File chunks start with a 1-based - ``L{begin}-{end}`` line range like ``read_file``; memory chunks (generic - tool output) are served from a short-lived in-memory store that is forgotten - when the agent exits. + fresh token, so chunks chain indefinitely. File chunks resume in the same + view as the original read: numbered reads continue as numbered lines + (editable via ``edit_lineno``), plain reads as raw text with a 1-based + ``L{begin}-{end}`` range. Memory chunks (generic tool output) are served + from a short-lived in-memory store that is forgotten when the agent exits. """ parsed = decode_truncate_token(token) if parsed is None: @@ -33,26 +75,14 @@ async def get_truncated(token: str, *, max_chars: int = DEFAULT_TOOL_RESULT_MAX_ if parsed.kind == "http": return await fetch.handler(parsed.location, offset=parsed.offset, max_chars=parsed.limit) if parsed.kind == "memory": - remainder = get_remainder(parsed.location) - if remainder is None: - return "error: truncate token expired (no longer in memory); re-run the tool" - start = parsed.offset - segment = remainder[start : start + max_chars] - chunk, _ = truncate_with_token( - segment, - max_chars, - kind="memory", - location=parsed.location, - offset=start, - limit=max_chars, - total_len=len(remainder), - ) - return chunk + return _memory_chunk(parsed, max_chars) text, err = read_raw_text(parsed.location) if err: return err text = cast("str", text) start = parsed.offset + if parsed.numbered: + return await _numbered_continuation(text, parsed.location, start, max_chars) segment = text[start : start + max_chars] chunk, _ = truncate_with_token( segment, diff --git a/src/plyngent/tools/file/read.py b/src/plyngent/tools/file/read.py index 69a352d..edf2695 100644 --- a/src/plyngent/tools/file/read.py +++ b/src/plyngent/tools/file/read.py @@ -4,7 +4,7 @@ from typing import cast from plyngent.agent import ToolTag, mark_lineno_read, tool from plyngent.agent.budget import DEFAULT_TOOL_RESULT_MAX_CHARS -from plyngent.tools.truncate_token import truncate_with_token +from plyngent.tools.truncate_token import TruncateToken, truncate_with_token, truncation_marker from plyngent.tools.workspace import resolve_path _LINENO_WIDTH = 6 @@ -84,8 +84,8 @@ async def read_file( return "" slice_lines = lines[start:end] body = _format_with_lineno(slice_lines, start_lineno=start + 1) if with_lineno else "".join(slice_lines) + char_start = len("".join(lines[:start])) if max_chars >= 1: - char_start = len("".join(lines[:start])) body, token = truncate_with_token( body, max_chars, @@ -110,7 +110,25 @@ async def read_file( counted = body if token is not None: counted = body.split("\n[Truncated", 1)[0] - shown = counted.count("\n") + shown = counted.count("\n") + # The truncate cursor counts *numbered* chars (each line carries an + # "N|" prefix), which would over-advance into raw text and skip + # lines on continuation. Rebase the token onto raw characters of the + # complete lines shown (at least one, so the cursor always moves) + # and mark it as the numbered view for get_truncated. + consumed_lines = max(shown, 1) + raw_consumed = sum(len(ln) for ln in slice_lines[:consumed_lines]) + corrected = TruncateToken( + kind="file", + location=path, + offset=char_start + raw_consumed, + limit=max_chars, + numbered=True, + ) + omitted = len(text) - (char_start + raw_consumed) + body = counted + truncation_marker(max_chars, omitted, corrected) + else: + shown = counted.count("\n") mark_lineno_read(str(target), set(range(start + 1, start + 1 + shown))) return body begin = start + 1 diff --git a/src/plyngent/tools/truncate_token.py b/src/plyngent/tools/truncate_token.py index efc207d..01c73ad 100644 --- a/src/plyngent/tools/truncate_token.py +++ b/src/plyngent/tools/truncate_token.py @@ -37,13 +37,15 @@ class TruncateToken(msgspec.Struct, frozen=True): lines (files is line-based via ``read_file``); ``get_truncated`` interprets by ``kind``. For ``memory``, ``location`` is a key into the in-memory remainder store and ``offset``/``limit`` index characters of that stored - remainder. + remainder. ``numbered`` marks file tokens from ``read_file(with_lineno)``: + the continuation resumes the numbered view instead of raw text. """ kind: TruncateKind location: str offset: int limit: int + numbered: bool = False # In-memory remainders for generic (non-resumable) tool output. Process-lifetime diff --git a/tests/test_tools/test_get_truncated.py b/tests/test_tools/test_get_truncated.py index b446354..b6e6c52 100644 --- a/tests/test_tools/test_get_truncated.py +++ b/tests/test_tools/test_get_truncated.py @@ -44,6 +44,40 @@ async def test_get_truncated_file_chains_without_gap(workspace: Path) -> None: assert total_read == 1000 +async def test_get_truncated_numbered_continues_and_marks(workspace: Path) -> None: + """A numbered read's token resumes the numbered view, gap-free and editable.""" + from plyngent.agent import reset_lineno_tracker + from plyngent.tools.file import edit_lineno, read_file + + (workspace / "big.txt").write_text("".join(f"line {i}\n" for i in range(1, 101)), encoding="utf-8") + reset_lineno_tracker() + first = await call_async(read_file, "big.txt", with_lineno=True, max_chars=120) + token = _extract_token(first) + assert token is not None + parsed = decode_truncate_token(token) + assert parsed is not None + assert parsed.kind == "file" + assert parsed.numbered is True + # The token points at the raw start of the next line (no skip, no overlap). + body = first.split("\n[Truncated", 1)[0] + shown = body.count("\n") + expected_raw = sum(len(f"line {i}\n") for i in range(1, shown + 1)) + assert parsed.offset == expected_raw + + # Continuation is numbered and starts right after the displayed lines. + cont = await call_async(get_truncated, token, max_chars=200) + numbered = [ln for ln in cont.splitlines() if "|" in ln and ln.strip().split("|", 1)[0].strip().isdigit()] + assert numbered + first_num = int(numbered[0].split("|", 1)[0].strip()) + assert first_num == shown + 1 + + # Resumed lines are editable via edit_lineno; unseen lines stay unread. + out = await call_async(edit_lineno, "big.txt", first_num, first_num, "EDITED\n") + assert "replaced lines" in out + out2 = await call_async(edit_lineno, "big.txt", 90, 90, "X\n") + assert "not read" in out2 + + async def test_get_truncated_file_missing(workspace: Path) -> None: token = encode_truncate_token(TruncateToken(kind="file", location="nope.txt", offset=0, limit=200)) out = await call_async(get_truncated, token, max_chars=200)