core/tools: numbered read continuation via get_truncated

Two fixes for the truncated numbered-read flow:

- read_file(with_lineno) tokens were rebased onto numbered-body chars
  (each line carries an N| prefix), so get_truncated over-advanced into
  raw text and silently skipped lines. Tokens now advance by raw chars
  of the complete lines shown and carry a numbered flag.
- get_truncated resumes numbered tokens as numbered lines (via
  read_file with_lineno), marking them readable for edit_lineno, so
  read -> get_truncated -> edit_lineno works end-to-end without gaps.
This commit is contained in:
2026-08-15 17:35:57 +08:00
parent 132ac520ec
commit d914590677
4 changed files with 108 additions and 24 deletions
+50 -20
View File
@@ -6,15 +6,56 @@ from typing import cast
from plyngent.agent import ToolTag, tool
from plyngent.agent.budget import DEFAULT_TOOL_RESULT_MAX_CHARS
from plyngent.tools.file.read import line_range_label, read_raw_text
from plyngent.tools.file.read import line_range_label, read_file, read_raw_text
from plyngent.tools.net.fetch import fetch
from plyngent.tools.truncate_token import (
TruncateToken,
decode_truncate_token,
get_remainder,
truncate_with_token,
)
def _memory_chunk(parsed: TruncateToken, max_chars: int) -> str:
"""Serve one chunk from the in-memory remainder store (or an error)."""
remainder = get_remainder(parsed.location)
if remainder is None:
return "error: truncate token expired (no longer in memory); re-run the tool"
start = parsed.offset
segment = remainder[start : start + max_chars]
chunk, _ = truncate_with_token(
segment,
max_chars,
kind="memory",
location=parsed.location,
offset=start,
limit=max_chars,
total_len=len(remainder),
)
return chunk
async def _numbered_continuation(text: str, location: str, char_offset: int, max_chars: int) -> str:
"""Resume a numbered read at the line containing *char_offset*.
Delegates to ``read_file(with_lineno=True)`` so the continuation carries
``N|`` line numbers and marks the resumed lines readable for ``edit_lineno``.
"""
pos = 0
line = 0
for ln in text.splitlines(keepends=True):
if pos + len(ln) > char_offset:
break
pos += len(ln)
line += 1
return await read_file.handler(
location,
offset=line,
with_lineno=True,
max_chars=max_chars,
)
@tool(tags=ToolTag.LOCAL | ToolTag.INSTANCE_STATE | ToolTag.READ_ONLY)
async def get_truncated(token: str, *, max_chars: int = DEFAULT_TOOL_RESULT_MAX_CHARS) -> str:
"""Fetch the next chunk of a truncated result using its ``truncate_token``.
@@ -22,10 +63,11 @@ async def get_truncated(token: str, *, max_chars: int = DEFAULT_TOOL_RESULT_MAX_
Pass the token shown at the end of a truncated ``read_file``, ``fetch``,
``run_command``, or earlier ``get_truncated`` result to keep reading without
re-requesting the whole source or raising limits. Truncated output carries a
fresh token, so chunks chain indefinitely. File chunks start with a 1-based
``L{begin}-{end}`` line range like ``read_file``; memory chunks (generic
tool output) are served from a short-lived in-memory store that is forgotten
when the agent exits.
fresh token, so chunks chain indefinitely. File chunks resume in the same
view as the original read: numbered reads continue as numbered lines
(editable via ``edit_lineno``), plain reads as raw text with a 1-based
``L{begin}-{end}`` range. Memory chunks (generic tool output) are served
from a short-lived in-memory store that is forgotten when the agent exits.
"""
parsed = decode_truncate_token(token)
if parsed is None:
@@ -33,26 +75,14 @@ async def get_truncated(token: str, *, max_chars: int = DEFAULT_TOOL_RESULT_MAX_
if parsed.kind == "http":
return await fetch.handler(parsed.location, offset=parsed.offset, max_chars=parsed.limit)
if parsed.kind == "memory":
remainder = get_remainder(parsed.location)
if remainder is None:
return "error: truncate token expired (no longer in memory); re-run the tool"
start = parsed.offset
segment = remainder[start : start + max_chars]
chunk, _ = truncate_with_token(
segment,
max_chars,
kind="memory",
location=parsed.location,
offset=start,
limit=max_chars,
total_len=len(remainder),
)
return chunk
return _memory_chunk(parsed, max_chars)
text, err = read_raw_text(parsed.location)
if err:
return err
text = cast("str", text)
start = parsed.offset
if parsed.numbered:
return await _numbered_continuation(text, parsed.location, start, max_chars)
segment = text[start : start + max_chars]
chunk, _ = truncate_with_token(
segment,
+21 -3
View File
@@ -4,7 +4,7 @@ from typing import cast
from plyngent.agent import ToolTag, mark_lineno_read, tool
from plyngent.agent.budget import DEFAULT_TOOL_RESULT_MAX_CHARS
from plyngent.tools.truncate_token import truncate_with_token
from plyngent.tools.truncate_token import TruncateToken, truncate_with_token, truncation_marker
from plyngent.tools.workspace import resolve_path
_LINENO_WIDTH = 6
@@ -84,8 +84,8 @@ async def read_file(
return ""
slice_lines = lines[start:end]
body = _format_with_lineno(slice_lines, start_lineno=start + 1) if with_lineno else "".join(slice_lines)
char_start = len("".join(lines[:start]))
if max_chars >= 1:
char_start = len("".join(lines[:start]))
body, token = truncate_with_token(
body,
max_chars,
@@ -110,7 +110,25 @@ async def read_file(
counted = body
if token is not None:
counted = body.split("\n[Truncated", 1)[0]
shown = counted.count("\n")
shown = counted.count("\n")
# The truncate cursor counts *numbered* chars (each line carries an
# "N|" prefix), which would over-advance into raw text and skip
# lines on continuation. Rebase the token onto raw characters of the
# complete lines shown (at least one, so the cursor always moves)
# and mark it as the numbered view for get_truncated.
consumed_lines = max(shown, 1)
raw_consumed = sum(len(ln) for ln in slice_lines[:consumed_lines])
corrected = TruncateToken(
kind="file",
location=path,
offset=char_start + raw_consumed,
limit=max_chars,
numbered=True,
)
omitted = len(text) - (char_start + raw_consumed)
body = counted + truncation_marker(max_chars, omitted, corrected)
else:
shown = counted.count("\n")
mark_lineno_read(str(target), set(range(start + 1, start + 1 + shown)))
return body
begin = start + 1
+3 -1
View File
@@ -37,13 +37,15 @@ class TruncateToken(msgspec.Struct, frozen=True):
lines (files is line-based via ``read_file``); ``get_truncated`` interprets
by ``kind``. For ``memory``, ``location`` is a key into the in-memory
remainder store and ``offset``/``limit`` index characters of that stored
remainder.
remainder. ``numbered`` marks file tokens from ``read_file(with_lineno)``:
the continuation resumes the numbered view instead of raw text.
"""
kind: TruncateKind
location: str
offset: int
limit: int
numbered: bool = False
# In-memory remainders for generic (non-resumable) tool output. Process-lifetime