mirror of
https://github.com/NCBM/plyngent.git
synced 2026-11-07 15:37:55 +08:00
core/tools: numbered read continuation via get_truncated
Two fixes for the truncated numbered-read flow: - read_file(with_lineno) tokens were rebased onto numbered-body chars (each line carries an N| prefix), so get_truncated over-advanced into raw text and silently skipped lines. Tokens now advance by raw chars of the complete lines shown and carry a numbered flag. - get_truncated resumes numbered tokens as numbered lines (via read_file with_lineno), marking them readable for edit_lineno, so read -> get_truncated -> edit_lineno works end-to-end without gaps.
This commit is contained in:
@@ -6,15 +6,56 @@ from typing import cast
|
||||
|
||||
from plyngent.agent import ToolTag, tool
|
||||
from plyngent.agent.budget import DEFAULT_TOOL_RESULT_MAX_CHARS
|
||||
from plyngent.tools.file.read import line_range_label, read_raw_text
|
||||
from plyngent.tools.file.read import line_range_label, read_file, read_raw_text
|
||||
from plyngent.tools.net.fetch import fetch
|
||||
from plyngent.tools.truncate_token import (
|
||||
TruncateToken,
|
||||
decode_truncate_token,
|
||||
get_remainder,
|
||||
truncate_with_token,
|
||||
)
|
||||
|
||||
|
||||
def _memory_chunk(parsed: TruncateToken, max_chars: int) -> str:
|
||||
"""Serve one chunk from the in-memory remainder store (or an error)."""
|
||||
remainder = get_remainder(parsed.location)
|
||||
if remainder is None:
|
||||
return "error: truncate token expired (no longer in memory); re-run the tool"
|
||||
start = parsed.offset
|
||||
segment = remainder[start : start + max_chars]
|
||||
chunk, _ = truncate_with_token(
|
||||
segment,
|
||||
max_chars,
|
||||
kind="memory",
|
||||
location=parsed.location,
|
||||
offset=start,
|
||||
limit=max_chars,
|
||||
total_len=len(remainder),
|
||||
)
|
||||
return chunk
|
||||
|
||||
|
||||
async def _numbered_continuation(text: str, location: str, char_offset: int, max_chars: int) -> str:
|
||||
"""Resume a numbered read at the line containing *char_offset*.
|
||||
|
||||
Delegates to ``read_file(with_lineno=True)`` so the continuation carries
|
||||
``N|`` line numbers and marks the resumed lines readable for ``edit_lineno``.
|
||||
"""
|
||||
pos = 0
|
||||
line = 0
|
||||
for ln in text.splitlines(keepends=True):
|
||||
if pos + len(ln) > char_offset:
|
||||
break
|
||||
pos += len(ln)
|
||||
line += 1
|
||||
return await read_file.handler(
|
||||
location,
|
||||
offset=line,
|
||||
with_lineno=True,
|
||||
max_chars=max_chars,
|
||||
)
|
||||
|
||||
|
||||
@tool(tags=ToolTag.LOCAL | ToolTag.INSTANCE_STATE | ToolTag.READ_ONLY)
|
||||
async def get_truncated(token: str, *, max_chars: int = DEFAULT_TOOL_RESULT_MAX_CHARS) -> str:
|
||||
"""Fetch the next chunk of a truncated result using its ``truncate_token``.
|
||||
@@ -22,10 +63,11 @@ async def get_truncated(token: str, *, max_chars: int = DEFAULT_TOOL_RESULT_MAX_
|
||||
Pass the token shown at the end of a truncated ``read_file``, ``fetch``,
|
||||
``run_command``, or earlier ``get_truncated`` result to keep reading without
|
||||
re-requesting the whole source or raising limits. Truncated output carries a
|
||||
fresh token, so chunks chain indefinitely. File chunks start with a 1-based
|
||||
``L{begin}-{end}`` line range like ``read_file``; memory chunks (generic
|
||||
tool output) are served from a short-lived in-memory store that is forgotten
|
||||
when the agent exits.
|
||||
fresh token, so chunks chain indefinitely. File chunks resume in the same
|
||||
view as the original read: numbered reads continue as numbered lines
|
||||
(editable via ``edit_lineno``), plain reads as raw text with a 1-based
|
||||
``L{begin}-{end}`` range. Memory chunks (generic tool output) are served
|
||||
from a short-lived in-memory store that is forgotten when the agent exits.
|
||||
"""
|
||||
parsed = decode_truncate_token(token)
|
||||
if parsed is None:
|
||||
@@ -33,26 +75,14 @@ async def get_truncated(token: str, *, max_chars: int = DEFAULT_TOOL_RESULT_MAX_
|
||||
if parsed.kind == "http":
|
||||
return await fetch.handler(parsed.location, offset=parsed.offset, max_chars=parsed.limit)
|
||||
if parsed.kind == "memory":
|
||||
remainder = get_remainder(parsed.location)
|
||||
if remainder is None:
|
||||
return "error: truncate token expired (no longer in memory); re-run the tool"
|
||||
start = parsed.offset
|
||||
segment = remainder[start : start + max_chars]
|
||||
chunk, _ = truncate_with_token(
|
||||
segment,
|
||||
max_chars,
|
||||
kind="memory",
|
||||
location=parsed.location,
|
||||
offset=start,
|
||||
limit=max_chars,
|
||||
total_len=len(remainder),
|
||||
)
|
||||
return chunk
|
||||
return _memory_chunk(parsed, max_chars)
|
||||
text, err = read_raw_text(parsed.location)
|
||||
if err:
|
||||
return err
|
||||
text = cast("str", text)
|
||||
start = parsed.offset
|
||||
if parsed.numbered:
|
||||
return await _numbered_continuation(text, parsed.location, start, max_chars)
|
||||
segment = text[start : start + max_chars]
|
||||
chunk, _ = truncate_with_token(
|
||||
segment,
|
||||
|
||||
@@ -4,7 +4,7 @@ from typing import cast
|
||||
|
||||
from plyngent.agent import ToolTag, mark_lineno_read, tool
|
||||
from plyngent.agent.budget import DEFAULT_TOOL_RESULT_MAX_CHARS
|
||||
from plyngent.tools.truncate_token import truncate_with_token
|
||||
from plyngent.tools.truncate_token import TruncateToken, truncate_with_token, truncation_marker
|
||||
from plyngent.tools.workspace import resolve_path
|
||||
|
||||
_LINENO_WIDTH = 6
|
||||
@@ -84,8 +84,8 @@ async def read_file(
|
||||
return ""
|
||||
slice_lines = lines[start:end]
|
||||
body = _format_with_lineno(slice_lines, start_lineno=start + 1) if with_lineno else "".join(slice_lines)
|
||||
char_start = len("".join(lines[:start]))
|
||||
if max_chars >= 1:
|
||||
char_start = len("".join(lines[:start]))
|
||||
body, token = truncate_with_token(
|
||||
body,
|
||||
max_chars,
|
||||
@@ -110,7 +110,25 @@ async def read_file(
|
||||
counted = body
|
||||
if token is not None:
|
||||
counted = body.split("\n[Truncated", 1)[0]
|
||||
shown = counted.count("\n")
|
||||
shown = counted.count("\n")
|
||||
# The truncate cursor counts *numbered* chars (each line carries an
|
||||
# "N|" prefix), which would over-advance into raw text and skip
|
||||
# lines on continuation. Rebase the token onto raw characters of the
|
||||
# complete lines shown (at least one, so the cursor always moves)
|
||||
# and mark it as the numbered view for get_truncated.
|
||||
consumed_lines = max(shown, 1)
|
||||
raw_consumed = sum(len(ln) for ln in slice_lines[:consumed_lines])
|
||||
corrected = TruncateToken(
|
||||
kind="file",
|
||||
location=path,
|
||||
offset=char_start + raw_consumed,
|
||||
limit=max_chars,
|
||||
numbered=True,
|
||||
)
|
||||
omitted = len(text) - (char_start + raw_consumed)
|
||||
body = counted + truncation_marker(max_chars, omitted, corrected)
|
||||
else:
|
||||
shown = counted.count("\n")
|
||||
mark_lineno_read(str(target), set(range(start + 1, start + 1 + shown)))
|
||||
return body
|
||||
begin = start + 1
|
||||
|
||||
@@ -37,13 +37,15 @@ class TruncateToken(msgspec.Struct, frozen=True):
|
||||
lines (files is line-based via ``read_file``); ``get_truncated`` interprets
|
||||
by ``kind``. For ``memory``, ``location`` is a key into the in-memory
|
||||
remainder store and ``offset``/``limit`` index characters of that stored
|
||||
remainder.
|
||||
remainder. ``numbered`` marks file tokens from ``read_file(with_lineno)``:
|
||||
the continuation resumes the numbered view instead of raw text.
|
||||
"""
|
||||
|
||||
kind: TruncateKind
|
||||
location: str
|
||||
offset: int
|
||||
limit: int
|
||||
numbered: bool = False
|
||||
|
||||
|
||||
# In-memory remainders for generic (non-resumable) tool output. Process-lifetime
|
||||
|
||||
Reference in New Issue
Block a user