mirror of
https://github.com/NCBM/plyngent.git
synced 2026-11-07 23:47:54 +08:00
core/tools: numbered read continuation via get_truncated
Two fixes for the truncated numbered-read flow: - read_file(with_lineno) tokens were rebased onto numbered-body chars (each line carries an N| prefix), so get_truncated over-advanced into raw text and silently skipped lines. Tokens now advance by raw chars of the complete lines shown and carry a numbered flag. - get_truncated resumes numbered tokens as numbered lines (via read_file with_lineno), marking them readable for edit_lineno, so read -> get_truncated -> edit_lineno works end-to-end without gaps.
This commit is contained in:
@@ -6,15 +6,56 @@ from typing import cast
|
|||||||
|
|
||||||
from plyngent.agent import ToolTag, tool
|
from plyngent.agent import ToolTag, tool
|
||||||
from plyngent.agent.budget import DEFAULT_TOOL_RESULT_MAX_CHARS
|
from plyngent.agent.budget import DEFAULT_TOOL_RESULT_MAX_CHARS
|
||||||
from plyngent.tools.file.read import line_range_label, read_raw_text
|
from plyngent.tools.file.read import line_range_label, read_file, read_raw_text
|
||||||
from plyngent.tools.net.fetch import fetch
|
from plyngent.tools.net.fetch import fetch
|
||||||
from plyngent.tools.truncate_token import (
|
from plyngent.tools.truncate_token import (
|
||||||
|
TruncateToken,
|
||||||
decode_truncate_token,
|
decode_truncate_token,
|
||||||
get_remainder,
|
get_remainder,
|
||||||
truncate_with_token,
|
truncate_with_token,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _memory_chunk(parsed: TruncateToken, max_chars: int) -> str:
|
||||||
|
"""Serve one chunk from the in-memory remainder store (or an error)."""
|
||||||
|
remainder = get_remainder(parsed.location)
|
||||||
|
if remainder is None:
|
||||||
|
return "error: truncate token expired (no longer in memory); re-run the tool"
|
||||||
|
start = parsed.offset
|
||||||
|
segment = remainder[start : start + max_chars]
|
||||||
|
chunk, _ = truncate_with_token(
|
||||||
|
segment,
|
||||||
|
max_chars,
|
||||||
|
kind="memory",
|
||||||
|
location=parsed.location,
|
||||||
|
offset=start,
|
||||||
|
limit=max_chars,
|
||||||
|
total_len=len(remainder),
|
||||||
|
)
|
||||||
|
return chunk
|
||||||
|
|
||||||
|
|
||||||
|
async def _numbered_continuation(text: str, location: str, char_offset: int, max_chars: int) -> str:
|
||||||
|
"""Resume a numbered read at the line containing *char_offset*.
|
||||||
|
|
||||||
|
Delegates to ``read_file(with_lineno=True)`` so the continuation carries
|
||||||
|
``N|`` line numbers and marks the resumed lines readable for ``edit_lineno``.
|
||||||
|
"""
|
||||||
|
pos = 0
|
||||||
|
line = 0
|
||||||
|
for ln in text.splitlines(keepends=True):
|
||||||
|
if pos + len(ln) > char_offset:
|
||||||
|
break
|
||||||
|
pos += len(ln)
|
||||||
|
line += 1
|
||||||
|
return await read_file.handler(
|
||||||
|
location,
|
||||||
|
offset=line,
|
||||||
|
with_lineno=True,
|
||||||
|
max_chars=max_chars,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@tool(tags=ToolTag.LOCAL | ToolTag.INSTANCE_STATE | ToolTag.READ_ONLY)
|
@tool(tags=ToolTag.LOCAL | ToolTag.INSTANCE_STATE | ToolTag.READ_ONLY)
|
||||||
async def get_truncated(token: str, *, max_chars: int = DEFAULT_TOOL_RESULT_MAX_CHARS) -> str:
|
async def get_truncated(token: str, *, max_chars: int = DEFAULT_TOOL_RESULT_MAX_CHARS) -> str:
|
||||||
"""Fetch the next chunk of a truncated result using its ``truncate_token``.
|
"""Fetch the next chunk of a truncated result using its ``truncate_token``.
|
||||||
@@ -22,10 +63,11 @@ async def get_truncated(token: str, *, max_chars: int = DEFAULT_TOOL_RESULT_MAX_
|
|||||||
Pass the token shown at the end of a truncated ``read_file``, ``fetch``,
|
Pass the token shown at the end of a truncated ``read_file``, ``fetch``,
|
||||||
``run_command``, or earlier ``get_truncated`` result to keep reading without
|
``run_command``, or earlier ``get_truncated`` result to keep reading without
|
||||||
re-requesting the whole source or raising limits. Truncated output carries a
|
re-requesting the whole source or raising limits. Truncated output carries a
|
||||||
fresh token, so chunks chain indefinitely. File chunks start with a 1-based
|
fresh token, so chunks chain indefinitely. File chunks resume in the same
|
||||||
``L{begin}-{end}`` line range like ``read_file``; memory chunks (generic
|
view as the original read: numbered reads continue as numbered lines
|
||||||
tool output) are served from a short-lived in-memory store that is forgotten
|
(editable via ``edit_lineno``), plain reads as raw text with a 1-based
|
||||||
when the agent exits.
|
``L{begin}-{end}`` range. Memory chunks (generic tool output) are served
|
||||||
|
from a short-lived in-memory store that is forgotten when the agent exits.
|
||||||
"""
|
"""
|
||||||
parsed = decode_truncate_token(token)
|
parsed = decode_truncate_token(token)
|
||||||
if parsed is None:
|
if parsed is None:
|
||||||
@@ -33,26 +75,14 @@ async def get_truncated(token: str, *, max_chars: int = DEFAULT_TOOL_RESULT_MAX_
|
|||||||
if parsed.kind == "http":
|
if parsed.kind == "http":
|
||||||
return await fetch.handler(parsed.location, offset=parsed.offset, max_chars=parsed.limit)
|
return await fetch.handler(parsed.location, offset=parsed.offset, max_chars=parsed.limit)
|
||||||
if parsed.kind == "memory":
|
if parsed.kind == "memory":
|
||||||
remainder = get_remainder(parsed.location)
|
return _memory_chunk(parsed, max_chars)
|
||||||
if remainder is None:
|
|
||||||
return "error: truncate token expired (no longer in memory); re-run the tool"
|
|
||||||
start = parsed.offset
|
|
||||||
segment = remainder[start : start + max_chars]
|
|
||||||
chunk, _ = truncate_with_token(
|
|
||||||
segment,
|
|
||||||
max_chars,
|
|
||||||
kind="memory",
|
|
||||||
location=parsed.location,
|
|
||||||
offset=start,
|
|
||||||
limit=max_chars,
|
|
||||||
total_len=len(remainder),
|
|
||||||
)
|
|
||||||
return chunk
|
|
||||||
text, err = read_raw_text(parsed.location)
|
text, err = read_raw_text(parsed.location)
|
||||||
if err:
|
if err:
|
||||||
return err
|
return err
|
||||||
text = cast("str", text)
|
text = cast("str", text)
|
||||||
start = parsed.offset
|
start = parsed.offset
|
||||||
|
if parsed.numbered:
|
||||||
|
return await _numbered_continuation(text, parsed.location, start, max_chars)
|
||||||
segment = text[start : start + max_chars]
|
segment = text[start : start + max_chars]
|
||||||
chunk, _ = truncate_with_token(
|
chunk, _ = truncate_with_token(
|
||||||
segment,
|
segment,
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ from typing import cast
|
|||||||
|
|
||||||
from plyngent.agent import ToolTag, mark_lineno_read, tool
|
from plyngent.agent import ToolTag, mark_lineno_read, tool
|
||||||
from plyngent.agent.budget import DEFAULT_TOOL_RESULT_MAX_CHARS
|
from plyngent.agent.budget import DEFAULT_TOOL_RESULT_MAX_CHARS
|
||||||
from plyngent.tools.truncate_token import truncate_with_token
|
from plyngent.tools.truncate_token import TruncateToken, truncate_with_token, truncation_marker
|
||||||
from plyngent.tools.workspace import resolve_path
|
from plyngent.tools.workspace import resolve_path
|
||||||
|
|
||||||
_LINENO_WIDTH = 6
|
_LINENO_WIDTH = 6
|
||||||
@@ -84,8 +84,8 @@ async def read_file(
|
|||||||
return ""
|
return ""
|
||||||
slice_lines = lines[start:end]
|
slice_lines = lines[start:end]
|
||||||
body = _format_with_lineno(slice_lines, start_lineno=start + 1) if with_lineno else "".join(slice_lines)
|
body = _format_with_lineno(slice_lines, start_lineno=start + 1) if with_lineno else "".join(slice_lines)
|
||||||
|
char_start = len("".join(lines[:start]))
|
||||||
if max_chars >= 1:
|
if max_chars >= 1:
|
||||||
char_start = len("".join(lines[:start]))
|
|
||||||
body, token = truncate_with_token(
|
body, token = truncate_with_token(
|
||||||
body,
|
body,
|
||||||
max_chars,
|
max_chars,
|
||||||
@@ -110,7 +110,25 @@ async def read_file(
|
|||||||
counted = body
|
counted = body
|
||||||
if token is not None:
|
if token is not None:
|
||||||
counted = body.split("\n[Truncated", 1)[0]
|
counted = body.split("\n[Truncated", 1)[0]
|
||||||
shown = counted.count("\n")
|
shown = counted.count("\n")
|
||||||
|
# The truncate cursor counts *numbered* chars (each line carries an
|
||||||
|
# "N|" prefix), which would over-advance into raw text and skip
|
||||||
|
# lines on continuation. Rebase the token onto raw characters of the
|
||||||
|
# complete lines shown (at least one, so the cursor always moves)
|
||||||
|
# and mark it as the numbered view for get_truncated.
|
||||||
|
consumed_lines = max(shown, 1)
|
||||||
|
raw_consumed = sum(len(ln) for ln in slice_lines[:consumed_lines])
|
||||||
|
corrected = TruncateToken(
|
||||||
|
kind="file",
|
||||||
|
location=path,
|
||||||
|
offset=char_start + raw_consumed,
|
||||||
|
limit=max_chars,
|
||||||
|
numbered=True,
|
||||||
|
)
|
||||||
|
omitted = len(text) - (char_start + raw_consumed)
|
||||||
|
body = counted + truncation_marker(max_chars, omitted, corrected)
|
||||||
|
else:
|
||||||
|
shown = counted.count("\n")
|
||||||
mark_lineno_read(str(target), set(range(start + 1, start + 1 + shown)))
|
mark_lineno_read(str(target), set(range(start + 1, start + 1 + shown)))
|
||||||
return body
|
return body
|
||||||
begin = start + 1
|
begin = start + 1
|
||||||
|
|||||||
@@ -37,13 +37,15 @@ class TruncateToken(msgspec.Struct, frozen=True):
|
|||||||
lines (files is line-based via ``read_file``); ``get_truncated`` interprets
|
lines (files is line-based via ``read_file``); ``get_truncated`` interprets
|
||||||
by ``kind``. For ``memory``, ``location`` is a key into the in-memory
|
by ``kind``. For ``memory``, ``location`` is a key into the in-memory
|
||||||
remainder store and ``offset``/``limit`` index characters of that stored
|
remainder store and ``offset``/``limit`` index characters of that stored
|
||||||
remainder.
|
remainder. ``numbered`` marks file tokens from ``read_file(with_lineno)``:
|
||||||
|
the continuation resumes the numbered view instead of raw text.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
kind: TruncateKind
|
kind: TruncateKind
|
||||||
location: str
|
location: str
|
||||||
offset: int
|
offset: int
|
||||||
limit: int
|
limit: int
|
||||||
|
numbered: bool = False
|
||||||
|
|
||||||
|
|
||||||
# In-memory remainders for generic (non-resumable) tool output. Process-lifetime
|
# In-memory remainders for generic (non-resumable) tool output. Process-lifetime
|
||||||
|
|||||||
@@ -44,6 +44,40 @@ async def test_get_truncated_file_chains_without_gap(workspace: Path) -> None:
|
|||||||
assert total_read == 1000
|
assert total_read == 1000
|
||||||
|
|
||||||
|
|
||||||
|
async def test_get_truncated_numbered_continues_and_marks(workspace: Path) -> None:
|
||||||
|
"""A numbered read's token resumes the numbered view, gap-free and editable."""
|
||||||
|
from plyngent.agent import reset_lineno_tracker
|
||||||
|
from plyngent.tools.file import edit_lineno, read_file
|
||||||
|
|
||||||
|
(workspace / "big.txt").write_text("".join(f"line {i}\n" for i in range(1, 101)), encoding="utf-8")
|
||||||
|
reset_lineno_tracker()
|
||||||
|
first = await call_async(read_file, "big.txt", with_lineno=True, max_chars=120)
|
||||||
|
token = _extract_token(first)
|
||||||
|
assert token is not None
|
||||||
|
parsed = decode_truncate_token(token)
|
||||||
|
assert parsed is not None
|
||||||
|
assert parsed.kind == "file"
|
||||||
|
assert parsed.numbered is True
|
||||||
|
# The token points at the raw start of the next line (no skip, no overlap).
|
||||||
|
body = first.split("\n[Truncated", 1)[0]
|
||||||
|
shown = body.count("\n")
|
||||||
|
expected_raw = sum(len(f"line {i}\n") for i in range(1, shown + 1))
|
||||||
|
assert parsed.offset == expected_raw
|
||||||
|
|
||||||
|
# Continuation is numbered and starts right after the displayed lines.
|
||||||
|
cont = await call_async(get_truncated, token, max_chars=200)
|
||||||
|
numbered = [ln for ln in cont.splitlines() if "|" in ln and ln.strip().split("|", 1)[0].strip().isdigit()]
|
||||||
|
assert numbered
|
||||||
|
first_num = int(numbered[0].split("|", 1)[0].strip())
|
||||||
|
assert first_num == shown + 1
|
||||||
|
|
||||||
|
# Resumed lines are editable via edit_lineno; unseen lines stay unread.
|
||||||
|
out = await call_async(edit_lineno, "big.txt", first_num, first_num, "EDITED\n")
|
||||||
|
assert "replaced lines" in out
|
||||||
|
out2 = await call_async(edit_lineno, "big.txt", 90, 90, "X\n")
|
||||||
|
assert "not read" in out2
|
||||||
|
|
||||||
|
|
||||||
async def test_get_truncated_file_missing(workspace: Path) -> None:
|
async def test_get_truncated_file_missing(workspace: Path) -> None:
|
||||||
token = encode_truncate_token(TruncateToken(kind="file", location="nope.txt", offset=0, limit=200))
|
token = encode_truncate_token(TruncateToken(kind="file", location="nope.txt", offset=0, limit=200))
|
||||||
out = await call_async(get_truncated, token, max_chars=200)
|
out = await call_async(get_truncated, token, max_chars=200)
|
||||||
|
|||||||
Reference in New Issue
Block a user