NCBM/plyngent
1
0
Fork 0
mirror of https://github.com/NCBM/plyngent.git synced 2026-11-07 23:47:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

core/tools: make truncate tokens gap-free (advance by returned length)

Browse Source
This commit is contained in:
worldmozara
2026-08-11 15:16:55 +08:00
parent 4c60e9b1b3
commit 9640f86b89
1 changed files with 26 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/plyngent/tools/truncate_token.py
+26 -15
View File
@@ -46,6 +46,13 @@ def decode_truncate_token(text: str) -> TruncateToken | None:
return token return token
def _truncated_marker(omitted: int, token: TruncateToken) -> str:
return (
f"\n...[truncated {omitted} chars; more via get_truncated token] "
f"[TRUNCATE_TOKEN: {encode_truncate_token(token)}]"
)
def truncate_with_token( def truncate_with_token(
text: str, text: str,
max_chars: int, max_chars: int,
@@ -61,25 +68,29 @@ def truncate_with_token(
``text`` is a slice of the full source starting at ``offset``; ``total_len`` ``text`` is a slice of the full source starting at ``offset``; ``total_len``
is the full source length. A token is embedded whenever more content remains is the full source length. A token is embedded whenever more content remains
after the returned chunk, so ``get_truncated`` chains through truncations after the returned chunk, so ``get_truncated`` chains through truncations
and each chunk may itself carry a fresh token. The marker is budgeted so the and each chunk may itself carry a fresh token. The token advances by the
total returned length stays at or under ``max_chars`` (the agent loop's own length actually returned (no content is skipped), and the marker is budgeted
so the total returned length stays near ``max_chars`` (the agent loop's own
cap never re-cuts it and never strips the token). cap never re-cuts it and never strips the token).
""" """
if max_chars < 1: if max_chars < 1:
return text, None return text, None
overflow = len(text) > max_chars if len(text) > max_chars:
more_after = offset + max_chars < total_len
if not overflow and not more_after:
return text, None
token = TruncateToken(kind=kind, location=location, offset=offset + max_chars, limit=limit)
tok_text = encode_truncate_token(token)
if overflow:
omitted = len(text) - max_chars omitted = len(text) - max_chars
marker = f"\n...[truncated {omitted} chars; more via get_truncated token] [TRUNCATE_TOKEN: {tok_text}]" # First pass sizes the marker with an offset+max_chars token; re-encode
cut = max_chars - len(marker) # with the real cut so the cursor lands exactly where content stops.
first = TruncateToken(kind=kind, location=location, offset=offset + max_chars, limit=limit)
cut = max_chars - len(_truncated_marker(omitted, first))
if cut < 1: if cut < 1:
# Marker longer than the budget: emit anyway so chaining never drops. return text[:max_chars] + _truncated_marker(omitted, first), first
return text[:max_chars] + marker, token token = TruncateToken(kind=kind, location=location, offset=offset + cut, limit=limit)
return text[:cut] + marker, token return text[:cut] + _truncated_marker(omitted, token), token
marker = f"\n...[more content available; use get_truncated with the token] [TRUNCATE_TOKEN: {tok_text}]" more_after = offset + len(text) < total_len
if not more_after:
return text, None
token = TruncateToken(kind=kind, location=location, offset=offset + len(text), limit=limit)
marker = (
f"\n...[more content available; use get_truncated with the token] "
f"[TRUNCATE_TOKEN: {encode_truncate_token(token)}]"
)
return text + marker, token return text + marker, token