NCBM/plyngent
1
0
Fork 0
mirror of https://github.com/NCBM/plyngent.git synced 2026-11-07 15:37:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

tools/truncate_token: scope the token cursor to file reads and mentions

Browse Source 
Tool results no longer use a truncate token, so the shared `truncate_generic`
wrapper goes. `read_file` keeps its file token, and a user `@path` mention
keeps the memory token (its inlined text has no re-readable source).

Co-authored-by: CommandCodeBot <noreply@commandcode.ai>
This commit is contained in:
worldmozara
2026-10-29 14:16:40 +08:00
parent 7d6e367143
commit 140268f983
4 changed files with 35 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_tools/test_get_truncated.py
+3 -3
View File
@@ -85,15 +85,15 @@ async def test_get_truncated_file_missing(workspace: Path) -> None:
async def test_get_truncated_memory_chains() -> None:
from plyngent.tools.truncate_token import truncate_generic
from plyngent.tools.truncate_token import truncate_to_memory
bounded = truncate_generic("m" * 1000, 200)
bounded, _token = truncate_to_memory("m" * 1000, 200)
assert "truncate_token=" in bounded
token = bounded.split("truncate_token=", 1)[1].rstrip("]")
parsed = decode_truncate_token(token)
assert parsed is not None
assert parsed.kind == "memory"
cursor = parsed.offset # chars already delivered by truncate_generic
cursor = parsed.offset # chars already delivered by truncate_to_memory
delivered = cursor
for _ in range(20):
out = await call_async(get_truncated, token, max_chars=200)