tools/truncate_token: scope the token cursor to file reads and mentions

Tool results no longer use a truncate token, so the shared `truncate_generic`
wrapper goes. `read_file` keeps its file token, and a user `@path` mention
keeps the memory token (its inlined text has no re-readable source).

Co-authored-by: CommandCodeBot <noreply@commandcode.ai>

tests/test_tools/test_get_truncated.py

@@ -85,15 +85,15 @@ async def test_get_truncated_file_missing(workspace: Path) -> None:
 
 
 async def test_get_truncated_memory_chains() -> None:
-    from plyngent.tools.truncate_token import truncate_generic
+    from plyngent.tools.truncate_token import truncate_to_memory
 
-    bounded = truncate_generic("m" * 1000, 200)
+    bounded, _token = truncate_to_memory("m" * 1000, 200)
     assert "truncate_token=" in bounded
     token = bounded.split("truncate_token=", 1)[1].rstrip("]")
     parsed = decode_truncate_token(token)
     assert parsed is not None
     assert parsed.kind == "memory"
-    cursor = parsed.offset  # chars already delivered by truncate_generic
+    cursor = parsed.offset  # chars already delivered by truncate_to_memory
     delivered = cursor
     for _ in range(20):
         out = await call_async(get_truncated, token, max_chars=200)

tests/test_tools/test_truncate_token.py

@@ -5,7 +5,7 @@ from plyngent.tools.truncate_token import (
     decode_truncate_token,
     encode_truncate_token,
     get_remainder,
-    truncate_generic,
+    truncate_to_memory,
     truncate_with_token,
     truncation_marker,
 )
@@ -125,26 +125,26 @@ def test_truncate_with_token_nonzero_start() -> None:
     assert len(out) <= 200
 
 
-def test_truncate_generic_short_untouched() -> None:
-    assert truncate_generic("tiny", 200) == "tiny"
+def test_truncate_to_memory_short_untouched() -> None:
+    out, token = truncate_to_memory("tiny", 200)
+    assert out == "tiny"
+    assert token is None
 
 
-def test_truncate_generic_embeds_memory_token() -> None:
+def test_truncate_to_memory_embeds_memory_token() -> None:
     text = "a" * 500
-    out = truncate_generic(text, 200)
+    out, token = truncate_to_memory(text, 200)
+    assert token is not None
     assert len(out) <= 200
     assert "[Truncated (200 chars max." in out
-    token_str = out.split("truncate_token=", 1)[1].rstrip("]")
-    token = decode_truncate_token(token_str)
-    assert token is not None
     assert token.kind == "memory"
     # Remainder is the full original text; token offset points into it.
-    remainder = get_remainder(token.location)
-    assert remainder == text
+    assert get_remainder(token.location) == text
     assert token.offset < len(text)
 
 
-def test_truncate_generic_no_marker_when_fits() -> None:
-    out = truncate_generic("a" * 50, 200)
+def test_truncate_to_memory_no_marker_when_fits() -> None:
+    out, token = truncate_to_memory("a" * 50, 200)
     assert out == "a" * 50
     assert "Truncated" not in out
+    assert token is None