tools/truncate_token: scope the token cursor to file reads and mentions

Tool results no longer use a truncate token, so the shared `truncate_generic`
wrapper goes. `read_file` keeps its file token, and a user `@path` mention
keeps the memory token (its inlined text has no re-readable source).

Co-authored-by: CommandCodeBot <noreply@commandcode.ai>

src/plyngent/cli/mentions.py

@@ -15,8 +15,9 @@ the file:
   ``[providers.*.models.*].image_in`` skips it with a note instead of letting
   the provider answer 400.
 - any other file is read as text and appended to the message under a
-  ``--- @path ---`` header, capped like a tool result; the remainder stays
+  ``--- @path ---`` header, capped at the tool-result size; the remainder waits
-  resumable through the ``truncate_token`` in the marker.
+  in memory and the marker's ``truncate_token`` resumes it through
+  ``get_truncated``.
 
 Images and text both keep the typed ``@path`` in the message, so the model can
 tell which attachment is which.

src/plyngent/tools/truncate_token.py

@@ -1,24 +1,21 @@
-"""Truncate-token cursors for resuming truncated tool results.
+"""Truncate-token cursors for resuming truncated reads.
 
-Every truncation site (the agent loop's generic tool-result cap,
+Two sources use a truncate token:
-``run_argv`` / ``run_argv_batch`` per-stream caps, ``read_file``,
-``fetch`` / ``fetch_local``, and request-time compact shrinks) appends the same
-compact marker when output is cut short::
 
-    [Truncated (32000 chars max.; 500 omitted). truncate_token=...]
+- ``read_file`` caps a slice with a ``file`` token; the remainder is re-read from
+  disk by ``location`` + ``offset``, so it survives for as long as the file does
+  (and ``numbered`` resumes the ``N|`` view for ``edit_lineno``).
+- A user ``@path`` mention inlines a file with a ``memory`` token. The mention
+  carries text into a user turn rather than a tool message, so the remainder
+  lives in a short-lived in-memory store that is cleared when the agent process
+  exits.
 
-The model passes the token to ``get_truncated`` to read the next chunk without
+Tool results no longer use this module. A result is stored whole in its own
-re-requesting the whole source or raising limits. Tokens chain: each truncated
+message, and the request-prep pass trims the request copy with a marker naming a
-chunk carries a fresh token, so ``get_truncated`` keeps resuming until the
+document (``read_document`` / ``search_document``; see ``tools/documents.py``).
-source is exhausted (then no marker is emitted).
 
-A ``file`` remainder is re-read from disk by ``location`` + ``offset``, so it
+Either way the model passes the token to ``get_truncated`` to read the next
-survives for as long as the file does. Everything else — arbitrary tool output
+chunk without re-requesting the whole source.
-(run_argv stdout, todo renders, …) and ``fetch`` / ``fetch_local`` bodies — have
-no source worth re-running (a re-fetch costs a round trip and re-hits the remote),
-so its remainder lives in a short-lived in-memory store (``kind="memory"``) that is
-forgotten when the agent process exits; the model must re-run the tool after
-that.
 """
 
 from __future__ import annotations
@@ -141,7 +138,9 @@ def truncate_to_memory(text: str, max_chars: int) -> tuple[str, TruncateToken |
 
     Nothing is stored when the text fits, so a caller that only needs the
     bounded text does not pay for a remainder. Returns ``(bounded_text, token)``
-    where ``token`` is ``None`` when the source is exhausted.
+    where ``token`` is ``None`` when the source is exhausted. A user ``@path``
+    mention is the caller: its inlined text has no re-readable source, so the
+    remainder waits in memory until the process exits.
     """
     if max_chars < 1 or len(text) <= max_chars:
         return text, None
@@ -155,14 +154,3 @@ def truncate_to_memory(text: str, max_chars: int) -> tuple[str, TruncateToken |
         limit=max_chars,
         total_len=len(text),
     )
-
-
-def truncate_generic(text: str, max_chars: int) -> str:
-    """Cap arbitrary tool output; embed a memory truncate token (chainable).
-
-    The full remainder is kept in the in-memory store so ``get_truncated`` can
-    resume it. Returns ``text`` unchanged (no marker) when it fits within
-    ``max_chars``.
-    """
-    bounded, _ = truncate_to_memory(text, max_chars)
-    return bounded

tests/test_tools/test_get_truncated.py

@@ -85,15 +85,15 @@ async def test_get_truncated_file_missing(workspace: Path) -> None:
 
 
 async def test_get_truncated_memory_chains() -> None:
-    from plyngent.tools.truncate_token import truncate_generic
+    from plyngent.tools.truncate_token import truncate_to_memory
 
-    bounded = truncate_generic("m" * 1000, 200)
+    bounded, _token = truncate_to_memory("m" * 1000, 200)
     assert "truncate_token=" in bounded
     token = bounded.split("truncate_token=", 1)[1].rstrip("]")
     parsed = decode_truncate_token(token)
     assert parsed is not None
     assert parsed.kind == "memory"
-    cursor = parsed.offset  # chars already delivered by truncate_generic
+    cursor = parsed.offset  # chars already delivered by truncate_to_memory
     delivered = cursor
     for _ in range(20):
         out = await call_async(get_truncated, token, max_chars=200)

tests/test_tools/test_truncate_token.py

@@ -5,7 +5,7 @@ from plyngent.tools.truncate_token import (
     decode_truncate_token,
     encode_truncate_token,
     get_remainder,
-    truncate_generic,
+    truncate_to_memory,
     truncate_with_token,
     truncation_marker,
 )
@@ -125,26 +125,26 @@ def test_truncate_with_token_nonzero_start() -> None:
     assert len(out) <= 200
 
 
-def test_truncate_generic_short_untouched() -> None:
+def test_truncate_to_memory_short_untouched() -> None:
-    assert truncate_generic("tiny", 200) == "tiny"
+    out, token = truncate_to_memory("tiny", 200)
+    assert out == "tiny"
+    assert token is None
 
 
-def test_truncate_generic_embeds_memory_token() -> None:
+def test_truncate_to_memory_embeds_memory_token() -> None:
     text = "a" * 500
-    out = truncate_generic(text, 200)
+    out, token = truncate_to_memory(text, 200)
+    assert token is not None
     assert len(out) <= 200
     assert "[Truncated (200 chars max." in out
-    token_str = out.split("truncate_token=", 1)[1].rstrip("]")
-    token = decode_truncate_token(token_str)
-    assert token is not None
     assert token.kind == "memory"
     # Remainder is the full original text; token offset points into it.
-    remainder = get_remainder(token.location)
+    assert get_remainder(token.location) == text
-    assert remainder == text
     assert token.offset < len(text)
 
 
-def test_truncate_generic_no_marker_when_fits() -> None:
+def test_truncate_to_memory_no_marker_when_fits() -> None:
-    out = truncate_generic("a" * 50, 200)
+    out, token = truncate_to_memory("a" * 50, 200)
     assert out == "a" * 50
     assert "Truncated" not in out
+    assert token is None