core/agent: cap tool results in the request-prep pass

A tool result now stays whole in its message (up to a storage ceiling) and
is trimmed on the request copy by the request-prep pass, where soft compact
already runs. The trimmed copy names the document that holds the rest, so
every surface (chat-completions, Responses, Messages) shares one rule.

New [agent] hard_max_tool_result_chars bounds the stored body; the wire cap
stays [agent] max_tool_result_chars.

Co-authored-by: CommandCodeBot <noreply@commandcode.ai>
This commit is contained in:
2026-10-29 10:04:32 +08:00
parent dfaaad563c
commit 63109ef9f2
12 changed files with 158 additions and 95 deletions
+4
View File
@@ -113,6 +113,10 @@
# system_prompt = "" # persona off; keep default tool_directives unless also cleared
# tool_directives = "" # playbook off (set both "" to disable system entirely)
max_tool_result_chars = 32000
# Storage ceiling for one tool result kept whole in its message. The wire cap
# above trims a result on the request copy and names a document for the rest
# (read_document / search_document); a result above this ceiling is cut on store.
# hard_max_tool_result_chars = 2000000
parallel_tools = true
confirm_destructive = true
path_denylist = ["/secrets/", ".ssh/", ".gnupg/"]
+49 -22
View File
@@ -21,6 +21,11 @@ if TYPE_CHECKING:
type TokenMeasure = Callable[[Sequence[AnyChatMessage]], int]
DEFAULT_TOOL_RESULT_MAX_CHARS = 32_000
# Storage ceiling for one tool result kept whole in its message. A result at or
# below this is stored in full (the request-prep cap trims it on the wire); one
# above it is cut here and carries no document, since its remainder is past what
# a message keeps.
DEFAULT_HARD_MAX_TOOL_RESULT_CHARS = 2_000_000
# Soft context budget in tokens (API-calibrated when possible; else ~4 chars/token).
DEFAULT_CONTEXT_MAX_TOKENS = 200_000
DEFAULT_OLD_TOOL_RESULT_CHARS = 800
@@ -36,26 +41,53 @@ IMAGE_CHARS = 4_096
DEFAULT_CONTEXT_MAX_CHARS = DEFAULT_CONTEXT_MAX_TOKENS * 4
def truncate_tool_result(
text: str,
max_chars: int = DEFAULT_TOOL_RESULT_MAX_CHARS,
) -> str:
"""Cap tool output so huge dumps do not flood model context.
def cap_hard_tool_text(text: str, max_chars: int = DEFAULT_HARD_MAX_TOOL_RESULT_CHARS) -> str:
"""Bound what one tool message stores (the storage ceiling).
This is the generic agent-loop cap for arbitrary tool results. It embeds a
memory truncate token via :func:`truncate_generic` so the model can always
resume the remainder with ``get_truncated`` — resumable or not (remainders
live in the in-memory store for the process lifetime). The compact marker
``[Truncated (N chars max.; M omitted). truncate_token=...]`` is emitted
only when output is actually cut.
The request-prep pass trims a result on the wire and names a document for the
remainder; this ceiling bounds the stored body itself. A result above it is
cut here and carries no document, since its remainder is past what a message
keeps. ``max_chars < 1`` stores the result whole.
"""
if max_chars < 1:
if max_chars < 1 or len(text) <= max_chars:
return text
if len(text) <= max_chars:
return text
from plyngent.tools.truncate_token import truncate_generic
omitted = len(text) - max_chars
return f"{text[:max_chars]}\n[Stored at the {max_chars}-char ceiling: {omitted} chars beyond it]"
return truncate_generic(text, max_chars)
def cap_tool_message(message: ToolChatMessage, max_chars: int) -> ToolChatMessage:
"""Trim one tool result for the wire, naming the document that holds the rest.
The fixed rule of the request-prep pass: a result over ``max_chars`` keeps its
head and gains a marker whose ``document_id`` (the message's ``tool_call_id``)
lets the model read the rest with ``read_document`` / ``search_document``. The
stored message stays whole; only the returned copy is trimmed.
"""
if max_chars < 1 or len(message.content) <= max_chars:
return message
from plyngent.tools.documents import document_marker
total = len(message.content)
head_len = max(1, max_chars - len(document_marker(message.tool_call_id, max_chars, total)))
marker = document_marker(message.tool_call_id, head_len, total)
return msgspec.structs.replace(message, content=message.content[:head_len] + marker)
def truncate_for_request(
messages: Sequence[AnyChatMessage],
*,
max_tool_result_chars: int = DEFAULT_TOOL_RESULT_MAX_CHARS,
) -> list[AnyChatMessage]:
"""The request copy with every tool result trimmed to the fixed wire cap.
Runs in the request-prep pass, after soft compact. Each trimmed result names
its document, so the model reads the rest on demand instead of carrying it in
every request.
"""
return [
cap_tool_message(message, max_tool_result_chars) if isinstance(message, ToolChatMessage) else message
for message in messages
]
def _content_chars(content: str | list[ChatTextPart | ChatImagePart]) -> int:
@@ -131,12 +163,7 @@ def measure_messages_tokens(
def _shrink_tool(message: ToolChatMessage, max_chars: int) -> ToolChatMessage:
if len(message.content) <= max_chars:
return message
return msgspec.structs.replace(
message,
content=truncate_tool_result(message.content, max_chars),
)
return cap_tool_message(message, max_chars)
def _tool_indices(messages: Sequence[AnyChatMessage]) -> list[int]:
+6
View File
@@ -20,6 +20,7 @@ from plyngent.lmproto.openai_compatible.model import (
from .budget import (
DEFAULT_CONTEXT_MAX_TOKENS,
DEFAULT_HARD_MAX_TOOL_RESULT_CHARS,
DEFAULT_TOOL_RESULT_MAX_CHARS,
estimate_messages_tokens,
)
@@ -328,6 +329,7 @@ class ChatAgent:
stream: bool
system_prompt: str | None
max_tool_result_chars: int
hard_max_tool_result_chars: int
parallel_tools: bool
max_context_tokens: int
todo_stack: TodoStack | None
@@ -362,6 +364,7 @@ class ChatAgent:
stream: bool = True,
system_prompt: str | None = None,
max_tool_result_chars: int = DEFAULT_TOOL_RESULT_MAX_CHARS,
hard_max_tool_result_chars: int = DEFAULT_HARD_MAX_TOOL_RESULT_CHARS,
parallel_tools: bool = True,
max_context_tokens: int = DEFAULT_CONTEXT_MAX_TOKENS,
todo_stack: TodoStack | None = None,
@@ -388,6 +391,7 @@ class ChatAgent:
self.stream = stream
self.system_prompt = system_prompt
self.max_tool_result_chars = max_tool_result_chars
self.hard_max_tool_result_chars = hard_max_tool_result_chars
self.parallel_tools = parallel_tools
self.max_context_tokens = max_context_tokens
self.todo_stack = todo_stack
@@ -648,6 +652,7 @@ class ChatAgent:
on_limit=self.on_limit,
stream=self.stream,
max_tool_result_chars=self.max_tool_result_chars,
hard_max_tool_result_chars=self.hard_max_tool_result_chars,
parallel_tools=self.parallel_tools,
max_context_tokens=self.max_context_tokens,
todo_stack=self.todo_stack,
@@ -809,6 +814,7 @@ class ChatAgent:
stream=self.stream,
system_prompt=self.system_prompt if system_prompt is None else system_prompt,
max_tool_result_chars=self.max_tool_result_chars,
hard_max_tool_result_chars=self.hard_max_tool_result_chars,
parallel_tools=self.parallel_tools,
max_context_tokens=self.max_context_tokens,
todo_stack=None,
+2 -1
View File
@@ -25,7 +25,8 @@ use `run_argv_batch` for ordered pipelines.
`skill_read` slices a skill file the same way).
- `regex_files`: `pattern` is a Python `re` regex (not a glob, so `*.py` fails) and \
`path` is a literal file/dir (no glob expansion).
- Truncated results carry a `truncate_token=...`; continue with `get_truncated`.
- A long tool result carries a `document_id=...`; read the rest with `read_document` \
(line window) or `search_document` (regex).
- Prefer `fetch` for public pages (clean Markdown); `fetch_local` for localhost/LAN, \
non-GET methods, headers, or raw status; `offset` (0-based) skips leading body characters.
- PTY secrets only via `ask_into_pty`; denylists and confirms still apply.
+32 -19
View File
@@ -25,10 +25,12 @@ from plyngent.typedef import Unset # noqa: TC001
from .budget import (
DEFAULT_CONTEXT_MAX_TOKENS,
DEFAULT_HARD_MAX_TOOL_RESULT_CHARS,
DEFAULT_TOOL_RESULT_MAX_CHARS,
cap_hard_tool_text,
compact_messages_for_request,
estimate_messages_tokens,
truncate_tool_result,
truncate_for_request,
)
from .directive_checkpoint import (
DEFAULT_DIRECTIVE_REMINDER_TOKENS,
@@ -88,7 +90,7 @@ async def _run_one_tool(
tools: ToolRegistry,
call: AnyAssistantToolCall,
*,
max_result_chars: int,
hard_max_chars: int,
decision: BatchConfirmDecision | None = None,
) -> tuple[ToolChatMessage, ErrorEvent | None]:
if isinstance(call, AssistantFunctionToolCall):
@@ -99,13 +101,13 @@ async def _run_one_tool(
except Exception as exc: # noqa: BLE001
result_text = f"error: tool {call.function.name!r} failed: {exc}"
err = ErrorEvent(message=result_text, retryable=True, source="tool")
truncated = truncate_tool_result(result_text, max_result_chars)
return ToolChatMessage(content=truncated, tool_call_id=call.id), err
# Only the text is capped; an image the tool returned rides along whole.
truncated = truncate_tool_result(output.text, max_result_chars)
return ToolChatMessage(content=cap_hard_tool_text(result_text, hard_max_chars), tool_call_id=call.id), err
# The result is stored whole (up to the storage ceiling); the
# request-prep pass trims it on the wire and names its document. An image
# the tool returned rides along whole.
return (
ToolChatMessage(
content=truncated,
content=cap_hard_tool_text(output.text, hard_max_chars),
tool_call_id=call.id,
images=output.images or UNSET,
),
@@ -127,7 +129,7 @@ async def _execute_tool_calls(
tool_calls: Sequence[AnyAssistantToolCall],
messages: list[AnyChatMessage],
*,
max_result_chars: int = DEFAULT_TOOL_RESULT_MAX_CHARS,
hard_max_chars: int = DEFAULT_HARD_MAX_TOOL_RESULT_CHARS,
parallel: bool = True,
) -> AsyncIterator[AgentEvent]:
for call in tool_calls:
@@ -149,13 +151,13 @@ async def _execute_tool_calls(
if parallel and len(tool_calls) > 1:
results = await asyncio.gather(
*[
_run_one_tool(tools, call, max_result_chars=max_result_chars, decision=decision)
_run_one_tool(tools, call, hard_max_chars=hard_max_chars, decision=decision)
for call, decision in zip(tool_calls, decisions, strict=True)
]
)
else:
results = [
await _run_one_tool(tools, call, max_result_chars=max_result_chars, decision=decision)
await _run_one_tool(tools, call, hard_max_chars=hard_max_chars, decision=decision)
for call, decision in zip(tool_calls, decisions, strict=True)
]
@@ -532,6 +534,7 @@ async def run_chat_loop( # noqa: C901, PLR0915 — multi-phase tool loop
on_limit: LimitContinueHook | None = None,
stream: bool = True,
max_tool_result_chars: int = DEFAULT_TOOL_RESULT_MAX_CHARS,
hard_max_tool_result_chars: int = DEFAULT_HARD_MAX_TOOL_RESULT_CHARS,
parallel_tools: bool = True,
max_context_tokens: int = DEFAULT_CONTEXT_MAX_TOKENS,
todo_stack: TodoStack | None = None,
@@ -547,7 +550,13 @@ async def run_chat_loop( # noqa: C901, PLR0915 — multi-phase tool loop
When ``stream=True``, uses ``chat_completions(..., stream=True)`` and yields
text deltas as chunks arrive; tool calls are merged from stream deltas.
Multiple tool calls in one round run in parallel when ``parallel_tools``.
Request payloads may shrink older tool results when over ``max_context_tokens``.
Each round builds a request copy: soft compact shrinks older tool dumps when
over ``max_context_tokens``, then every tool result is trimmed to
``max_tool_result_chars`` with a marker naming its document (``read_document``
/ ``search_document`` read the rest). A tool result is stored whole in its
message, up to ``hard_max_tool_result_chars``; a result above that ceiling is
cut on store and has no document.
*reasoning_effort* is the resolved thinking strength (``""`` = send nothing,
the provider's own default); each API surface maps it onto its own wire field
@@ -588,13 +597,17 @@ async def run_chat_loop( # noqa: C901, PLR0915 — multi-phase tool loop
while True:
while rounds_used < allowance:
rounds_used += 1
# Request copy: shrink old tool dumps, then rewrite forged todo nags
# so cleaned stacks do not re-appear with stale OPEN WORK text.
request_messages = compact_messages_for_request(
messages,
max_tokens=max_context_tokens,
prompt_tokens_hint=prompt_tokens_hint,
sent_estimate_tokens=sent_estimate_tokens,
# Request copy: soft-compact old tool dumps, then trim every result to
# the fixed wire cap (each names its document), then rewrite forged
# todo nags so cleaned stacks do not re-appear with stale OPEN WORK text.
request_messages = truncate_for_request(
compact_messages_for_request(
messages,
max_tokens=max_context_tokens,
prompt_tokens_hint=prompt_tokens_hint,
sent_estimate_tokens=sent_estimate_tokens,
),
max_tool_result_chars=max_tool_result_chars,
)
if todo_stack is not None:
_ = refresh_synthetic_todo_nags(request_messages, todo_stack)
@@ -651,7 +664,7 @@ async def run_chat_loop( # noqa: C901, PLR0915 — multi-phase tool loop
tools,
tool_calls,
messages,
max_result_chars=max_tool_result_chars,
hard_max_chars=hard_max_tool_result_chars,
parallel=parallel_tools,
):
yield event
+1
View File
@@ -33,6 +33,7 @@ _MINIMAL_CONFIG = """\
# # system_prompt = "" / tool_directives = "" disable each part; both "" → no system.
# # Or multi-line '''...''' to override.
# max_tool_result_chars = 32000
# hard_max_tool_result_chars = 2000000
# parallel_tools = true
# confirm_destructive = true
# path_denylist = ["/secrets/", ".ssh/"]
+1
View File
@@ -499,6 +499,7 @@ class ReplState:
stream=self.stream_enabled,
system_prompt=system_prompt,
max_tool_result_chars=agent_cfg.max_tool_result_chars,
hard_max_tool_result_chars=agent_cfg.hard_max_tool_result_chars,
parallel_tools=agent_cfg.parallel_tools,
max_context_tokens=agent_cfg.max_context_tokens,
todo_stack=self.todo_stack,
+8 -3
View File
@@ -64,8 +64,9 @@ skill file the same way.
(`*.py` / `**/*.py` fail with `invalid regex`), and a literal file/dir `path` \
(no glob expansion, so `src/**/*.py` fails with `path does not exist`); there is \
no file-type filter, so narrow with `path` and filter hits yourself.
- Truncated results (any tool) carry a `truncate_token=...`; use `get_truncated` \
with it to keep reading (chains through truncations).
- A long tool result is stored whole and trimmed on each request with a \
`document_id=...` marker; read the rest with `read_document` (line window) or \
`search_document` (regex), no re-run.
### Commands
- Prefer `run_argv` / `run_argv_batch` (argv lists, no shell) over `bash -c` or similar.
@@ -84,7 +85,7 @@ for ordered pipelines (`pipe_out` / `mix_stderr` as needed).
(GET/POST/PUT/DELETE) from this host for localhost/LAN, APIs, custom headers, and \
status/redirect inspection.
- `offset` (0-based) skips leading body characters to page through a large body; \
truncated bodies resume with `get_truncated` and are not re-requested.
a long body is stored whole, so its `document_id` reads the rest with no new request.
- Set `user_agent` (or a User-Agent header) on `fetch_local` when the remote expects a \
specific client; otherwise a small default is used. Never rely on shell to spoof identity.
- Private/loopback/LAN hosts need an explicit human policy allow (not skipped by YOLO); \
@@ -151,6 +152,10 @@ class AgentConfig(Struct, omit_defaults=True):
# into the agent system prompt when tools are on. false = ignore server text.
mcp_instructions: bool = True
max_tool_result_chars: int = 32_000
# Storage ceiling for one tool result kept whole in its message. The wire cap
# above trims a result on the request copy and names a document for the rest;
# a result above this ceiling is cut on store and has no document.
hard_max_tool_result_chars: int = 2_000_000
parallel_tools: bool = True
confirm_destructive: bool = True
path_denylist: list[str] = field(default_factory=list)
+37 -15
View File
@@ -1,27 +1,49 @@
from __future__ import annotations
from plyngent.agent.budget import estimate_message_chars, truncate_tool_result
from plyngent.agent.budget import (
cap_hard_tool_text,
cap_tool_message,
estimate_message_chars,
truncate_for_request,
)
from plyngent.lmproto.openai_compatible.model import ToolChatMessage, UserChatMessage
def test_truncate_tool_result_short() -> None:
assert truncate_tool_result("hello", 100) == "hello"
def test_cap_hard_tool_text_short_untouched() -> None:
assert cap_hard_tool_text("hello", 100) == "hello"
def test_truncate_tool_result_long() -> None:
text = "a" * 50
out = truncate_tool_result(text, 20)
assert "Truncated" in out
assert "20" in out
assert "30" in out
# Generic tool results now embed a memory token too (resumable via get_truncated).
assert "truncate_token=" in out
assert out.startswith(("a" * 10, "a" * 20)) # marker is budgeted
def test_cap_hard_tool_text_cuts_with_ceiling_marker() -> None:
out = cap_hard_tool_text("a" * 50, 20)
assert out.startswith("a" * 20)
assert "[Stored at the 20-char ceiling: 30 chars beyond it]" in out
def test_truncate_tool_result_fits_no_marker() -> None:
assert truncate_tool_result("tiny", 100) == "tiny"
assert "Truncated" not in truncate_tool_result("tiny", 100)
def test_cap_tool_message_names_the_document() -> None:
message = ToolChatMessage(content="a" * 50, tool_call_id="call-7")
capped = cap_tool_message(message, 20)
assert capped is not message
assert capped.tool_call_id == "call-7"
assert "document_id=call-7" in capped.content
assert capped.content.startswith("a")
# The stored message keeps its body whole.
assert message.content == "a" * 50
def test_cap_tool_message_short_untouched() -> None:
message = ToolChatMessage(content="tiny", tool_call_id="call-1")
assert cap_tool_message(message, 100) is message
def test_truncate_for_request_caps_only_tool_messages() -> None:
user = UserChatMessage(content="u" * 1000)
tool = ToolChatMessage(content="t" * 1000, tool_call_id="call-1")
out = truncate_for_request([user, tool], max_tool_result_chars=200)
assert out[0] is user
capped = out[1]
assert isinstance(capped, ToolChatMessage)
assert "document_id=call-1" in capped.content
assert len(capped.content) <= 200
def test_estimate_message_chars() -> None:
+2 -24
View File
@@ -8,7 +8,6 @@ from plyngent.agent.budget import (
compact_messages_for_request,
estimate_messages_tokens,
measure_messages_tokens,
truncate_tool_result,
)
from plyngent.agent.loop import run_chat_loop
from plyngent.lmproto.openai_compatible.model import (
@@ -33,27 +32,6 @@ if TYPE_CHECKING:
from plyngent.lmproto.openai_compatible.model import AnyChatMessage
def test_truncate_tool_result_short() -> None:
assert truncate_tool_result("hello", 100) == "hello"
def test_truncate_tool_result_long() -> None:
text = "a" * 50
out = truncate_tool_result(text, 20)
assert "Truncated" in out
assert "20" in out
assert "30" in out
# Generic tool results now embed a memory token too (resumable via get_truncated).
assert "truncate_token=" in out
def test_truncate_tool_result_no_hint() -> None:
text = "a" * 50
out = truncate_tool_result(text, 20)
assert "Truncated" in out
assert "truncate_token=" in out
def test_compact_shrinks_old_tool_results() -> None:
messages: list[AnyChatMessage] = [
UserChatMessage(content="start"),
@@ -92,8 +70,8 @@ def test_compact_shrinks_old_tool_results() -> None:
assert isinstance(compacted[2], ToolChatMessage)
assert len(compacted[2].content) < original_len
assert "Truncated" in compacted[2].content
# Compact shrinks carry a token too (resumable via get_truncated).
assert "truncate_token=" in compacted[2].content
# A shrunk result names its document (read_document / search_document).
assert "document_id=1" in compacted[2].content
# Full history unchanged
assert isinstance(messages[2], ToolChatMessage)
assert len(messages[2].content) == original_len
+11 -6
View File
@@ -1065,7 +1065,7 @@ async def test_chat_agent_system_prompt_prepended() -> None:
async def test_tool_result_char_budget() -> None:
@tool(register=False)
def big() -> str:
return "x" * 100
return "x" * 2000
registry = ToolRegistry([big])
client = ScriptedClient(
@@ -1093,16 +1093,21 @@ async def test_tool_result_char_budget() -> None:
model="m",
tools=registry,
stream=False,
max_tool_result_chars=20,
max_tool_result_chars=200,
parallel_tools=False,
)
]
from plyngent.lmproto.openai_compatible.model import ToolChatMessage
tool_msgs = [m for m in messages if isinstance(m, ToolChatMessage)]
assert len(tool_msgs) == 1
assert tool_msgs[0].content.startswith("x" * 20)
assert "Truncated" in tool_msgs[0].content
# The stored result stays whole; only the wire copy is capped, and it names
# the document that reads the rest.
stored = [m for m in messages if isinstance(m, ToolChatMessage)]
assert len(stored) == 1
assert stored[0].content == "x" * 2000
wire = next(m for m in client.calls[-1].messages if isinstance(m, ToolChatMessage))
assert wire.content.startswith("x")
assert "document_id=1" in wire.content
assert len(wire.content) < 2000
async def test_empty_completion_raises() -> None:
+5 -5
View File
@@ -71,7 +71,7 @@ async def test_an_unknown_tool_is_error_text_without_images() -> None:
async def test_the_loop_keeps_tool_images_on_the_tool_message() -> None:
message, err = await _run_one_tool(_registry(), _call("shot"), max_result_chars=32_000)
message, err = await _run_one_tool(_registry(), _call("shot"), hard_max_chars=32_000)
assert err is None
assert isinstance(message, ToolChatMessage)
@@ -80,16 +80,16 @@ async def test_the_loop_keeps_tool_images_on_the_tool_message() -> None:
async def test_the_loop_leaves_the_images_field_unset_for_text_tools() -> None:
message, err = await _run_one_tool(_registry(), _call("plain"), max_result_chars=32_000)
message, err = await _run_one_tool(_registry(), _call("plain"), hard_max_chars=32_000)
assert err is None
assert message.images is UNSET
async def test_a_truncated_tool_result_keeps_its_images() -> None:
message, _ = await _run_one_tool(_registry(), _call("shot"), max_result_chars=1)
async def test_a_tool_result_above_the_storage_ceiling_keeps_its_images() -> None:
message, _ = await _run_one_tool(_registry(), _call("shot"), hard_max_chars=1)
assert "[Truncated" in message.content
assert "[Stored at the 1-char ceiling" in message.content
assert message.images == [_image()]