mirror of
https://github.com/NCBM/plyngent.git
synced 2026-11-07 15:37:55 +08:00
core/agent+cli: keep a round cut off at the output limit
This commit is contained in:
@@ -17,6 +17,7 @@ from .events import ReasoningDeltaEvent as ReasoningDeltaEvent
|
|||||||
from .events import TextDeltaEvent as TextDeltaEvent
|
from .events import TextDeltaEvent as TextDeltaEvent
|
||||||
from .events import ToolCallEvent as ToolCallEvent
|
from .events import ToolCallEvent as ToolCallEvent
|
||||||
from .events import ToolResultEvent as ToolResultEvent
|
from .events import ToolResultEvent as ToolResultEvent
|
||||||
|
from .events import TruncatedEvent as TruncatedEvent
|
||||||
from .events import UsageEvent as UsageEvent
|
from .events import UsageEvent as UsageEvent
|
||||||
from .loop import DEFAULT_MAX_ROUNDS as DEFAULT_MAX_ROUNDS
|
from .loop import DEFAULT_MAX_ROUNDS as DEFAULT_MAX_ROUNDS
|
||||||
from .loop import run_chat_loop as run_chat_loop
|
from .loop import run_chat_loop as run_chat_loop
|
||||||
@@ -26,6 +27,7 @@ from .notices import format_moment as format_moment
|
|||||||
from .notices import has_notice as has_notice
|
from .notices import has_notice as has_notice
|
||||||
from .notices import resume_notice as resume_notice
|
from .notices import resume_notice as resume_notice
|
||||||
from .notices import skill_tools_notice as skill_tools_notice
|
from .notices import skill_tools_notice as skill_tools_notice
|
||||||
|
from .notices import truncated_turn_notice as truncated_turn_notice
|
||||||
from .tools import DangerClassifier as DangerClassifier
|
from .tools import DangerClassifier as DangerClassifier
|
||||||
from .tools import ToolConfirmHook as ToolConfirmHook
|
from .tools import ToolConfirmHook as ToolConfirmHook
|
||||||
from .tools import ToolDefinition as ToolDefinition
|
from .tools import ToolDefinition as ToolDefinition
|
||||||
|
|||||||
@@ -27,7 +27,8 @@ from .directive_checkpoint import (
|
|||||||
parse_checkpoint_bands,
|
parse_checkpoint_bands,
|
||||||
)
|
)
|
||||||
from .events import UsageEvent
|
from .events import UsageEvent
|
||||||
from .loop import DEFAULT_MAX_ROUNDS, run_chat_loop
|
from .loop import DEFAULT_MAX_ROUNDS, DEFAULT_MAX_TRUNCATION_CONTINUES, run_chat_loop
|
||||||
|
from .notices import TRUNCATED_NOTICE_KIND, notice_of_kind
|
||||||
from .todo_nag import (
|
from .todo_nag import (
|
||||||
DEFAULT_TODO_NAG_STRATEGY,
|
DEFAULT_TODO_NAG_STRATEGY,
|
||||||
parse_todo_nag_strategy,
|
parse_todo_nag_strategy,
|
||||||
@@ -140,8 +141,10 @@ def incomplete_turn_user_text(messages: Sequence[AnyChatMessage]) -> str | None:
|
|||||||
"""User text for an incomplete turn, if history can be retried.
|
"""User text for an incomplete turn, if history can be retried.
|
||||||
|
|
||||||
Incomplete when the last non-system message is a user message (failed before
|
Incomplete when the last non-system message is a user message (failed before
|
||||||
any commit), or tool results after a complete tool batch (failed on a later
|
any commit), tool results after a complete tool batch (failed on a later
|
||||||
model round — keep tools so retry does not re-execute side effects).
|
model round — keep tools so retry does not re-execute side effects), or a
|
||||||
|
partial answer under a truncation notice (the round hit the output limit; its
|
||||||
|
answer was kept, so retry continues it instead of regenerating).
|
||||||
Trailing developer messages (host notices, directive checkpoints) belong to
|
Trailing developer messages (host notices, directive checkpoints) belong to
|
||||||
the turn but never end it, so they are skipped when reading the tail.
|
the turn but never end it, so they are skipped when reading the tail.
|
||||||
"""
|
"""
|
||||||
@@ -150,14 +153,17 @@ def incomplete_turn_user_text(messages: Sequence[AnyChatMessage]) -> str | None:
|
|||||||
if messages and isinstance(messages[0], SystemChatMessage):
|
if messages and isinstance(messages[0], SystemChatMessage):
|
||||||
start = 1
|
start = 1
|
||||||
body = list(messages[start:])
|
body = list(messages[start:])
|
||||||
|
truncated = False
|
||||||
while body and isinstance(body[-1], DeveloperChatMessage):
|
while body and isinstance(body[-1], DeveloperChatMessage):
|
||||||
|
if notice_of_kind(body[-1], TRUNCATED_NOTICE_KIND):
|
||||||
|
truncated = True
|
||||||
_ = body.pop()
|
_ = body.pop()
|
||||||
if not body:
|
if not body:
|
||||||
return None
|
return None
|
||||||
last = body[-1]
|
last = body[-1]
|
||||||
if isinstance(last, UserChatMessage):
|
if isinstance(last, UserChatMessage):
|
||||||
return last.content
|
return last.content
|
||||||
if isinstance(last, ToolChatMessage):
|
if isinstance(last, ToolChatMessage) or (truncated and isinstance(last, AssistantChatMessage)):
|
||||||
# Walk back to the user that opened this turn.
|
# Walk back to the user that opened this turn.
|
||||||
for msg in reversed(body):
|
for msg in reversed(body):
|
||||||
if isinstance(msg, UserChatMessage):
|
if isinstance(msg, UserChatMessage):
|
||||||
@@ -289,6 +295,8 @@ class ChatAgent:
|
|||||||
memory: MemoryStore | None
|
memory: MemoryStore | None
|
||||||
session_id: int | None
|
session_id: int | None
|
||||||
max_rounds: int
|
max_rounds: int
|
||||||
|
# Rounds a turn may carry on after hitting the output limit (0 = none).
|
||||||
|
max_truncation_continues: int
|
||||||
temperature: float | None
|
temperature: float | None
|
||||||
# Resolved thinking strength ("" = provider default); see config.reasoning.
|
# Resolved thinking strength ("" = provider default); see config.reasoning.
|
||||||
reasoning_effort: str
|
reasoning_effort: str
|
||||||
@@ -322,6 +330,7 @@ class ChatAgent:
|
|||||||
memory: MemoryStore | None = None,
|
memory: MemoryStore | None = None,
|
||||||
session_id: int | None = None,
|
session_id: int | None = None,
|
||||||
max_rounds: int = DEFAULT_MAX_ROUNDS,
|
max_rounds: int = DEFAULT_MAX_ROUNDS,
|
||||||
|
max_truncation_continues: int = DEFAULT_MAX_TRUNCATION_CONTINUES,
|
||||||
temperature: float | None = None,
|
temperature: float | None = None,
|
||||||
reasoning_effort: str = "",
|
reasoning_effort: str = "",
|
||||||
messages: Sequence[AnyChatMessage] | None = None,
|
messages: Sequence[AnyChatMessage] | None = None,
|
||||||
@@ -345,6 +354,7 @@ class ChatAgent:
|
|||||||
self.memory = memory
|
self.memory = memory
|
||||||
self.session_id = session_id
|
self.session_id = session_id
|
||||||
self.max_rounds = max_rounds
|
self.max_rounds = max_rounds
|
||||||
|
self.max_truncation_continues = max(0, int(max_truncation_continues))
|
||||||
self.temperature = temperature
|
self.temperature = temperature
|
||||||
self.reasoning_effort = reasoning_effort
|
self.reasoning_effort = reasoning_effort
|
||||||
self.on_limit = on_limit
|
self.on_limit = on_limit
|
||||||
@@ -603,6 +613,7 @@ class ChatAgent:
|
|||||||
model=self.model,
|
model=self.model,
|
||||||
tools=self.tools,
|
tools=self.tools,
|
||||||
max_rounds=self.max_rounds,
|
max_rounds=self.max_rounds,
|
||||||
|
max_truncation_continues=self.max_truncation_continues,
|
||||||
temperature=self.temperature,
|
temperature=self.temperature,
|
||||||
reasoning_effort=self.reasoning_effort,
|
reasoning_effort=self.reasoning_effort,
|
||||||
on_limit=self.on_limit,
|
on_limit=self.on_limit,
|
||||||
@@ -747,6 +758,7 @@ class ChatAgent:
|
|||||||
memory=None,
|
memory=None,
|
||||||
session_id=None,
|
session_id=None,
|
||||||
max_rounds=rounds,
|
max_rounds=rounds,
|
||||||
|
max_truncation_continues=self.max_truncation_continues,
|
||||||
temperature=self.temperature,
|
temperature=self.temperature,
|
||||||
reasoning_effort=self.reasoning_effort,
|
reasoning_effort=self.reasoning_effort,
|
||||||
on_limit=self.on_limit,
|
on_limit=self.on_limit,
|
||||||
|
|||||||
@@ -34,6 +34,19 @@ class MaxRoundsEvent(Struct, tag_field="type", tag="max_rounds"):
|
|||||||
continued: bool = False
|
continued: bool = False
|
||||||
|
|
||||||
|
|
||||||
|
class TruncatedEvent(Struct, tag_field="type", tag="truncated"):
|
||||||
|
"""A round the provider cut off at the output limit (``finish_reason=length``).
|
||||||
|
|
||||||
|
The partial answer stays in history rather than being discarded, and the
|
||||||
|
model is told where it stopped (see ``notices.truncated_turn_notice``);
|
||||||
|
``continued`` says whether the turn went on with another round or is waiting
|
||||||
|
for ``/retry``.
|
||||||
|
"""
|
||||||
|
|
||||||
|
kept_chars: int = 0
|
||||||
|
continued: bool = False
|
||||||
|
|
||||||
|
|
||||||
class ErrorEvent(Struct, tag_field="type", tag="error"):
|
class ErrorEvent(Struct, tag_field="type", tag="error"):
|
||||||
message: str
|
message: str
|
||||||
retryable: bool = True
|
retryable: bool = True
|
||||||
@@ -60,6 +73,7 @@ type AgentEvent = (
|
|||||||
| ToolCallEvent
|
| ToolCallEvent
|
||||||
| ToolResultEvent
|
| ToolResultEvent
|
||||||
| MaxRoundsEvent
|
| MaxRoundsEvent
|
||||||
|
| TruncatedEvent
|
||||||
| ErrorEvent
|
| ErrorEvent
|
||||||
| CancelledEvent
|
| CancelledEvent
|
||||||
| UsageEvent
|
| UsageEvent
|
||||||
|
|||||||
+81
-14
@@ -43,8 +43,11 @@ from .events import (
|
|||||||
TextDeltaEvent,
|
TextDeltaEvent,
|
||||||
ToolCallEvent,
|
ToolCallEvent,
|
||||||
ToolResultEvent,
|
ToolResultEvent,
|
||||||
|
TruncatedEvent,
|
||||||
UsageEvent,
|
UsageEvent,
|
||||||
)
|
)
|
||||||
|
from .finish_reason import chat_finish_reason
|
||||||
|
from .notices import truncated_turn_notice
|
||||||
from .todo_nag import (
|
from .todo_nag import (
|
||||||
DEFAULT_TODO_NAG_STRATEGY,
|
DEFAULT_TODO_NAG_STRATEGY,
|
||||||
inject_todo_nag_for_stack_with_events,
|
inject_todo_nag_for_stack_with_events,
|
||||||
@@ -68,6 +71,10 @@ if TYPE_CHECKING:
|
|||||||
|
|
||||||
|
|
||||||
DEFAULT_MAX_ROUNDS = 32
|
DEFAULT_MAX_ROUNDS = 32
|
||||||
|
# Rounds cut off at the output limit are kept and the model carries on in another
|
||||||
|
# round, this many times per turn; 0 stops after the truncated round (`/retry`
|
||||||
|
# continues it from the kept partial answer).
|
||||||
|
DEFAULT_MAX_TRUNCATION_CONTINUES = 2
|
||||||
|
|
||||||
|
|
||||||
async def _call_on_limit(on_limit: LimitContinueHook, reason: str) -> bool:
|
async def _call_on_limit(on_limit: LimitContinueHook, reason: str) -> bool:
|
||||||
@@ -150,6 +157,11 @@ def _assistant_has_payload(assistant: AssistantChatMessage) -> bool:
|
|||||||
return bool(isinstance(reasoning, str) and reasoning.strip())
|
return bool(isinstance(reasoning, str) and reasoning.strip())
|
||||||
|
|
||||||
|
|
||||||
|
def _kept_chars(assistant: AssistantChatMessage) -> int:
|
||||||
|
"""How much answer a cut-off round had written (0 if only reasoning/blocks)."""
|
||||||
|
return len(assistant.content) if isinstance(assistant.content, str) else 0
|
||||||
|
|
||||||
|
|
||||||
def _finish_reason_value(finish: object) -> str | None:
|
def _finish_reason_value(finish: object) -> str | None:
|
||||||
if finish is UNSET or finish is None:
|
if finish is UNSET or finish is None:
|
||||||
return None
|
return None
|
||||||
@@ -163,20 +175,26 @@ def _validate_assistant_terminal(
|
|||||||
*,
|
*,
|
||||||
finish_reason: str | None,
|
finish_reason: str | None,
|
||||||
stream_terminal: bool,
|
stream_terminal: bool,
|
||||||
) -> None:
|
) -> bool:
|
||||||
"""Raise when the round is not a usable agent stop.
|
"""Raise when the round is not a usable agent stop; True if it hit the output limit.
|
||||||
|
|
||||||
Distinguishes empty generation, truncated/filtered stops, and missing
|
Distinguishes empty generation, filtered stops, and missing stream terminals
|
||||||
stream terminals (network/client glitch) from a normal stop/tool_calls end.
|
(network/client glitch) from a normal stop/tool_calls end. A round cut off at
|
||||||
|
the output limit is not an error of its own: what it wrote is usable, so the
|
||||||
|
caller keeps it and tells the model where it stopped. A truncated round that
|
||||||
|
wrote nothing stays a failure — there is nothing to continue from.
|
||||||
"""
|
"""
|
||||||
reason = (finish_reason or "").lower() or None
|
reason = (finish_reason or "").lower() or None
|
||||||
|
canonical = chat_finish_reason(reason, has_tool_calls=False)
|
||||||
|
|
||||||
if reason in {"length", "content_filter"}:
|
if canonical == "length":
|
||||||
label = "truncated (max tokens)" if reason == "length" else "content filter"
|
if _assistant_has_payload(assistant):
|
||||||
detail = ""
|
return True
|
||||||
if isinstance(assistant.content, str) and assistant.content.strip():
|
msg = "model stopped early: output limit reached with nothing written"
|
||||||
detail = f"; partial text kept ({len(assistant.content)} chars)"
|
raise RuntimeError(msg)
|
||||||
msg = f"model stopped early: {label}{detail}"
|
|
||||||
|
if canonical == "content_filter":
|
||||||
|
msg = "model stopped early: content filter"
|
||||||
raise RuntimeError(msg)
|
raise RuntimeError(msg)
|
||||||
|
|
||||||
if reason in {"failed", "incomplete", "cancelled"}:
|
if reason in {"failed", "incomplete", "cancelled"}:
|
||||||
@@ -184,7 +202,7 @@ def _validate_assistant_terminal(
|
|||||||
raise RuntimeError(msg)
|
raise RuntimeError(msg)
|
||||||
|
|
||||||
if _assistant_has_payload(assistant):
|
if _assistant_has_payload(assistant):
|
||||||
return
|
return False
|
||||||
|
|
||||||
if not stream_terminal and reason is None:
|
if not stream_terminal and reason is None:
|
||||||
msg = (
|
msg = (
|
||||||
@@ -249,7 +267,7 @@ async def _non_stream_round(
|
|||||||
choice = response.choices[0]
|
choice = response.choices[0]
|
||||||
assistant = choice.message
|
assistant = choice.message
|
||||||
finish = _finish_reason_value(choice.finish_reason)
|
finish = _finish_reason_value(choice.finish_reason)
|
||||||
_validate_assistant_terminal(
|
truncated = _validate_assistant_terminal(
|
||||||
assistant,
|
assistant,
|
||||||
finish_reason=finish,
|
finish_reason=finish,
|
||||||
stream_terminal=True,
|
stream_terminal=True,
|
||||||
@@ -260,6 +278,8 @@ async def _non_stream_round(
|
|||||||
if isinstance(assistant.content, str) and assistant.content:
|
if isinstance(assistant.content, str) and assistant.content:
|
||||||
yield TextDeltaEvent(content=assistant.content)
|
yield TextDeltaEvent(content=assistant.content)
|
||||||
yield AssistantMessageEvent(message=assistant)
|
yield AssistantMessageEvent(message=assistant)
|
||||||
|
if truncated:
|
||||||
|
yield TruncatedEvent(kept_chars=_kept_chars(assistant))
|
||||||
yield UsageEvent(
|
yield UsageEvent(
|
||||||
usage=resolve_round_usage(response.usage, param.messages, assistant),
|
usage=resolve_round_usage(response.usage, param.messages, assistant),
|
||||||
)
|
)
|
||||||
@@ -376,12 +396,14 @@ async def _stream_round(
|
|||||||
# If we got payload but no finish_reason, treat as terminal (some providers
|
# If we got payload but no finish_reason, treat as terminal (some providers
|
||||||
# omit it); if empty and no finish_reason, flag as missing terminal.
|
# omit it); if empty and no finish_reason, flag as missing terminal.
|
||||||
stream_terminal = saw_terminal or _assistant_has_payload(assistant)
|
stream_terminal = saw_terminal or _assistant_has_payload(assistant)
|
||||||
_validate_assistant_terminal(
|
truncated = _validate_assistant_terminal(
|
||||||
assistant,
|
assistant,
|
||||||
finish_reason=finish_reason,
|
finish_reason=finish_reason,
|
||||||
stream_terminal=stream_terminal,
|
stream_terminal=stream_terminal,
|
||||||
)
|
)
|
||||||
yield AssistantMessageEvent(message=assistant)
|
yield AssistantMessageEvent(message=assistant)
|
||||||
|
if truncated:
|
||||||
|
yield TruncatedEvent(kept_chars=_kept_chars(assistant))
|
||||||
yield UsageEvent(
|
yield UsageEvent(
|
||||||
usage=resolve_round_usage(last_api_usage, param.messages, assistant),
|
usage=resolve_round_usage(last_api_usage, param.messages, assistant),
|
||||||
)
|
)
|
||||||
@@ -408,6 +430,27 @@ async def _assistant_round(
|
|||||||
yield event
|
yield event
|
||||||
|
|
||||||
|
|
||||||
|
def _keep_truncated_round(
|
||||||
|
messages: list[AnyChatMessage],
|
||||||
|
truncated: TruncatedEvent,
|
||||||
|
*,
|
||||||
|
continued: bool,
|
||||||
|
) -> TruncatedEvent:
|
||||||
|
"""Keep a round cut off at the output limit and tell the model where it stopped.
|
||||||
|
|
||||||
|
The answer and its thinking blocks stay in ``messages`` (the provider verifies
|
||||||
|
those blocks, so dropping them would make the round uncontinuable), but the
|
||||||
|
round's tool calls go: a half-written call must not run, its arguments may not
|
||||||
|
even parse, and the model re-issues it after reading the notice. Returns the
|
||||||
|
event to yield, with the continuation outcome filled in.
|
||||||
|
"""
|
||||||
|
last = messages[-1]
|
||||||
|
if isinstance(last, AssistantChatMessage) and last.tool_calls is not UNSET and last.tool_calls:
|
||||||
|
messages[-1] = msgspec.structs.replace(last, tool_calls=UNSET)
|
||||||
|
messages.append(truncated_turn_notice().to_message())
|
||||||
|
return msgspec.structs.replace(truncated, continued=continued)
|
||||||
|
|
||||||
|
|
||||||
def _last_assistant(messages: list[AnyChatMessage], pre_len: int) -> AssistantChatMessage:
|
def _last_assistant(messages: list[AnyChatMessage], pre_len: int) -> AssistantChatMessage:
|
||||||
if len(messages) <= pre_len:
|
if len(messages) <= pre_len:
|
||||||
msg = "model round produced no assistant message"
|
msg = "model round produced no assistant message"
|
||||||
@@ -448,13 +491,14 @@ async def _maybe_inject_directive_checkpoints(
|
|||||||
return new_band
|
return new_band
|
||||||
|
|
||||||
|
|
||||||
async def run_chat_loop( # noqa: C901 — multi-phase tool loop
|
async def run_chat_loop( # noqa: C901, PLR0915 — multi-phase tool loop
|
||||||
client: AnyLLMClient,
|
client: AnyLLMClient,
|
||||||
messages: list[AnyChatMessage],
|
messages: list[AnyChatMessage],
|
||||||
*,
|
*,
|
||||||
model: str,
|
model: str,
|
||||||
tools: ToolRegistry | None = None,
|
tools: ToolRegistry | None = None,
|
||||||
max_rounds: int = DEFAULT_MAX_ROUNDS,
|
max_rounds: int = DEFAULT_MAX_ROUNDS,
|
||||||
|
max_truncation_continues: int = DEFAULT_MAX_TRUNCATION_CONTINUES,
|
||||||
temperature: float | None = None,
|
temperature: float | None = None,
|
||||||
reasoning_effort: str = "",
|
reasoning_effort: str = "",
|
||||||
on_limit: LimitContinueHook | None = None,
|
on_limit: LimitContinueHook | None = None,
|
||||||
@@ -481,6 +525,12 @@ async def run_chat_loop( # noqa: C901 — multi-phase tool loop
|
|||||||
the provider's own default); each API surface maps it onto its own wire field
|
the provider's own default); each API surface maps it onto its own wire field
|
||||||
(see ``config.reasoning``).
|
(see ``config.reasoning``).
|
||||||
|
|
||||||
|
A round the provider cuts off at the output limit (``finish_reason=length``)
|
||||||
|
is kept instead of failing the turn: its answer and thinking blocks stay in
|
||||||
|
``messages``, its tool calls are dropped, and a truncation notice tells the
|
||||||
|
model where it stopped. Up to *max_truncation_continues* rounds per turn then
|
||||||
|
carry on from there (0 = stop, leaving the turn for ``/retry``).
|
||||||
|
|
||||||
*provider_tools* are hosted/provider-side tools (OpenAI Responses only, e.g.
|
*provider_tools* are hosted/provider-side tools (OpenAI Responses only, e.g.
|
||||||
``web_search``) merged into the request as opaque dicts; never executed by the
|
``web_search``) merged into the request as opaque dicts; never executed by the
|
||||||
:class:`ToolRegistry`. Ignored by non-Responses clients.
|
:class:`ToolRegistry`. Ignored by non-Responses clients.
|
||||||
@@ -500,6 +550,7 @@ async def run_chat_loop( # noqa: C901 — multi-phase tool loop
|
|||||||
|
|
||||||
rounds_used = 0
|
rounds_used = 0
|
||||||
allowance = max_rounds
|
allowance = max_rounds
|
||||||
|
truncation_continues = 0
|
||||||
# Calibrate soft-compact from last model call's prompt_tokens (API preferred).
|
# Calibrate soft-compact from last model call's prompt_tokens (API preferred).
|
||||||
prompt_tokens_hint: int | None = None
|
prompt_tokens_hint: int | None = None
|
||||||
sent_estimate_tokens: int | None = None
|
sent_estimate_tokens: int | None = None
|
||||||
@@ -532,6 +583,7 @@ async def run_chat_loop( # noqa: C901 — multi-phase tool loop
|
|||||||
|
|
||||||
pre_len = len(messages)
|
pre_len = len(messages)
|
||||||
last_usage_event: UsageEvent | None = None
|
last_usage_event: UsageEvent | None = None
|
||||||
|
truncated: TruncatedEvent | None = None
|
||||||
async for event in _assistant_round(
|
async for event in _assistant_round(
|
||||||
client,
|
client,
|
||||||
param,
|
param,
|
||||||
@@ -544,9 +596,24 @@ async def run_chat_loop( # noqa: C901 — multi-phase tool loop
|
|||||||
prompt_tokens_hint = event.usage.prompt_tokens
|
prompt_tokens_hint = event.usage.prompt_tokens
|
||||||
sent_estimate_tokens = sent_est
|
sent_estimate_tokens = sent_est
|
||||||
last_usage_event = event
|
last_usage_event = event
|
||||||
|
if isinstance(event, TruncatedEvent):
|
||||||
|
# Yielded below, once the continuation decision is known.
|
||||||
|
truncated = event
|
||||||
|
continue
|
||||||
yield event
|
yield event
|
||||||
|
|
||||||
assistant = _last_assistant(messages, pre_len)
|
assistant = _last_assistant(messages, pre_len)
|
||||||
|
if truncated is not None:
|
||||||
|
# The provider cut the round off at the output limit: keep what it
|
||||||
|
# wrote and tell the model where it stopped (see the helper).
|
||||||
|
continued = truncation_continues < max_truncation_continues
|
||||||
|
if continued:
|
||||||
|
truncation_continues += 1
|
||||||
|
yield _keep_truncated_round(messages, truncated, continued=continued)
|
||||||
|
if continued:
|
||||||
|
continue
|
||||||
|
return
|
||||||
|
|
||||||
tool_calls = assistant.tool_calls
|
tool_calls = assistant.tool_calls
|
||||||
has_tools = tool_calls is not UNSET and bool(tool_calls) and tools is not None
|
has_tools = tool_calls is not UNSET and bool(tool_calls) and tools is not None
|
||||||
if has_tools:
|
if has_tools:
|
||||||
|
|||||||
@@ -27,6 +27,11 @@ if TYPE_CHECKING:
|
|||||||
# developer messages (durable directive checkpoints share the role).
|
# developer messages (durable directive checkpoints share the role).
|
||||||
NOTICE_MARKER = "[plyngent notice"
|
NOTICE_MARKER = "[plyngent notice"
|
||||||
|
|
||||||
|
# Notice kind for a turn the provider cut off at the output limit. The turn's
|
||||||
|
# partial answer is kept, so the agent reads such a turn as unfinished and
|
||||||
|
# ``/retry`` continues it (see ``ChatAgent.pending_retry_text``).
|
||||||
|
TRUNCATED_NOTICE_KIND = "truncated"
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
@dataclass(frozen=True, slots=True)
|
||||||
class Notice:
|
class Notice:
|
||||||
@@ -41,14 +46,18 @@ class Notice:
|
|||||||
return DeveloperChatMessage(content=f"{NOTICE_MARKER}: {self.kind}]\n{self.body.strip()}")
|
return DeveloperChatMessage(content=f"{NOTICE_MARKER}: {self.kind}]\n{self.body.strip()}")
|
||||||
|
|
||||||
|
|
||||||
|
def notice_of_kind(message: AnyChatMessage, kind: str) -> bool:
|
||||||
|
"""Whether *message* is a notice of *kind*."""
|
||||||
|
return isinstance(message, DeveloperChatMessage) and message.content.startswith(f"{NOTICE_MARKER}: {kind}]")
|
||||||
|
|
||||||
|
|
||||||
def has_notice(messages: Sequence[AnyChatMessage], kind: str) -> bool:
|
def has_notice(messages: Sequence[AnyChatMessage], kind: str) -> bool:
|
||||||
"""Whether *messages* already carries a notice of *kind*.
|
"""Whether *messages* already carries a notice of *kind*.
|
||||||
|
|
||||||
Notices are persisted with the transcript, so a host asks this before
|
Notices are persisted with the transcript, so a host asks this before
|
||||||
pushing one that belongs to the session rather than to an event.
|
pushing one that belongs to the session rather than to an event.
|
||||||
"""
|
"""
|
||||||
marker = f"{NOTICE_MARKER}: {kind}]"
|
return any(notice_of_kind(message, kind) for message in messages)
|
||||||
return any(isinstance(message, DeveloperChatMessage) and message.content.startswith(marker) for message in messages)
|
|
||||||
|
|
||||||
|
|
||||||
def format_moment(moment: datetime | None) -> str:
|
def format_moment(moment: datetime | None) -> str:
|
||||||
@@ -82,6 +91,27 @@ def resume_notice(*, last_active: datetime | None) -> Notice:
|
|||||||
return Notice(kind="resume", body=body, summary=summary)
|
return Notice(kind="resume", body=body, summary=summary)
|
||||||
|
|
||||||
|
|
||||||
|
def truncated_turn_notice() -> Notice:
|
||||||
|
"""Notice for a round the provider cut off at the output limit.
|
||||||
|
|
||||||
|
The partial answer is kept in the transcript (thinking blocks included, so
|
||||||
|
the provider still verifies them), which is why the model has to be told the
|
||||||
|
round was not finished: it continues from what it already wrote, re-issues
|
||||||
|
the tool call that was dropped, and keeps the remainder short enough to fit.
|
||||||
|
"""
|
||||||
|
body = (
|
||||||
|
"Your previous round was cut off at the output limit (max_tokens). The partial answer it wrote "
|
||||||
|
"was kept in the transcript, and any tool call in that round was discarded — re-issue it if you "
|
||||||
|
"still need it. Continue from where the answer stopped instead of writing it again from the "
|
||||||
|
"start, and keep the rest short enough to finish: split the work, or hand off what is left."
|
||||||
|
)
|
||||||
|
return Notice(
|
||||||
|
kind=TRUNCATED_NOTICE_KIND,
|
||||||
|
body=body,
|
||||||
|
summary="output truncated (max_tokens); partial answer kept",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
_ASIDE_BODIES: dict[str, str] = {
|
_ASIDE_BODIES: dict[str, str] = {
|
||||||
"read": (
|
"read": (
|
||||||
"Side question: this exchange is not saved to the session transcript and does not "
|
"Side question: this exchange is not saved to the session transcript and does not "
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ from plyngent.agent import (
|
|||||||
TextDeltaEvent,
|
TextDeltaEvent,
|
||||||
ToolCallEvent,
|
ToolCallEvent,
|
||||||
ToolResultEvent,
|
ToolResultEvent,
|
||||||
|
TruncatedEvent,
|
||||||
UsageEvent,
|
UsageEvent,
|
||||||
)
|
)
|
||||||
from plyngent.lmproto.openai_compatible.model import AssistantFunctionToolCall
|
from plyngent.lmproto.openai_compatible.model import AssistantFunctionToolCall
|
||||||
@@ -1289,6 +1290,14 @@ async def render_events( # noqa: C901, PLR0915 — one branch per event type
|
|||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
click.secho(f"\n[max rounds reached: {event.rounds}]", fg="red")
|
click.secho(f"\n[max rounds reached: {event.rounds}]", fg="red")
|
||||||
|
elif isinstance(event, TruncatedEvent):
|
||||||
|
flush_assistant()
|
||||||
|
tool_lines.close_line()
|
||||||
|
tail = "continuing" if event.continued else "/retry continues from here"
|
||||||
|
click.secho(
|
||||||
|
f"\n[limit] output truncated ({event.kept_chars} chars kept); {tail}",
|
||||||
|
fg="yellow",
|
||||||
|
)
|
||||||
elif isinstance(event, UsageEvent):
|
elif isinstance(event, UsageEvent):
|
||||||
_ = event
|
_ = event
|
||||||
else:
|
else:
|
||||||
|
|||||||
@@ -14,10 +14,12 @@ from plyngent.agent import (
|
|||||||
ToolCallEvent,
|
ToolCallEvent,
|
||||||
ToolRegistry,
|
ToolRegistry,
|
||||||
ToolResultEvent,
|
ToolResultEvent,
|
||||||
|
TruncatedEvent,
|
||||||
UsageEvent,
|
UsageEvent,
|
||||||
run_chat_loop,
|
run_chat_loop,
|
||||||
tool,
|
tool,
|
||||||
)
|
)
|
||||||
|
from plyngent.agent.notices import TRUNCATED_NOTICE_KIND, notice_of_kind
|
||||||
from plyngent.config.models import DatabaseConfig
|
from plyngent.config.models import DatabaseConfig
|
||||||
from plyngent.lmproto.openai_compatible.model import (
|
from plyngent.lmproto.openai_compatible.model import (
|
||||||
AnyChatMessage,
|
AnyChatMessage,
|
||||||
@@ -987,20 +989,115 @@ async def test_empty_completion_raises() -> None:
|
|||||||
_ = [e async for e in run_chat_loop(client, messages, model="m", stream=False)]
|
_ = [e async for e in run_chat_loop(client, messages, model="m", stream=False)]
|
||||||
|
|
||||||
|
|
||||||
async def test_length_finish_raises() -> None:
|
async def test_length_finish_keeps_the_partial_answer_and_continues() -> None:
|
||||||
client = ScriptedClient(
|
client = ScriptedClient(
|
||||||
[
|
[
|
||||||
_response(
|
_response(AssistantChatMessage(content="partial"), finish_reason="length"),
|
||||||
AssistantChatMessage(content="partial"),
|
_response(AssistantChatMessage(content="rest")),
|
||||||
finish_reason="length",
|
|
||||||
)
|
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
messages: list[AnyChatMessage] = [UserChatMessage(content="hi")]
|
messages: list[AnyChatMessage] = [UserChatMessage(content="hi")]
|
||||||
with pytest.raises(RuntimeError, match="truncated"):
|
events = [e async for e in run_chat_loop(client, messages, model="m", stream=False)]
|
||||||
|
|
||||||
|
truncated = [e for e in events if isinstance(e, TruncatedEvent)]
|
||||||
|
assert len(truncated) == 1
|
||||||
|
assert truncated[0].kept_chars == len("partial")
|
||||||
|
assert truncated[0].continued
|
||||||
|
# The partial answer stays in history under a truncation notice, and the
|
||||||
|
# continuation round is sent that history — the model continues its answer
|
||||||
|
# rather than writing it again from the start.
|
||||||
|
assert isinstance(messages[1], AssistantChatMessage)
|
||||||
|
assert messages[1].content == "partial"
|
||||||
|
assert notice_of_kind(messages[2], TRUNCATED_NOTICE_KIND)
|
||||||
|
assert isinstance(messages[3], AssistantChatMessage)
|
||||||
|
assert messages[3].content == "rest"
|
||||||
|
assert any(notice_of_kind(m, TRUNCATED_NOTICE_KIND) for m in client.calls[1].messages)
|
||||||
|
|
||||||
|
|
||||||
|
async def test_length_finish_stops_once_the_continuation_budget_is_spent() -> None:
|
||||||
|
client = ScriptedClient([_response(AssistantChatMessage(content="partial"), finish_reason="length")] * 2)
|
||||||
|
messages: list[AnyChatMessage] = [UserChatMessage(content="hi")]
|
||||||
|
events = [
|
||||||
|
e
|
||||||
|
async for e in run_chat_loop(
|
||||||
|
client,
|
||||||
|
messages,
|
||||||
|
model="m",
|
||||||
|
stream=False,
|
||||||
|
max_truncation_continues=1,
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
assert [e.continued for e in events if isinstance(e, TruncatedEvent)] == [True, False]
|
||||||
|
assert len(client.calls) == 2
|
||||||
|
# Nothing raised: the turn ends with the partial answer kept, so `/retry`
|
||||||
|
# continues it instead of regenerating.
|
||||||
|
assert isinstance(messages[-2], AssistantChatMessage)
|
||||||
|
assert messages[-2].content == "partial"
|
||||||
|
assert notice_of_kind(messages[-1], TRUNCATED_NOTICE_KIND)
|
||||||
|
|
||||||
|
|
||||||
|
async def test_length_finish_drops_the_rounds_tool_calls() -> None:
|
||||||
|
ran: list[str] = []
|
||||||
|
|
||||||
|
@tool(name="boom", register=False)
|
||||||
|
async def _boom(argument: str) -> str:
|
||||||
|
ran.append(argument)
|
||||||
|
return "ran"
|
||||||
|
|
||||||
|
call = AssistantFunctionToolCall(
|
||||||
|
id="c1",
|
||||||
|
function=AssistantFunctionTool(name="boom", arguments='{"argument": "x"'),
|
||||||
|
)
|
||||||
|
client = ScriptedClient(
|
||||||
|
[
|
||||||
|
_response(
|
||||||
|
AssistantChatMessage(content="partial", tool_calls=[call]),
|
||||||
|
finish_reason="length",
|
||||||
|
),
|
||||||
|
_response(AssistantChatMessage(content="done")),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
messages: list[AnyChatMessage] = [UserChatMessage(content="hi")]
|
||||||
|
events = [e async for e in run_chat_loop(client, messages, model="m", tools=ToolRegistry([_boom]), stream=False)]
|
||||||
|
|
||||||
|
# A call cut off mid-write is not replayed: the model re-issues it after
|
||||||
|
# reading the notice.
|
||||||
|
assert ran == []
|
||||||
|
assert not [e for e in events if isinstance(e, (ToolCallEvent, ToolResultEvent))]
|
||||||
|
kept = messages[1]
|
||||||
|
assert isinstance(kept, AssistantChatMessage)
|
||||||
|
assert kept.content == "partial"
|
||||||
|
assert kept.tool_calls is UNSET
|
||||||
|
|
||||||
|
|
||||||
|
async def test_length_finish_without_content_still_raises() -> None:
|
||||||
|
client = ScriptedClient([_response(AssistantChatMessage(content=None), finish_reason="length")])
|
||||||
|
messages: list[AnyChatMessage] = [UserChatMessage(content="hi")]
|
||||||
|
with pytest.raises(RuntimeError, match="output limit reached with nothing written"):
|
||||||
_ = [e async for e in run_chat_loop(client, messages, model="m", stream=False)]
|
_ = [e async for e in run_chat_loop(client, messages, model="m", stream=False)]
|
||||||
|
|
||||||
|
|
||||||
|
async def test_truncated_turn_is_retryable() -> None:
|
||||||
|
store = await MemoryStore.open(DatabaseConfig())
|
||||||
|
session = await store.create_session(name="t")
|
||||||
|
client = ScriptedClient(
|
||||||
|
[
|
||||||
|
_response(AssistantChatMessage(content="partial"), finish_reason="length"),
|
||||||
|
_response(AssistantChatMessage(content="rest")),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
agent = ChatAgent(client, model="m", memory=store, session_id=session.sid, max_truncation_continues=0, stream=False)
|
||||||
|
_ = [e async for e in agent.run("hi")]
|
||||||
|
|
||||||
|
# Kept, not rolled back: the turn is unfinished, so retry continues it.
|
||||||
|
assert agent.pending_retry_text == "hi"
|
||||||
|
_ = [e async for e in agent.retry()]
|
||||||
|
assert any(notice_of_kind(m, TRUNCATED_NOTICE_KIND) for m in client.calls[1].messages)
|
||||||
|
assert agent.pending_retry_text is None
|
||||||
|
await store.close()
|
||||||
|
|
||||||
|
|
||||||
async def test_content_filter_finish_raises() -> None:
|
async def test_content_filter_finish_raises() -> None:
|
||||||
client = ScriptedClient(
|
client = ScriptedClient(
|
||||||
[
|
[
|
||||||
|
|||||||
@@ -2,7 +2,16 @@ from __future__ import annotations
|
|||||||
|
|
||||||
from datetime import UTC, datetime, timedelta, timezone
|
from datetime import UTC, datetime, timedelta, timezone
|
||||||
|
|
||||||
from plyngent.agent import ChatAgent, Notice, aside_notice, format_moment, has_notice, resume_notice, skill_tools_notice
|
from plyngent.agent import (
|
||||||
|
ChatAgent,
|
||||||
|
Notice,
|
||||||
|
aside_notice,
|
||||||
|
format_moment,
|
||||||
|
has_notice,
|
||||||
|
resume_notice,
|
||||||
|
skill_tools_notice,
|
||||||
|
truncated_turn_notice,
|
||||||
|
)
|
||||||
from plyngent.agent.chat import incomplete_turn_user_text
|
from plyngent.agent.chat import incomplete_turn_user_text
|
||||||
from plyngent.config.models import DatabaseConfig
|
from plyngent.config.models import DatabaseConfig
|
||||||
from plyngent.lmproto.openai_compatible.model import (
|
from plyngent.lmproto.openai_compatible.model import (
|
||||||
@@ -124,3 +133,26 @@ def test_trailing_developer_message_keeps_incomplete_turn_detectable() -> None:
|
|||||||
# A finished turn (trailing assistant) stays non-retryable past a notice.
|
# A finished turn (trailing assistant) stays non-retryable past a notice.
|
||||||
done = [UserChatMessage(content="do it"), AssistantChatMessage(content="ok"), notice]
|
done = [UserChatMessage(content="do it"), AssistantChatMessage(content="ok"), notice]
|
||||||
assert incomplete_turn_user_text(done) is None
|
assert incomplete_turn_user_text(done) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_truncated_turn_notice_keeps_the_partial_answer_retryable() -> None:
|
||||||
|
"""A kept partial answer is unfinished work, so retry continues it."""
|
||||||
|
tail = [
|
||||||
|
UserChatMessage(content="do it"),
|
||||||
|
AssistantChatMessage(content="partial answer"),
|
||||||
|
truncated_turn_notice().to_message(),
|
||||||
|
]
|
||||||
|
assert incomplete_turn_user_text(tail) == "do it"
|
||||||
|
# The same tail without the truncation notice is a finished turn.
|
||||||
|
assert incomplete_turn_user_text(tail[:2]) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_truncated_turn_notice_tells_the_model_where_it_stopped() -> None:
|
||||||
|
notice = truncated_turn_notice()
|
||||||
|
assert notice.kind == "truncated"
|
||||||
|
assert notice.to_message().content.startswith("[plyngent notice: truncated]")
|
||||||
|
assert "max_tokens" in notice.body
|
||||||
|
# The dropped call and the kept answer are both spelled out.
|
||||||
|
assert "discarded" in notice.body
|
||||||
|
assert "kept in the transcript" in notice.body
|
||||||
|
assert "max_tokens" in notice.summary
|
||||||
|
|||||||
@@ -2,7 +2,14 @@ from __future__ import annotations
|
|||||||
|
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
from plyngent.agent import ErrorEvent, ReasoningDeltaEvent, TextDeltaEvent, ToolCallEvent, ToolResultEvent
|
from plyngent.agent import (
|
||||||
|
ErrorEvent,
|
||||||
|
ReasoningDeltaEvent,
|
||||||
|
TextDeltaEvent,
|
||||||
|
ToolCallEvent,
|
||||||
|
ToolResultEvent,
|
||||||
|
TruncatedEvent,
|
||||||
|
)
|
||||||
from plyngent.cli.display import (
|
from plyngent.cli.display import (
|
||||||
_RUN_BATCH_CMD_LIMIT,
|
_RUN_BATCH_CMD_LIMIT,
|
||||||
_clear_streamed_lines,
|
_clear_streamed_lines,
|
||||||
@@ -1257,3 +1264,12 @@ async def test_pretty_prefix_closed_before_error_line(
|
|||||||
assert "* Read 'a.txt' \n" in out
|
assert "* Read 'a.txt' \n" in out
|
||||||
assert "[error] source=tool boom\n" in out
|
assert "[error] source=tool boom\n" in out
|
||||||
assert "* Read 'a.txt' (error)\n" in out
|
assert "* Read 'a.txt' (error)\n" in out
|
||||||
|
|
||||||
|
|
||||||
|
async def test_render_truncated_round(capsys: pytest.CaptureFixture[str]) -> None:
|
||||||
|
"""A round cut off at the output limit says so, and how the turn goes on."""
|
||||||
|
await render_events(_aiter([TruncatedEvent(kept_chars=42, continued=True)]))
|
||||||
|
assert "[limit] output truncated (42 chars kept); continuing\n" in capsys.readouterr().out
|
||||||
|
|
||||||
|
await render_events(_aiter([TruncatedEvent(kept_chars=42, continued=False)]))
|
||||||
|
assert "[limit] output truncated (42 chars kept); /retry continues from here\n" in capsys.readouterr().out
|
||||||
|
|||||||
Reference in New Issue
Block a user