test: cover the approval line and the policy records

Registry cases (none / once / round / grant / YOLO / denial / error / handler
record / closest answer wins), the wire cap keeping the line, the three policy
paths that record their answer, and a pretty detail that ignores the header.
The existing confirm tests now assert the line they gain.
This commit is contained in:
2026-10-29 18:29:28 +08:00
parent ed8013d6ce
commit 6d7c239718
9 changed files with 298 additions and 20 deletions
+1 -1
View File
@@ -33,7 +33,7 @@ async def test_confirm_allow() -> None:
danger=classify_danger,
on_confirm=on_confirm,
)
assert await registry.execute("delete_path", '{"path": "a.txt"}') == "deleted a.txt"
assert await registry.execute("delete_path", '{"path": "a.txt"}') == "[approval] user (once)\ndeleted a.txt"
assert len(decisions) == 1
assert await registry.execute("read_file", '{"path": "a.txt"}') == "read a.txt"
assert len(decisions) == 1
+1 -1
View File
@@ -253,5 +253,5 @@ async def test_a_write_holds_its_file_across_the_confirm() -> None:
await asyncio.sleep(0) # the second write waits for the file, not for the human
assert order == ["confirm write_a"]
answered.set()
assert await asyncio.gather(first, second) == ["a", "b"]
assert await asyncio.gather(first, second) == ["[approval] user (once)\na", "b"]
assert order == ["confirm write_a", "a", "b"]
+154
View File
@@ -0,0 +1,154 @@
"""The ``[approval]`` line: a confirmed call says how it got through."""
from __future__ import annotations
from typing import TYPE_CHECKING
from plyngent.agent import (
Approval,
BatchConfirmDecision,
ToolRegistry,
ToolTag,
format_approval,
most_specific_approval,
tool,
)
from plyngent.agent.budget import cap_tool_message
from plyngent.lmproto.openai_compatible.model import ToolChatMessage
from plyngent.tools.context import SessionState, bind_tool_context, record_approval
if TYPE_CHECKING:
from collections.abc import Mapping, Sequence
from plyngent.agent import ToolConfirmRequest
def _danger(name: str, _args: Mapping[str, object]) -> str | None:
return f"mutate {name}"
def _allow(_name: str, _args: Mapping[str, object], _reason: str) -> bool:
return True
@tool(register=False)
async def quiet() -> str:
return "quiet"
@tool(register=False)
async def mutate() -> str:
return "ran"
async def test_a_call_that_needs_no_confirmation_carries_no_line() -> None:
"""The common case stays byte-for-byte what the tool returned."""
def danger(_name: str, _args: Mapping[str, object]) -> str | None:
return None
def on_confirm(_name: str, _args: Mapping[str, object], _reason: str) -> bool:
msg = "no prompt expected"
raise AssertionError(msg)
reg = ToolRegistry([quiet], danger=danger, on_confirm=on_confirm)
assert await reg.execute("quiet", "{}") == "quiet"
async def test_a_single_prompt_leads_the_result() -> None:
reg = ToolRegistry([mutate], danger=_danger, on_confirm=_allow)
assert await reg.execute("mutate", "{}") == "[approval] user (once)\nran"
async def test_yolo_mode_says_it_dropped_the_prompt() -> None:
@tool(tags=ToolTag.LOCAL | ToolTag.YOLO, register=False)
async def yolo_ok() -> str:
return "y"
reg = ToolRegistry([yolo_ok], danger=_danger, on_confirm=_allow, yolo=True)
assert await reg.execute("yolo_ok", "{}") == "[approval] yolo (auto)\ny"
async def test_a_batch_answer_covers_its_round() -> None:
def on_batch(_requests: Sequence[ToolConfirmRequest]) -> BatchConfirmDecision:
return BatchConfirmDecision(allowed=True)
reg = ToolRegistry([mutate], danger=_danger, on_confirm_batch=on_batch)
calls = [("mutate", "{}"), ("mutate", "{}")]
decisions = await reg.prepare_batch_confirm(calls)
results = [
await reg.execute(name, arguments, decision=decisions[index]) for index, (name, arguments) in enumerate(calls)
]
assert results == ["[approval] user (round)\nran", "[approval] user (round)\nran"]
async def test_a_session_grant_says_nobody_was_asked_this_time() -> None:
@tool(tags=ToolTag.LOCAL | ToolTag.TRUSTABLE, register=False)
async def trust_me() -> str:
return "ok"
prompts: list[str] = []
def on_confirm(name: str, _args: Mapping[str, object], _reason: str) -> bool:
prompts.append(name)
return True
session = SessionState(session_id=1)
reg = ToolRegistry([trust_me], danger=_danger, on_confirm=on_confirm, session_state=session)
with bind_tool_context(session=session):
assert await reg.execute("trust_me", "{}") == "[approval] user (once)\nok"
assert await reg.execute("trust_me", "{}") == "[approval] grant (session)\nok"
assert prompts == ["trust_me"]
async def test_a_denied_call_carries_no_line() -> None:
reg = ToolRegistry([mutate], danger=_danger, on_confirm=lambda *_args: False)
out = await reg.execute("mutate", "{}")
assert out.startswith("error: ")
assert "[approval]" not in out
async def test_a_failed_call_carries_no_line() -> None:
"""An approved call that failed is an error first; the answer explains nothing."""
@tool(register=False)
async def boom() -> str:
return "error: boom"
reg = ToolRegistry([boom], danger=_danger, on_confirm=_allow)
assert await reg.execute("boom", "{}") == "error: boom"
async def test_a_handler_record_leads_the_result() -> None:
@tool(register=False)
async def policy_tool() -> str:
record_approval(Approval("user", "process"))
return "ran"
reg = ToolRegistry([policy_tool])
assert await reg.execute("policy_tool", "{}") == "[approval] user (process)\nran"
async def test_the_closest_answer_wins() -> None:
"""YOLO let the call run, but a human answered a policy prompt inside it."""
@tool(tags=ToolTag.LOCAL | ToolTag.YOLO, register=False)
async def yolo_policy() -> str:
record_approval(Approval("user", "process"))
return "ran"
reg = ToolRegistry([yolo_policy], danger=_danger, on_confirm=_allow, yolo=True)
assert await reg.execute("yolo_policy", "{}") == "[approval] user (process)\nran"
async def test_the_line_survives_the_wire_cap() -> None:
approved = f"{format_approval(Approval('user', 'once'))}\n{'x' * 500}"
capped = cap_tool_message(ToolChatMessage(content=approved, tool_call_id="call-1"), 200)
assert capped.content.startswith("[approval] user (once)\n")
assert "document_id=call-1" in capped.content
def test_the_most_specific_record_wins() -> None:
records = [Approval("yolo", "auto"), Approval("grant", "session"), Approval("user", "once")]
assert most_specific_approval(records) == Approval("user", "once")
assert most_specific_approval([]) is None
+31 -8
View File
@@ -26,6 +26,11 @@ def _danger(name: str, _args: Mapping[str, object]) -> str | None:
return f"mutate {name}"
def _approved(scope: str, text: str) -> str:
"""A result as the model reads it once a confirmation let the call run."""
return f"[approval] {scope}\n{text}"
async def test_loop_round_prompts_once_before_running_its_calls() -> None:
"""The loop aggregates a round's confirms and applies one answer to each call."""
batch_prompts: list[int] = []
@@ -54,7 +59,10 @@ async def test_loop_round_prompts_once_before_running_its_calls() -> None:
assert batch_prompts == [2]
assert single_prompts == []
assert [getattr(message, "content", None) for message in messages] == ["stamped a", "stamped b"]
assert [getattr(message, "content", None) for message in messages] == [
_approved("user (round)", "stamped a"),
_approved("user (round)", "stamped b"),
]
assert len(events) == len(tool_calls) * 2 # call event + result event per call
@@ -88,7 +96,11 @@ async def test_round_asks_once_for_many_mutating_calls() -> None:
results = [
await reg.execute(name, arguments, decision=decisions[index]) for index, (name, arguments) in enumerate(calls)
]
assert results == ["stamped a", "stamped b", "stamped c"]
assert results == [
_approved("user (round)", "stamped a"),
_approved("user (round)", "stamped b"),
_approved("user (round)", "stamped c"),
]
assert ran == ["a", "b", "c"]
assert single_prompts == []
@@ -129,7 +141,11 @@ async def test_yolo_calls_are_left_out_of_the_prompt() -> None:
results = [
await reg.execute(name, arguments, decision=decisions[index]) for index, (name, arguments) in enumerate(calls)
]
assert results == ["stamped a", "stamped b", "stamped c"]
assert results == [
_approved("yolo (auto)", "stamped a"),
_approved("user (round)", "stamped b"),
_approved("user (round)", "stamped c"),
]
assert single_prompts == []
@@ -153,7 +169,7 @@ async def test_single_request_keeps_the_one_call_prompt() -> None:
reg = ToolRegistry([stamp], danger=_danger, on_confirm=on_single, on_confirm_batch=on_batch)
assert await reg.prepare_batch_confirm([("stamp", '{"path": "a"}')]) == [None]
assert batch_prompts == []
assert await reg.execute("stamp", '{"path": "a"}') == "stamped a"
assert await reg.execute("stamp", '{"path": "a"}') == _approved("user (once)", "stamped a")
assert single_prompts == ["stamp"]
@@ -254,7 +270,10 @@ async def test_manual_hands_the_round_back_to_the_per_call_prompt() -> None:
decisions = await reg.prepare_batch_confirm(calls)
assert decisions == [None, None]
assert [await reg.execute(name, arguments) for name, arguments in calls] == ["stamped a", "stamped b"]
assert [await reg.execute(name, arguments) for name, arguments in calls] == [
_approved("user (once)", "stamped a"),
_approved("user (once)", "stamped b"),
]
assert batch_prompts == [2]
assert single_prompts == ["stamp", "stamp"]
@@ -274,7 +293,10 @@ async def test_without_a_batch_hook_every_call_prompts_on_its_own() -> None:
reg = ToolRegistry([stamp], danger=_danger, on_confirm=on_single)
calls = [("stamp", '{"path": "a"}'), ("stamp", '{"path": "b"}')]
assert await reg.prepare_batch_confirm(calls) == [None, None]
assert [await reg.execute(name, arguments) for name, arguments in calls] == ["stamped a", "stamped b"]
assert [await reg.execute(name, arguments) for name, arguments in calls] == [
_approved("user (once)", "stamped a"),
_approved("user (once)", "stamped b"),
]
assert single_prompts == ["stamp", "stamp"]
@@ -306,8 +328,9 @@ async def test_batch_allow_records_the_trustable_grant() -> None:
calls = [("trust_me", '{"path": "a"}'), ("trust_me", '{"path": "b"}')]
decisions = await reg.prepare_batch_confirm(calls)
for path, decision in zip(("a", "b"), decisions, strict=True):
assert await reg.execute("trust_me", f'{{"path": "{path}"}}', decision=decision) == f"done {path}"
approve = _approved("user (round)", f"done {path}")
assert await reg.execute("trust_me", f'{{"path": "{path}"}}', decision=decision) == approve
assert has_grant(session, "trust_me")
# The TRUSTABLE grant, not the batch answer, keeps later calls unchecked.
assert await reg.execute("trust_me", '{"path": "c"}') == "done c"
assert await reg.execute("trust_me", '{"path": "c"}') == _approved("grant (session)", "done c")
assert single_prompts == []