diff --git a/src/plyngent/tools/net/fetch_local.py b/src/plyngent/tools/net/fetch_local.py index 9b448ac..3c004f2 100644 --- a/src/plyngent/tools/net/fetch_local.py +++ b/src/plyngent/tools/net/fetch_local.py @@ -9,7 +9,6 @@ from plyngent.tools.net.client import FetchResult, http_fetch from plyngent.tools.net.policy import ( DEFAULT_MAX_BODY_CHARS_IN, DEFAULT_MAX_BYTES, - DEFAULT_MAX_CHARS, DEFAULT_MAX_REDIRECTS, DEFAULT_TIMEOUT_SECONDS, FetchPolicyError, @@ -18,7 +17,6 @@ from plyngent.tools.net.policy import ( parse_fetch_url, soft_confirm_reason, ) -from plyngent.tools.truncate_token import truncate_to_memory from plyngent.tools.workspace import image_input_refusal @@ -35,15 +33,13 @@ def format_fetch_result( security: str, warnings: list[str], body_kind: str, - max_chars: int, offset: int = 0, ) -> str: start = max(0, offset) - # The body is already in hand, so the remainder is kept in memory rather than - # re-requested: resuming a fetch must not cost another round trip (latency) - # nor re-hit the remote (rate limits / risk control). - body, token = truncate_to_memory(body_text[start:], max_chars) - char_truncated = token is not None + # The body is kept whole in the tool message (the request-prep pass trims it + # on the wire and names its document), so resuming reads the message rather + # than re-hitting the remote. + body = body_text[start:] warn_line = "; ".join(warnings) if warnings else "" parts = [ f"status={status}", @@ -52,7 +48,7 @@ def format_fetch_result( f"content_type={content_type}", f"body_kind={body_kind}", f"bytes={body_bytes}", - f"truncated={'true' if truncated or char_truncated else 'false'}", + f"truncated={'true' if truncated else 'false'}", f"redirects={redirects}", f"security={security}", ] @@ -73,19 +69,18 @@ async def fetch_local( user_agent: str | None = None, timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS, max_bytes: int = DEFAULT_MAX_BYTES, - max_chars: int = DEFAULT_MAX_CHARS, max_redirects: int = DEFAULT_MAX_REDIRECTS, follow_redirects: bool = True, allow_http_downgrade: bool = False, offset: int = 0, ) -> str | ToolOutput: - """HTTP request (GET/POST/PUT/DELETE); return status, metadata, and truncated body. + """HTTP request (GET/POST/PUT/DELETE); return status, metadata, and the body. Use for public docs/APIs or (after human policy allow) local/LAN servers. ``offset`` skips that many leading characters of the body (0-based), for - paging through a body larger than ``max_chars``; a body truncated by - ``max_chars`` keeps its remainder in memory instead, resumable via its - ``truncate_token`` without another request. + paging through a body the wire cap trimmed; a trimmed body keeps its + remainder in the tool message, so its ``document_id`` reads the rest with no + new request. ``user_agent`` sets User-Agent when provided and takes precedence over a User-Agent entry in ``headers``. A headers-only User-Agent is kept as-is. When both omit UA, a small default is used. @@ -138,7 +133,6 @@ async def fetch_local( security=result.security, warnings=result.warnings, body_kind=result.body_kind, - max_chars=max_chars, offset=offset, ) return _attach_image(text, result) diff --git a/src/plyngent/tools/net/fetch_remote.py b/src/plyngent/tools/net/fetch_remote.py index 0a770fd..57371d3 100644 --- a/src/plyngent/tools/net/fetch_remote.py +++ b/src/plyngent/tools/net/fetch_remote.py @@ -13,8 +13,6 @@ import inspect from typing import TYPE_CHECKING from plyngent.agent.tools import ToolDefinition, ToolTag, schema_from_callable -from plyngent.tools.net.policy import DEFAULT_MAX_CHARS -from plyngent.tools.truncate_token import truncate_to_memory if TYPE_CHECKING: from plyngent.lmproto.deepseek.webtools import DeepseekWebToolsClient @@ -27,31 +25,28 @@ def format_page_result( title: str, url: str, content: str, - max_chars: int, offset: int = 0, ) -> str: """Render a fetched page the way the other structured tool results read.""" start = max(0, offset) - body, token = truncate_to_memory(content[start:], max_chars) + # The page is kept whole in the tool message (the request-prep pass trims it + # on the wire and names its document), so neither a re-fetch nor a charge + # follows a read of the rest. + body = content[start:] parts = [ f"title={' '.join(title.split()) or '(untitled)'}", f"url={url}", f"chars={len(content)}", - f"truncated={'true' if token is not None else 'false'}", "--- content ---", body, ] return "\n".join(parts) -def build_fetch_tool( - client: DeepseekWebToolsClient, - *, - default_max_chars: int = DEFAULT_MAX_CHARS, -) -> ToolDefinition: +def build_fetch_tool(client: DeepseekWebToolsClient) -> ToolDefinition: """A ``fetch`` definition backed by DeepSeek's Fetch API.""" - async def fetch(url: str, offset: int = 0, max_chars: int = default_max_chars) -> str: + async def fetch(url: str, offset: int = 0) -> str: """Read a public web page as Markdown through DeepSeek's Fetch API. Use for public articles and docs when clean Markdown is enough. The page @@ -60,9 +55,9 @@ def build_fetch_tool( when you need the raw HTTP status/headers. A successful fetch is billed per request (failures are not). - ``offset`` (0-based) skips leading characters of the content. A body cut - by ``max_chars`` keeps its remainder in memory, resumable via its - ``truncate_token`` without another request (and without another charge). + ``offset`` (0-based) skips leading characters of the content. The page is + kept whole in the tool message, so a trimmed ``document_id`` reads the + rest with no new request (and no new charge). """ try: page = await client.fetch(url) @@ -72,7 +67,6 @@ def build_fetch_tool( title=page.title, url=url, content=page.content, - max_chars=max_chars, offset=offset, ) diff --git a/src/plyngent/tools/net/policy.py b/src/plyngent/tools/net/policy.py index 10c8977..dee1ae0 100644 --- a/src/plyngent/tools/net/policy.py +++ b/src/plyngent/tools/net/policy.py @@ -36,7 +36,6 @@ DEFAULT_USER_AGENT = "plyngent-fetch/0.2" DEFAULT_MAX_REDIRECTS = 5 DEFAULT_TIMEOUT_SECONDS = 30.0 DEFAULT_MAX_BYTES = 1_000_000 -DEFAULT_MAX_CHARS = 32_000 DEFAULT_MAX_BODY_CHARS_IN = 256_000 # request body size limit (tool arg) # Cloud / link-local style targets never granted via policy UI. diff --git a/src/plyngent/tools/process/command_exec.py b/src/plyngent/tools/process/command_exec.py index a8daf3f..50d96fc 100644 --- a/src/plyngent/tools/process/command_exec.py +++ b/src/plyngent/tools/process/command_exec.py @@ -18,26 +18,9 @@ from plyngent.tools.workspace import ( ) DEFAULT_TIMEOUT_SECONDS = 30 -DEFAULT_MAX_OUTPUT_CHARS = 32_000 DEFAULT_MAX_BATCH_STEPS = 20 -def truncate_output(text: str, label: str, max_chars: int = DEFAULT_MAX_OUTPUT_CHARS) -> str: - """Cap a per-stream output block; embed a truncate token when cut. - - ``label`` (stdout/stderr/batch) is kept for call-site clarity but the - marker itself is the uniform ``[Truncated (N chars max.; M omitted). - truncate_token=...]`` (no label), matching every other truncation site. - Returns ``text`` unchanged when it fits. - """ - del label - if max_chars < 1 or len(text) <= max_chars: - return text - from plyngent.tools.truncate_token import truncate_generic - - return truncate_generic(text, max_chars) - - def format_command_result( *, returncode: int | None, @@ -53,9 +36,9 @@ def format_command_result( f"cwd={workdir_display}", f"cmd={shlex.join(command)}", "--- stdout ---", - truncate_output(stdout, "stdout"), + stdout, "--- stderr ---", - truncate_output(stderr, "stderr"), + stderr, ] return "\n".join(parts) diff --git a/src/plyngent/tools/process/run_argv_batch.py b/src/plyngent/tools/process/run_argv_batch.py index 2e1e88c..0078b11 100644 --- a/src/plyngent/tools/process/run_argv_batch.py +++ b/src/plyngent/tools/process/run_argv_batch.py @@ -9,13 +9,14 @@ from plyngent.tools.workspace import WorkspaceError from .command_exec import ( DEFAULT_MAX_BATCH_STEPS, - DEFAULT_MAX_OUTPUT_CHARS, DEFAULT_TIMEOUT_SECONDS, CommandStepResult, execute_argv, - truncate_output, ) +# Execution budget: stop the batch once accumulated output crosses this factor +# of the base budget, so a runaway batch cannot keep spawning huge-output steps. +BATCH_OUTPUT_BUDGET_CHARS = 32_000 BATCH_OUTPUT_SOFT_CAP_FACTOR = 4 @@ -102,9 +103,9 @@ def _format_step(index: int, result: CommandStepResult, *, pipe_out: bool) -> st f"pipe_out={'true' if pipe_out else 'false'}", f"mix_stderr={'true' if result.mix_stderr else 'false'}", "--- stdout ---", - truncate_output(result.stdout, "stdout"), + result.stdout, "--- stderr ---", - truncate_output(result.stderr, "stderr"), + result.stderr, ] ) @@ -147,7 +148,7 @@ async def _run_batch_steps( effective_stop = stop_on_error if step["stop_on_error"] is None else bool(step["stop_on_error"]) if failed and effective_stop: return results, True - if total_chars > DEFAULT_MAX_OUTPUT_CHARS * BATCH_OUTPUT_SOFT_CAP_FACTOR: + if total_chars > BATCH_OUTPUT_BUDGET_CHARS * BATCH_OUTPUT_SOFT_CAP_FACTOR: return results, True del index # used for clarity in loop only @@ -178,7 +179,9 @@ async def run_argv_batch( } ``pipe_out`` is set on the **provider** step. Last-step ``pipe_out`` is an error. - Default ``stop_on_error`` is true. Max 20 steps; total output budget 32k chars. + Default ``stop_on_error`` is true. Max 20 steps; the batch stops early once + accumulated output passes the run budget (128k chars). Each step's output is + kept whole for the model; a long result carries a ``document_id`` to read on. """ try: raw_steps = _normalize_steps(steps) @@ -195,8 +198,6 @@ async def run_argv_batch( body = "\n".join( _format_step(i, result, pipe_out=bool(step["pipe_out"])) for i, (step, result) in enumerate(results) ) - if len(body) > DEFAULT_MAX_OUTPUT_CHARS: - body = truncate_output(body, "batch") last_exit = results[-1][1].exit_code if results else "" return "\n".join( [ diff --git a/src/plyngent/tools/vcs/git_backend.py b/src/plyngent/tools/vcs/git_backend.py index 19b2e5f..8d78561 100644 --- a/src/plyngent/tools/vcs/git_backend.py +++ b/src/plyngent/tools/vcs/git_backend.py @@ -8,7 +8,6 @@ if TYPE_CHECKING: from pathlib import Path DEFAULT_TIMEOUT_SECONDS = 30.0 -DEFAULT_MAX_OUTPUT_CHARS = 64_000 def is_git_repo(root: Path) -> bool: @@ -32,13 +31,6 @@ class _GitResult: self.returncode = returncode -def _truncate(text: str) -> str: - if len(text) <= DEFAULT_MAX_OUTPUT_CHARS: - return text - omitted = len(text) - DEFAULT_MAX_OUTPUT_CHARS - return f"{text[:DEFAULT_MAX_OUTPUT_CHARS]}\n...[truncated {omitted} characters]" - - def _run_git(root: Path, args: list[str], *, timeout: float = DEFAULT_TIMEOUT_SECONDS) -> _GitResult: argv = ["git", "-c", "color.ui=false", *args] try: @@ -57,12 +49,12 @@ def _run_git(root: Path, args: list[str], *, timeout: float = DEFAULT_TIMEOUT_SE except OSError as exc: return _GitResult(ok=False, stdout="", stderr=str(exc), returncode=1) - stdout = _truncate(completed.stdout or "") - stderr = _truncate(completed.stderr or "") + # Output is kept whole; the request-prep pass trims it on the wire and names + # its document, so a long log/diff reads on rather than re-running git. return _GitResult( ok=completed.returncode == 0, - stdout=stdout, - stderr=stderr, + stdout=completed.stdout or "", + stderr=completed.stderr or "", returncode=completed.returncode, ) diff --git a/tests/test_tools/test_fetch_local.py b/tests/test_tools/test_fetch_local.py index 721e645..f5057bc 100644 --- a/tests/test_tools/test_fetch_local.py +++ b/tests/test_tools/test_fetch_local.py @@ -23,7 +23,6 @@ from plyngent.tools.net.policy import ( parse_fetch_url, soft_confirm_reason, ) -from plyngent.tools.truncate_token import decode_truncate_token from tests.test_tools.helpers import call_async if TYPE_CHECKING: @@ -299,37 +298,30 @@ async def test_fetch_truncation_and_binary(workspace: Path, http_server: str) -> assert "binary body omitted" in out_bin -async def test_fetch_char_truncation_embeds_token(workspace: Path, http_server: str) -> None: - del workspace - port = int(http_server.rsplit(":", 1)[1]) - grant_private_host("127.0.0.1", port) - - out = await call_async(fetch_local, f"{http_server}/big", max_chars=1000) - assert "truncated=true" in out - body = out.split("--- body ---", 1)[-1] - assert "truncate_token=" in body - token_line = next(ln for ln in body.splitlines() if "truncate_token=" in ln) - token_str = token_line.split("truncate_token=", 1)[1].rstrip("]") - token = decode_truncate_token(token_str) - assert token is not None - # The body is kept in memory, so resuming never repeats the request. - assert token.kind == "memory" - assert token.limit == 1000 - assert 0 < token.offset < 1000 - - async def test_fetch_offset_skips_body(workspace: Path, http_server: str) -> None: del workspace port = int(http_server.rsplit(":", 1)[1]) grant_private_host("127.0.0.1", port) - # /big body is 5000 chars; offset 4900 leaves 100 chars, no token needed. - out = await call_async(fetch_local, f"{http_server}/big", max_chars=200, offset=4900) + # /big body is 5000 chars; offset 4900 leaves the last 100 chars. + out = await call_async(fetch_local, f"{http_server}/big", offset=4900) body = out.split("--- body ---", 1)[-1] assert body.strip() == "x" * 100 assert "truncate_token=" not in out +async def test_fetch_keeps_the_body_whole(workspace: Path, http_server: str) -> None: + """The body is stored whole (the request-prep pass caps it on the wire).""" + del workspace + port = int(http_server.rsplit(":", 1)[1]) + grant_private_host("127.0.0.1", port) + + out = await call_async(fetch_local, f"{http_server}/big") + body = out.split("--- body ---", 1)[-1].strip() + assert body == "x" * 5000 + assert "truncate_token=" not in out + + async def test_fetch_private_denied_without_grant(workspace: Path, http_server: str) -> None: del workspace # No grant, no policy hook → hard deny (even though server is up). diff --git a/tests/test_tools/test_fetch_remote.py b/tests/test_tools/test_fetch_remote.py index 882420d..4a39ebc 100644 --- a/tests/test_tools/test_fetch_remote.py +++ b/tests/test_tools/test_fetch_remote.py @@ -24,8 +24,8 @@ class _Client: return self.page -def _tool(client: _Client, *, max_chars: int = 32_000): - return build_fetch_tool(client, default_max_chars=max_chars) +def _tool(client: _Client): + return build_fetch_tool(client) def test_definition_shape() -> None: @@ -33,7 +33,7 @@ def test_definition_shape() -> None: assert definition.name == FETCH_TOOL_NAME == "fetch" assert definition.tags == ToolTag.LOCAL | ToolTag.READ_ONLY properties = definition.parameters["properties"] - assert set(properties) == {"url", "offset", "max_chars"} + assert set(properties) == {"url", "offset"} assert definition.parameters["required"] == ["url"] assert "fetch_local" in definition.description @@ -48,7 +48,6 @@ async def test_handler_renders_title_url_and_content() -> None: "title=Example article", "url=https://example.com/article", "chars=16", - "truncated=false", "--- content ---", "# Heading", "", @@ -57,14 +56,13 @@ async def test_handler_renders_title_url_and_content() -> None: @pytest.mark.asyncio -async def test_offset_and_truncation_keep_the_remainder_resumable() -> None: +async def test_offset_skips_leading_content() -> None: client = _Client(page=FetchResponse(title="", content="0123456789")) - result = await _tool(client, max_chars=3).handler(url="https://example.com/", offset=5) + result = await _tool(client).handler(url="https://example.com/", offset=5) assert "title=(untitled)" in result - assert "--- content ---\n567" in result - assert "truncated=true" in result - assert "truncate_token=" in result + assert "--- content ---\n56789" in result + assert "truncate_token=" not in result @pytest.mark.asyncio diff --git a/tests/test_tools/test_get_truncated.py b/tests/test_tools/test_get_truncated.py index 02a7311..31afe5a 100644 --- a/tests/test_tools/test_get_truncated.py +++ b/tests/test_tools/test_get_truncated.py @@ -84,42 +84,6 @@ async def test_get_truncated_file_missing(workspace: Path) -> None: assert out.startswith("error:") -async def test_get_truncated_serves_a_fetch_body_without_refetching(monkeypatch) -> None: - """A fetch body's token reads the stored remainder; the request is not repeated.""" - from plyngent.tools.net.fetch_local import format_fetch_result - - fetch_module = importlib.import_module("plyngent.tools.net.fetch_local") - - async def _must_not_refetch(*_args: object, **_kwargs: object) -> str: - msg = "resuming a fetch body must not re-run the request" - raise AssertionError(msg) - - monkeypatch.setattr(fetch_module, "http_fetch", _must_not_refetch) - first = format_fetch_result( - status=200, - final_url="https://example.com/x", - content_type="text/plain", - body_text="x" * 600, - body_bytes=600, - truncated=False, - redirects=0, - method="GET", - security="public", - warnings=[], - body_kind="text", - max_chars=200, - ) - token = _extract_token(first) - assert token is not None - parsed = decode_truncate_token(token) - assert parsed is not None - assert parsed.kind == "memory" - out = await call_async(get_truncated, token, max_chars=200) - content, _, _ = out.partition("\n[Truncated") - assert content == "x" * len(content) - assert 0 < len(content) <= 200 - - async def test_get_truncated_memory_chains() -> None: from plyngent.tools.truncate_token import truncate_generic diff --git a/tests/test_tools/test_process.py b/tests/test_tools/test_process.py index 0a9908e..7c6ca21 100644 --- a/tests/test_tools/test_process.py +++ b/tests/test_tools/test_process.py @@ -192,14 +192,14 @@ async def test_run_argv_timeout_keeps_partial_output(workspace: object) -> None: assert "partial-out" in out -async def test_run_argv_stdout_cap_embeds_token(workspace: object) -> None: +async def test_run_argv_returns_the_full_stdout(workspace: object) -> None: del workspace - # Per-stream cap default is 32000 chars: a bigger stdout embeds a token. + # The result is stored whole; the request-prep pass caps it on the wire and + # names its document, so nothing is cut at the tool. out = await call_async(run_argv, _py("print('x' * 40000)")) - assert "truncate_token=" in out - token_line = next(ln for ln in out.splitlines() if "truncate_token=" in ln) - assert "[Truncated (32000 chars max.;" in token_line - assert "[Truncated" in out + body = out.split("--- stdout ---", 1)[1].split("--- stderr ---", 1)[0].strip() + assert body == "x" * 40000 + assert "truncate_token=" not in out async def test_run_argv_small_stdout_no_marker(workspace: object) -> None: