diff --git a/src/plyngent/cli/state.py b/src/plyngent/cli/state.py index eb3fb6a..37b0e5f 100644 --- a/src/plyngent/cli/state.py +++ b/src/plyngent/cli/state.py @@ -15,6 +15,7 @@ if TYPE_CHECKING: from plyngent.agent.tools import ToolDefinition from plyngent.agent.types import AnyLLMClient + from plyngent.tools.net.search import SearchBackend from plyngent.cli.models_source import ( DEFAULT_MODELS_CACHE_TTL, client_supports_models, @@ -416,9 +417,8 @@ class ReplState: def _web_search_tool(self) -> ToolDefinition | None: """Local ``web_search`` for a provider whose surface cannot host one. - DeepSeek searches only on its Anthropic-compatible surface, so the other - conventions get the tool instead of a hosted definition (see - :func:`~plyngent.config.routing.local_web_search`). + The sources come from ``provider.search_sources``, in fallback order + (see :func:`~plyngent.config.routing.local_web_search`). """ from plyngent.config.routing import ( deepseek_search_base_url, @@ -426,18 +426,34 @@ class ReplState: ) from plyngent.lmproto.anthropic.config import AnthropicConfig from plyngent.lmproto.deepseek.anthropic.search import DeepseekSearchClient - from plyngent.tools.net.search import build_web_search_tool + from plyngent.tools.net.bing import BingSearchBackend + from plyngent.tools.net.search import ( + DeepseekSearchBackend, + build_web_search_tool, + ) provider = self.effective_provider if not local_web_search(provider) or not self.model: return None - client = DeepseekSearchClient( - AnthropicConfig( - api_key=provider.access_key_or_token, - base_url=deepseek_search_base_url(provider), - ) - ) - return build_web_search_tool(client, model=self.model) + backends: list[SearchBackend] = [] + for source in provider.search_sources: + if source == "deepseek": + backends.append( + DeepseekSearchBackend( + DeepseekSearchClient( + AnthropicConfig( + api_key=provider.access_key_or_token, + base_url=deepseek_search_base_url(provider), + ) + ), + model=self.model, + ) + ) + elif source == "bing": + backends.append(BingSearchBackend()) + if not backends: + return None + return build_web_search_tool(backends) def _mcp_instructions_text(self) -> str: """Compose connected MCP servers' initialize ``instructions`` into one block. diff --git a/src/plyngent/config/models.py b/src/plyngent/config/models.py index 54e38b8..6fcfc4a 100644 --- a/src/plyngent/config/models.py +++ b/src/plyngent/config/models.py @@ -9,6 +9,8 @@ from msgspec import Struct, field # - "anthropic" → Anthropic Messages API (``POST /messages`` on # ``https://api.deepseek.com/anthropic``) type DeepSeekConvention = Literal["", "openai", "anthropic", "responses"] +# Sources the local ``web_search`` tool can use (see ``SearchBackend``). +type SearchSource = Literal["deepseek", "bing"] # Static directory pre-allow mode for ``[agent].allow_paths`` (path → mode). type AllowPathMode = Literal["read", "write", "exec"] @@ -321,6 +323,9 @@ class ProviderConfig(Struct, tag_field="preset", omit_defaults=True): # Thinking strength for every model of this provider ("" / 0 = inherit [agent]). reasoning_effort: ReasoningEffortConfig = "" thinking_budget_tokens: int = 0 + # Sources the local ``web_search`` tool tries, in fallback order (``[]`` + # disables the tool; see ``config.routing.local_web_search``). + search_sources: list[SearchSource] = field(default_factory=list) def _default_openai_models() -> dict[str, ModelConfig]: @@ -371,6 +376,15 @@ def _default_deepseek_models() -> dict[str, ModelConfig]: } +def _default_deepseek_search_sources() -> list[SearchSource]: + """Sources for the local ``web_search`` tool, in fallback order. + + DeepSeek's own index first (it is the provider's own search), Bing's RSS as + the backstop when a search fails or finds nothing. + """ + return ["deepseek", "bing"] + + class DeepseekProvider(ProviderConfig, tag="deepseek"): """DeepSeek API provider. @@ -383,7 +397,9 @@ class DeepseekProvider(ProviderConfig, tag="deepseek"): ``provider_tools`` asks for a search tool like OpenAI's does (``[]`` disables it). DeepSeek runs a hosted search only on the Anthropic surface, so the other conventions get a local ``web_search`` tool instead (see - ``config.routing.local_web_search``). + ``config.routing.local_web_search``); ``search_sources`` names what that tool + searches with, in fallback order — ``["deepseek", "bing"]`` by default, + ``[]`` to not offer it. ``extras`` keeps arbitrary provider keys; the legacy ``extras.convention`` key is parsed but ignored (prefer the typed ``convention`` field). @@ -392,6 +408,7 @@ class DeepseekProvider(ProviderConfig, tag="deepseek"): models: dict[str, ModelConfig] = field(default_factory=_default_deepseek_models) convention: DeepSeekConvention = "" provider_tools: list[dict[str, Any]] = field(default_factory=_default_provider_tools) + search_sources: list[SearchSource] = field(default_factory=_default_deepseek_search_sources) extras: dict[str, str] = field(default_factory=dict) diff --git a/src/plyngent/config/routing.py b/src/plyngent/config/routing.py index d96a78b..b0726b1 100644 --- a/src/plyngent/config/routing.py +++ b/src/plyngent/config/routing.py @@ -33,6 +33,8 @@ class EffectiveProvider: model: str | None timeout: float | HttpTimeoutConfig | None provider_tools: list[dict[str, object]] + # Sources the local ``web_search`` tool tries, in fallback order. + search_sources: list[str] source: Provider # DeepSeek API surface (see DeepSeekConvention): "openai" / "anthropic" / # "responses" for deepseek presets; "" (unused) for other presets. @@ -101,11 +103,13 @@ def local_web_search(effective: EffectiveProvider) -> bool: ignore a hosted tool — a live probe answers "I can't browse the web" with either ``{"type": "web_search"}`` or the server-tool shape — so a DeepSeek provider that talks them searches through - :func:`~plyngent.tools.net.search.build_web_search_tool`, which calls the - search-capable endpoint behind the scenes. + :func:`~plyngent.tools.net.search.build_web_search_tool`, which asks the + sources named by ``search_sources`` behind the scenes. """ if effective.preset != "deepseek" or effective.convention == "anthropic": return False + if not effective.search_sources: + return False return wants_web_search(effective) @@ -147,13 +151,15 @@ def resolve_effective_provider(provider: Provider, *, model: str | None = None) else: url = default_url_for_preset(preset) - # Hosted tools belong to the preset that declares them: OpenAI's Responses - # surface, and DeepSeek's (whose Anthropic surface runs the search inside the - # response while its other surfaces get a local tool — see + # Hosted tools and search sources belong to the preset that declares them: + # OpenAI's Responses surface, and DeepSeek's (whose Anthropic surface runs the + # search inside the response while its other surfaces get a local tool — see # :func:`local_web_search`). provider_tools: list[dict[str, object]] = [] + search_sources: list[str] = [] if parent_preset == preset and isinstance(provider, OpenAIProvider | DeepseekProvider): provider_tools = [dict(item) for item in provider.provider_tools] + search_sources = [str(item) for item in provider.search_sources] return EffectiveProvider( preset=preset, @@ -162,6 +168,7 @@ def resolve_effective_provider(provider: Provider, *, model: str | None = None) model=model, timeout=provider.timeout, provider_tools=provider_tools, + search_sources=search_sources, source=provider, convention=convention, ) diff --git a/src/plyngent/lmproto/deepseek/anthropic/search.py b/src/plyngent/lmproto/deepseek/anthropic/search.py index c6da305..f226d95 100644 --- a/src/plyngent/lmproto/deepseek/anthropic/search.py +++ b/src/plyngent/lmproto/deepseek/anthropic/search.py @@ -34,10 +34,16 @@ _SEARCH_TOOL: dict[str, Any] = {"type": "web_search_20250305", "name": "web_sear class SearchHit(msgspec.Struct, frozen=True, omit_defaults=True): - """One page the search index matched.""" + """One page a search source matched. + + DeepSeek's index keeps the page bodies server-side, so its hits carry a + title and a URL only; a source that ships snippets (Bing's RSS) fills + ``snippet`` too. + """ title: str = "" url: str = "" + snippet: str = "" def hits_from_content(content: object) -> list[SearchHit]: diff --git a/src/plyngent/tools/net/bing.py b/src/plyngent/tools/net/bing.py new file mode 100644 index 0000000..43c81b5 --- /dev/null +++ b/src/plyngent/tools/net/bing.py @@ -0,0 +1,66 @@ +"""Bing's RSS endpoint as a search source for the local ``web_search`` tool. + +Bing answers a query with a plain RSS document (``?format=rss``): no key, no +page scraping, and each ```` already carries a title, a link and a +description (the snippet DeepSeek's own index does not hand out). The request +goes through the fetch client the ``fetch`` tool uses, so the SSRF and +private-host grant policy applies to it as well. +""" + +from __future__ import annotations + +import urllib.parse +import xml.etree.ElementTree as ET + +from plyngent.lmproto.deepseek.anthropic.search import SearchHit +from plyngent.tools.net.client import http_fetch + +SEARCH_URL = "https://www.bing.com/search" +DEFAULT_TIMEOUT_SECONDS = 30.0 +DEFAULT_MAX_BYTES = 2_000_000 +_HTTP_ERROR = 400 + + +def parse_bing_rss(document: str) -> list[SearchHit]: + """Map Bing's RSS items to hits (title, link, description as the snippet).""" + try: + root = ET.fromstring(document) + except ET.ParseError as exc: + msg = f"bing search returned unparseable XML: {exc}" + raise RuntimeError(msg) from exc + hits: list[SearchHit] = [] + for item in root.iter("item"): + title = (item.findtext("title") or "").strip() + url = (item.findtext("link") or "").strip() + if not url: + continue + hits.append(SearchHit(title=title, url=url, snippet=(item.findtext("description") or "").strip())) + return hits + + +class BingSearchBackend: + """Search through Bing's RSS results (no key, snippets included).""" + + timeout_seconds: float + + def __init__(self, *, timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS) -> None: + self.timeout_seconds = timeout_seconds + + async def search(self, query: str) -> list[SearchHit]: + """Run *query* against Bing and return the items it answered with.""" + query_string = urllib.parse.urlencode({"format": "rss", "q": query}) + result = await http_fetch( + method="GET", + url=f"{SEARCH_URL}?{query_string}", + headers={}, + body=None, + timeout_seconds=self.timeout_seconds, + max_bytes=DEFAULT_MAX_BYTES, + ) + if result.status >= _HTTP_ERROR: + msg = f"bing search HTTP {result.status}" + raise RuntimeError(msg) + return parse_bing_rss(result.body_text) + + +__all__ = ["SEARCH_URL", "BingSearchBackend", "parse_bing_rss"] diff --git a/src/plyngent/tools/net/search.py b/src/plyngent/tools/net/search.py index eb1d4ee..2474582 100644 --- a/src/plyngent/tools/net/search.py +++ b/src/plyngent/tools/net/search.py @@ -3,43 +3,99 @@ DeepSeek only searches on its Anthropic-compatible surface, so a DeepSeek provider that talks the Responses or chat-completions surface gets this local tool instead of a hosted one (see ``config.routing.local_web_search``): the -handler asks the search-capable endpoint behind the scenes and hands the model -the pages it matched. The index keeps the page bodies, so a hit is a title plus a -URL — read one with ``fetch`` before relying on it. +model calls it like any other tool and the handler asks the configured search +sources behind the scenes — ``provider.search_sources``, in fallback order, so +the next source is tried when one fails or finds nothing. A hit is a title, a +URL and (when the source has one) a snippet: read a page with ``fetch`` before +relying on its content. """ from __future__ import annotations import inspect -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Protocol from plyngent.agent.tools import ToolDefinition, ToolTag, schema_from_callable if TYPE_CHECKING: - from plyngent.lmproto.deepseek.anthropic.search import DeepseekSearchClient + from collections.abc import Sequence + + from plyngent.lmproto.deepseek.anthropic.search import ( + DeepseekSearchClient, + SearchHit, + ) WEB_SEARCH_TOOL_NAME = "web_search" DEFAULT_MAX_HITS = 8 +class SearchBackend(Protocol): + """One way to answer a query (``provider.search_sources`` picks the order).""" + + async def search(self, query: str) -> Sequence[SearchHit]: ... + + +class DeepseekSearchBackend: + """DeepSeek's own index: one server-side search turn per query.""" + + _client: DeepseekSearchClient + _model: str + + def __init__(self, client: DeepseekSearchClient, *, model: str) -> None: + self._client = client + self._model = model + + async def search(self, query: str) -> Sequence[SearchHit]: + return await self._client.search(query, model=self._model) + + +async def search_in_order( + backends: Sequence[SearchBackend], + query: str, +) -> tuple[list[SearchHit], list[str]]: + """Ask each backend in turn; the first one that answers wins. + + Returns ``(hits, failures)`` — a source that raises or finds nothing is + recorded and the next one gets its chance, so a broken or rate-limited + source never takes the tool down with it. + """ + failures: list[str] = [] + for backend in backends: + name = type(backend).__name__ + try: + hits = list(await backend.search(query)) + except Exception as exc: # noqa: BLE001 — a failing source falls through + failures.append(f"{name}: {exc}") + continue + if hits: + return hits, failures + failures.append(f"{name}: no results") + return [], failures + + def build_web_search_tool( - client: DeepseekSearchClient, + backends: Sequence[SearchBackend], *, - model: str, max_hits: int = DEFAULT_MAX_HITS, ) -> ToolDefinition: - """A ``web_search`` definition over *client* (one per provider/session).""" + """A ``web_search`` definition over *backends*, tried in order.""" async def web_search(query: str, hits: int = max_hits) -> str: - """Search the web with DeepSeek and return the pages that matched. + """Search the web and return the pages that matched. - One line per hit (``title — url``, the search index keeps the page bodies - on its side): read a page with ``fetch`` before relying on its content. + One entry per hit (``title — url``, plus the snippet when the source has + one): read a page with ``fetch`` before relying on its content. """ - found = await client.search(query, model=model) + found, failures = await search_in_order(backends, query) if not found: - return "error: the search returned no results" - return "\n".join(f"- {hit.title or '(untitled)'} — {hit.url}" for hit in found[:hits]) + detail = "; ".join(failures) if failures else "no source configured" + return f"error: the search found nothing ({detail})" + lines: list[str] = [] + for hit in found[:hits]: + lines.append(f"- {hit.title or '(untitled)'} — {hit.url}") + if hit.snippet: + lines.append(f" {hit.snippet}") + return "\n".join(lines) return ToolDefinition( name=WEB_SEARCH_TOOL_NAME, @@ -50,4 +106,11 @@ def build_web_search_tool( ) -__all__ = ["DEFAULT_MAX_HITS", "WEB_SEARCH_TOOL_NAME", "build_web_search_tool"] +__all__ = [ + "DEFAULT_MAX_HITS", + "WEB_SEARCH_TOOL_NAME", + "DeepseekSearchBackend", + "SearchBackend", + "build_web_search_tool", + "search_in_order", +] diff --git a/tests/test_config/test_routing.py b/tests/test_config/test_routing.py index e164f23..8f6f343 100644 --- a/tests/test_config/test_routing.py +++ b/tests/test_config/test_routing.py @@ -21,10 +21,15 @@ def _effective( convention: str = "", url: str = "", tools: list[dict[str, object]] | None = None, + sources: list[str] | None = None, ) -> EffectiveProvider: provider = DeepseekProvider(access_key_or_token="sk-test", convention=convention, url=url) effective = resolve_effective_provider(provider, model="deepseek-flash") - return effective if tools is None else replace(effective, provider_tools=tools) + if tools is not None: + effective = replace(effective, provider_tools=tools) + if sources is not None: + effective = replace(effective, search_sources=sources) + return effective def test_a_deepseek_provider_asks_for_a_search_tool_like_openai_does() -> None: @@ -45,6 +50,14 @@ def test_without_provider_tools_there_is_no_search() -> None: assert local_web_search(_effective(convention="responses", tools=[])) is False +def test_without_search_sources_there_is_no_local_tool() -> None: + # ``search_sources = []`` leaves the tool out even though provider_tools ask + # for a search (the hosted path on the Anthropic convention is unaffected). + assert _effective(convention="responses", sources=[]).search_sources == [] + assert local_web_search(_effective(convention="responses", sources=[])) is False + assert _effective(convention="responses").search_sources == ["deepseek", "bing"] + + def test_search_base_url_prefers_an_anthropic_endpoint() -> None: assert deepseek_search_base_url(_effective(convention="responses")) == "https://api.deepseek.com/anthropic" assert ( diff --git a/tests/test_tools/test_bing.py b/tests/test_tools/test_bing.py new file mode 100644 index 0000000..3587805 --- /dev/null +++ b/tests/test_tools/test_bing.py @@ -0,0 +1,45 @@ +"""Bing's RSS answers as a search source (parsing only — no network).""" + +from __future__ import annotations + +import pytest + +from plyngent.lmproto.deepseek.anthropic.search import SearchHit +from plyngent.tools.net.bing import parse_bing_rss + +DOCUMENT = """ + + 必应：DeepSeek Harness + + DeepSeek Harness v0.2 + https://pandaily.com/deepseek-harness-v0-2 + Desktop installers and a plugin manager. + Sat, 03 Oct 2026 12:06:00 GMT + + + No link here + Dropped: a hit without a URL is nothing to read. + + + Spaced title + https://example.com/spaced + + +""" + + +def test_parse_bing_rss_maps_items_to_hits() -> None: + hits = parse_bing_rss(DOCUMENT) + assert hits == [ + SearchHit( + title="DeepSeek Harness v0.2", + url="https://pandaily.com/deepseek-harness-v0-2", + snippet="Desktop installers and a plugin manager.", + ), + SearchHit(title="Spaced title", url="https://example.com/spaced", snippet=""), + ] + + +def test_parse_bing_rss_rejects_a_non_rss_body() -> None: + with pytest.raises(RuntimeError, match="unparseable XML"): + _ = parse_bing_rss("rate limited") diff --git a/tests/test_tools/test_search.py b/tests/test_tools/test_search.py index 7df300c..11af7fe 100644 --- a/tests/test_tools/test_search.py +++ b/tests/test_tools/test_search.py @@ -8,28 +8,34 @@ import pytest from plyngent.agent.tools import ToolTag from plyngent.lmproto.deepseek.anthropic.search import SearchHit -from plyngent.tools.net.search import WEB_SEARCH_TOOL_NAME, build_web_search_tool +from plyngent.tools.net.search import ( + WEB_SEARCH_TOOL_NAME, + build_web_search_tool, + search_in_order, +) -class _StubSearch: - """Stands in for :class:`DeepseekSearchClient`.""" +class _StubBackend: + """Stands in for a search source (raises, finds nothing, or answers).""" - def __init__(self, hits: list[SearchHit]) -> None: - self.hits = hits - self.queries: list[tuple[str, str]] = [] + def __init__(self, *, hits: list[SearchHit] | None = None, error: str | None = None) -> None: + self.hits = hits or [] + self.error = error + self.queries: list[str] = [] - async def search(self, query: str, *, model: str, **kwargs: object) -> list[SearchHit]: - self.queries.append((query, model)) + async def search(self, query: str) -> list[SearchHit]: + self.queries.append(query) + if self.error is not None: + raise RuntimeError(self.error) return self.hits -def _definition(hits: list[SearchHit]) -> tuple[Any, _StubSearch]: - client = _StubSearch(hits) - return build_web_search_tool(client, model="deepseek-flash"), client +def _hits(*pairs: tuple[str, str]) -> list[SearchHit]: + return [SearchHit(title=title, url=url) for title, url in pairs] def test_tool_definition_shape() -> None: - definition, _ = _definition([]) + definition = build_web_search_tool([_StubBackend()]) assert definition.name == WEB_SEARCH_TOOL_NAME assert definition.tags == ToolTag.LOCAL | ToolTag.READ_ONLY assert "fetch" in definition.description @@ -39,20 +45,73 @@ def test_tool_definition_shape() -> None: @pytest.mark.asyncio -async def test_handler_returns_one_line_per_hit() -> None: - definition, client = _definition([SearchHit(title="DeepSeek-V4 preview", url="https://api-docs.deepseek.com/x")]) - assert await definition.handler(query="DeepSeek-V4") == ("- DeepSeek-V4 preview — https://api-docs.deepseek.com/x") - assert client.queries == [("DeepSeek-V4", "deepseek-flash")] +async def test_handler_returns_one_entry_per_hit() -> None: + backend = _StubBackend(hits=_hits(("DeepSeek-V4 preview", "https://api-docs.deepseek.com/x"))) + definition = build_web_search_tool([backend]) + assert await definition.handler(query="DeepSeek-V4") == "- DeepSeek-V4 preview — https://api-docs.deepseek.com/x" + assert backend.queries == ["DeepSeek-V4"] + + +@pytest.mark.asyncio +async def test_handler_prints_the_snippet_under_its_hit() -> None: + backend = _StubBackend(hits=[SearchHit(title="T", url="https://example.com/1", snippet="One line of it.")]) + definition = build_web_search_tool([backend]) + assert await definition.handler(query="q") == "- T — https://example.com/1\n One line of it." + + +@pytest.mark.asyncio +async def test_the_first_source_that_answers_wins() -> None: + failing = _StubBackend(error="HTTP 503") + empty = _StubBackend() + answering = _StubBackend(hits=_hits(("Found", "https://example.com/found"))) + hits, failures = await search_in_order([failing, empty, answering], "q") + assert [hit.url for hit in hits] == ["https://example.com/found"] + assert failures == ["_StubBackend: HTTP 503", "_StubBackend: no results"] @pytest.mark.asyncio async def test_handler_caps_the_hits_and_names_an_untitled_page() -> None: - definition, _ = _definition([SearchHit(title="", url=f"https://example.com/{index}") for index in range(5)]) - lines = (await definition.handler(query="anything", hits=2)).splitlines() - assert lines == ["- (untitled) — https://example.com/0", "- (untitled) — https://example.com/1"] + backend = _StubBackend(hits=_hits(*[("", f"https://example.com/{index}") for index in range(5)])) + definition = build_web_search_tool([backend], max_hits=2) + assert (await definition.handler(query="anything")).splitlines() == [ + "- (untitled) — https://example.com/0", + "- (untitled) — https://example.com/1", + ] @pytest.mark.asyncio -async def test_handler_reports_an_empty_search() -> None: - definition, _ = _definition([]) - assert "no results" in await definition.handler(query="nothing matches this") +async def test_handler_reports_every_source_that_failed() -> None: + definition = build_web_search_tool([_StubBackend(error="timeout"), _StubBackend()]) + message = await definition.handler(query="nothing matches this") + assert "found nothing" in message + assert "timeout" in message and "no results" in message + + +@pytest.mark.asyncio +async def test_handler_without_sources_says_so() -> None: + definition = build_web_search_tool([]) + assert "no source configured" in await definition.handler(query="q") + + +def test_deepseek_backend_passes_the_model_through() -> None: + from plyngent.tools.net.search import DeepseekSearchBackend + + class _Client: + def __init__(self) -> None: + self.seen: list[tuple[str, str]] = [] + + async def search(self, query: str, *, model: str, **kwargs: object) -> list[SearchHit]: + self.seen.append((query, model)) + return _hits(("T", "https://example.com")) + + client: Any = _Client() + backend = DeepseekSearchBackend(client, model="deepseek-flash") + + async def run() -> list[SearchHit]: + return list(await backend.search("q")) + + import asyncio + + hits = asyncio.run(run()) + assert [hit.url for hit in hits] == ["https://example.com"] + assert client.seen == [("q", "deepseek-flash")]