From fa1055dd4ce832280c0049d8865a1b486d86215a Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 1 Jul 2026 17:08:58 +0200 Subject: [PATCH] feat(capabilities): add web.discover schemas and provider seam --- .../app/capabilities/web/discover/__init__.py | 3 + .../app/capabilities/web/discover/executor.py | 40 ++++++++++ .../web/discover/providers/__init__.py | 3 + .../web/discover/providers/base.py | 22 ++++++ .../app/capabilities/web/discover/schemas.py | 21 +++++ .../web/discover/test_executor.py | 79 +++++++++++++++++++ 6 files changed, 168 insertions(+) create mode 100644 surfsense_backend/app/capabilities/web/discover/__init__.py create mode 100644 surfsense_backend/app/capabilities/web/discover/executor.py create mode 100644 surfsense_backend/app/capabilities/web/discover/providers/__init__.py create mode 100644 surfsense_backend/app/capabilities/web/discover/providers/base.py create mode 100644 surfsense_backend/app/capabilities/web/discover/schemas.py create mode 100644 surfsense_backend/tests/unit/capabilities/web/discover/test_executor.py diff --git a/surfsense_backend/app/capabilities/web/discover/__init__.py b/surfsense_backend/app/capabilities/web/discover/__init__.py new file mode 100644 index 000000000..37b153535 --- /dev/null +++ b/surfsense_backend/app/capabilities/web/discover/__init__.py @@ -0,0 +1,3 @@ +"""``web.discover`` verb: a query → candidate URLs the agent can feed to ``web.scrape``.""" + +from __future__ import annotations diff --git a/surfsense_backend/app/capabilities/web/discover/executor.py b/surfsense_backend/app/capabilities/web/discover/executor.py new file mode 100644 index 000000000..85f6bf83b --- /dev/null +++ b/surfsense_backend/app/capabilities/web/discover/executor.py @@ -0,0 +1,40 @@ +"""``web.discover`` executor: route a query to the first configured search provider.""" + +from __future__ import annotations + +from collections.abc import Sequence + +from app.capabilities.types import Executor +from app.capabilities.web.discover.providers.base import DiscoverProvider +from app.capabilities.web.discover.schemas import DiscoverInput, DiscoverOutput + + +class NoDiscoverProviderError(RuntimeError): + """Raised when no search provider is configured (no platform key/host set).""" + + +def build_discover_executor( + providers: Sequence[DiscoverProvider] | None = None, +) -> Executor: + """Bind the executor to a provider set (defaults to the real env-keyed providers).""" + registry = list(providers) if providers is not None else _default_providers() + + async def execute(payload: DiscoverInput) -> DiscoverOutput: + provider = next((p for p in registry if p.is_available()), None) + if provider is None: + raise NoDiscoverProviderError( + "web.discover has no configured search provider " + "(set a SearXNG host or a Linkup/Baidu key)." + ) + hits = await provider.search(payload.query, payload.top_k) + return DiscoverOutput(hits=hits) + + return execute + + +def _default_providers() -> list[DiscoverProvider]: + from app.capabilities.web.discover.providers.baidu import BaiduProvider + from app.capabilities.web.discover.providers.linkup import LinkupProvider + from app.capabilities.web.discover.providers.searxng import SearxngProvider + + return [SearxngProvider(), LinkupProvider(), BaiduProvider()] diff --git a/surfsense_backend/app/capabilities/web/discover/providers/__init__.py b/surfsense_backend/app/capabilities/web/discover/providers/__init__.py new file mode 100644 index 000000000..68ca2b374 --- /dev/null +++ b/surfsense_backend/app/capabilities/web/discover/providers/__init__.py @@ -0,0 +1,3 @@ +"""``web.discover`` search providers (env-keyed: SearXNG, Linkup, Baidu).""" + +from __future__ import annotations diff --git a/surfsense_backend/app/capabilities/web/discover/providers/base.py b/surfsense_backend/app/capabilities/web/discover/providers/base.py new file mode 100644 index 000000000..c245072eb --- /dev/null +++ b/surfsense_backend/app/capabilities/web/discover/providers/base.py @@ -0,0 +1,22 @@ +"""The seam every ``web.discover`` provider implements.""" + +from __future__ import annotations + +from typing import Protocol, runtime_checkable + +from app.capabilities.web.discover.schemas import DiscoverHit + + +@runtime_checkable +class DiscoverProvider(Protocol): + """An env-keyed search backend that suggests candidate URLs for a query.""" + + name: str + + def is_available(self) -> bool: + """True when this provider's platform key/host is configured.""" + ... + + async def search(self, query: str, top_k: int) -> list[DiscoverHit]: + """Return up to ``top_k`` candidate hits for ``query``.""" + ... diff --git a/surfsense_backend/app/capabilities/web/discover/schemas.py b/surfsense_backend/app/capabilities/web/discover/schemas.py new file mode 100644 index 000000000..77622b210 --- /dev/null +++ b/surfsense_backend/app/capabilities/web/discover/schemas.py @@ -0,0 +1,21 @@ +"""``web.discover`` I/O contracts.""" + +from __future__ import annotations + +from pydantic import BaseModel + + +class DiscoverInput(BaseModel): + query: str + top_k: int = 10 + + +class DiscoverHit(BaseModel): + url: str + title: str + snippet: str | None = None + provider: str + + +class DiscoverOutput(BaseModel): + hits: list[DiscoverHit] diff --git a/surfsense_backend/tests/unit/capabilities/web/discover/test_executor.py b/surfsense_backend/tests/unit/capabilities/web/discover/test_executor.py new file mode 100644 index 000000000..5d50d5f6f --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/web/discover/test_executor.py @@ -0,0 +1,79 @@ +"""`web.discover` executor: pick the first configured provider; self-disable when none. + +Boundary mocked: the providers (injected fakes). NOT mocked: the executor's +provider-selection and self-disable behavior. +""" + +from __future__ import annotations + +import pytest + +from app.capabilities.web.discover.executor import ( + NoDiscoverProviderError, + build_discover_executor, +) +from app.capabilities.web.discover.schemas import ( + DiscoverHit, + DiscoverInput, + DiscoverOutput, +) + +pytestmark = pytest.mark.unit + + +class _FakeProvider: + def __init__( + self, name: str, available: bool, hits: list[DiscoverHit] | None = None + ): + self.name = name + self._available = available + self._hits = hits or [] + self.calls: list[tuple[str, int]] = [] + + def is_available(self) -> bool: + return self._available + + async def search(self, query: str, top_k: int) -> list[DiscoverHit]: + self.calls.append((query, top_k)) + return self._hits + + +def _hit(url: str, provider: str) -> DiscoverHit: + return DiscoverHit(url=url, title=url, snippet="s", provider=provider) + + +async def test_uses_the_first_available_provider(): + first = _FakeProvider( + "searxng", available=True, hits=[_hit("https://a.com", "searxng")] + ) + second = _FakeProvider( + "linkup", available=True, hits=[_hit("https://b.com", "linkup")] + ) + execute = build_discover_executor(providers=[first, second]) + + out = await execute(DiscoverInput(query="acme pricing", top_k=5)) + + assert isinstance(out, DiscoverOutput) + assert [h.url for h in out.hits] == ["https://a.com"] + assert first.calls == [("acme pricing", 5)] + assert second.calls == [] # first available short-circuits + + +async def test_skips_unavailable_providers(): + off = _FakeProvider("searxng", available=False) + on = _FakeProvider("linkup", available=True, hits=[_hit("https://b.com", "linkup")]) + execute = build_discover_executor(providers=[off, on]) + + out = await execute(DiscoverInput(query="q")) + + assert [h.provider for h in out.hits] == ["linkup"] + assert off.calls == [] + + +async def test_self_disables_when_no_provider_is_configured(): + execute = build_discover_executor( + providers=[_FakeProvider("searxng", available=False)] + ) + + with pytest.raises(NoDiscoverProviderError): + await execute(DiscoverInput(query="q"))