diff --git a/surfsense_backend/app/capabilities/web/discover/providers/baidu.py b/surfsense_backend/app/capabilities/web/discover/providers/baidu.py new file mode 100644 index 000000000..fc632a9ea --- /dev/null +++ b/surfsense_backend/app/capabilities/web/discover/providers/baidu.py @@ -0,0 +1,53 @@ +"""Baidu AI Search discover provider (env-keyed via ``BAIDU_API_KEY``).""" + +from __future__ import annotations + +import httpx + +from app.capabilities.web.discover.schemas import DiscoverHit +from app.config import config + +_ENDPOINT = "https://qianfan.baidubce.com/v2/ai_search/chat/completions" +_SNIPPET_MAX = 300 + + +class BaiduProvider: + name = "baidu" + + def is_available(self) -> bool: + return bool(config.BAIDU_API_KEY) + + async def search(self, query: str, top_k: int) -> list[DiscoverHit]: + max_per_type = min(top_k, 20) + payload = { + "messages": [{"role": "user", "content": query}], + "model": config.BAIDU_MODEL, + "search_source": config.BAIDU_SEARCH_SOURCE, + "resource_type_filter": [{"type": "web", "top_k": max_per_type}], + "stream": False, + } + headers = { + "X-Appbuilder-Authorization": f"Bearer {config.BAIDU_API_KEY}", + "Content-Type": "application/json", + } + async with httpx.AsyncClient(timeout=90.0) as client: + response = await client.post(_ENDPOINT, headers=headers, json=payload) + response.raise_for_status() + + hits: list[DiscoverHit] = [] + for reference in response.json().get("references", []): + url = reference.get("url", "") + if not url: + continue + content = reference.get("content", "") or "" + hits.append( + DiscoverHit( + url=url, + title=reference.get("title") or url, + snippet=content[:_SNIPPET_MAX] or None, + provider=self.name, + ) + ) + if len(hits) >= top_k: + break + return hits diff --git a/surfsense_backend/app/capabilities/web/discover/providers/linkup.py b/surfsense_backend/app/capabilities/web/discover/providers/linkup.py new file mode 100644 index 000000000..77393de08 --- /dev/null +++ b/surfsense_backend/app/capabilities/web/discover/providers/linkup.py @@ -0,0 +1,40 @@ +"""Linkup discover provider (env-keyed via ``LINKUP_API_KEY``).""" + +from __future__ import annotations + +import asyncio + +from linkup import LinkupClient + +from app.capabilities.web.discover.schemas import DiscoverHit +from app.config import config + + +class LinkupProvider: + name = "linkup" + + def is_available(self) -> bool: + return bool(config.LINKUP_API_KEY) + + async def search(self, query: str, top_k: int) -> list[DiscoverHit]: + client = LinkupClient(api_key=config.LINKUP_API_KEY) + response = await asyncio.to_thread( + client.search, query=query, depth="standard", output_type="searchResults" + ) + hits: list[DiscoverHit] = [] + for result in getattr(response, "results", None) or []: + url = getattr(result, "url", "") or "" + if not url: + continue + content = getattr(result, "content", "") or "" + hits.append( + DiscoverHit( + url=url, + title=getattr(result, "name", "") or url, + snippet=content or None, + provider=self.name, + ) + ) + if len(hits) >= top_k: + break + return hits diff --git a/surfsense_backend/app/capabilities/web/discover/providers/searxng.py b/surfsense_backend/app/capabilities/web/discover/providers/searxng.py new file mode 100644 index 000000000..92489862a --- /dev/null +++ b/surfsense_backend/app/capabilities/web/discover/providers/searxng.py @@ -0,0 +1,28 @@ +"""SearXNG discover provider, wrapping the platform-env SearXNG search service.""" + +from __future__ import annotations + +from app.capabilities.web.discover.schemas import DiscoverHit +from app.services import web_search_service + + +class SearxngProvider: + """Env-keyed via ``SEARXNG_DEFAULT_HOST`` (the platform SearXNG instance).""" + + name = "searxng" + + def is_available(self) -> bool: + return web_search_service.is_available() + + async def search(self, query: str, top_k: int) -> list[DiscoverHit]: + result_object, _documents = await web_search_service.search(query, top_k) + return [ + DiscoverHit( + url=source["url"], + title=source.get("title") or source["url"], + snippet=source.get("description") or None, + provider=self.name, + ) + for source in result_object.get("sources", []) + if source.get("url") + ] diff --git a/surfsense_backend/tests/unit/capabilities/web/discover/test_baidu_provider.py b/surfsense_backend/tests/unit/capabilities/web/discover/test_baidu_provider.py new file mode 100644 index 000000000..dc3035da7 --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/web/discover/test_baidu_provider.py @@ -0,0 +1,91 @@ +"""BaiduProvider maps the Baidu AI Search references to DiscoverHits, env-keyed. + +Boundary mocked: httpx.AsyncClient + config key. NOT mocked: reference→hit mapping. +""" + +from __future__ import annotations + +import pytest + +import app.capabilities.web.discover.providers.baidu as baidu_module +from app.capabilities.web.discover.providers.baidu import BaiduProvider +from app.config import config + +pytestmark = pytest.mark.unit + + +class _FakeResponse: + def __init__(self, payload): + self._payload = payload + + def raise_for_status(self): + return None + + def json(self): + return self._payload + + +class _FakeAsyncClient: + """Minimal async-context httpx stand-in returning a canned payload.""" + + payload: dict = {} + + def __init__(self, *args, **kwargs): + pass + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + return False + + async def post(self, url, headers=None, json=None): + return _FakeResponse(type(self).payload) + + +def _install(monkeypatch, payload): + monkeypatch.setattr(config, "BAIDU_API_KEY", "k") + _FakeAsyncClient.payload = payload + monkeypatch.setattr(baidu_module.httpx, "AsyncClient", _FakeAsyncClient) + + +def test_is_available_reflects_the_env_key(monkeypatch): + monkeypatch.setattr(config, "BAIDU_API_KEY", "k") + assert BaiduProvider().is_available() is True + monkeypatch.setattr(config, "BAIDU_API_KEY", None) + assert BaiduProvider().is_available() is False + + +async def test_maps_references_to_hits(monkeypatch): + _install( + monkeypatch, + { + "references": [ + {"title": "Acme", "url": "https://acme.cn", "content": "hello"}, + {"title": "No URL", "url": "", "content": "skip"}, + ] + }, + ) + + hits = await BaiduProvider().search("acme", top_k=10) + + assert [h.url for h in hits] == ["https://acme.cn"] + assert hits[0].title == "Acme" + assert hits[0].snippet == "hello" + assert hits[0].provider == "baidu" + + +async def test_respects_top_k(monkeypatch): + _install( + monkeypatch, + { + "references": [ + {"title": f"n{i}", "url": f"https://{i}.cn", "content": "c"} + for i in range(5) + ] + }, + ) + + hits = await BaiduProvider().search("q", top_k=2) + + assert len(hits) == 2 diff --git a/surfsense_backend/tests/unit/capabilities/web/discover/test_linkup_provider.py b/surfsense_backend/tests/unit/capabilities/web/discover/test_linkup_provider.py new file mode 100644 index 000000000..d846e0d94 --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/web/discover/test_linkup_provider.py @@ -0,0 +1,75 @@ +"""LinkupProvider maps the Linkup SDK results to DiscoverHits, env-keyed. + +Boundary mocked: the LinkupClient SDK + config key. NOT mocked: result→hit mapping. +""" + +from __future__ import annotations + +import pytest + +import app.capabilities.web.discover.providers.linkup as linkup_module +from app.capabilities.web.discover.providers.linkup import LinkupProvider +from app.config import config + +pytestmark = pytest.mark.unit + + +class _Result: + def __init__(self, name, url, content): + self.name = name + self.url = url + self.content = content + self.type = "text" + + +class _Response: + def __init__(self, results): + self.results = results + + +class _FakeClient: + def __init__(self, api_key): + self.api_key = api_key + + def search(self, *, query, depth, output_type): + return _Response( + [ + _Result("Acme", "https://acme.com", "acme home"), + _Result("No URL", "", "skip"), + ] + ) + + +def test_is_available_reflects_the_env_key(monkeypatch): + monkeypatch.setattr(config, "LINKUP_API_KEY", "k") + assert LinkupProvider().is_available() is True + monkeypatch.setattr(config, "LINKUP_API_KEY", None) + assert LinkupProvider().is_available() is False + + +async def test_maps_results_to_hits(monkeypatch): + monkeypatch.setattr(config, "LINKUP_API_KEY", "k") + monkeypatch.setattr(linkup_module, "LinkupClient", _FakeClient) + + hits = await LinkupProvider().search("acme", top_k=10) + + assert [h.url for h in hits] == ["https://acme.com"] + assert hits[0].title == "Acme" + assert hits[0].snippet == "acme home" + assert hits[0].provider == "linkup" + + +async def test_respects_top_k(monkeypatch): + monkeypatch.setattr(config, "LINKUP_API_KEY", "k") + + class _ManyClient(_FakeClient): + def search(self, *, query, depth, output_type): + return _Response( + [_Result(f"n{i}", f"https://{i}.com", "c") for i in range(5)] + ) + + monkeypatch.setattr(linkup_module, "LinkupClient", _ManyClient) + + hits = await LinkupProvider().search("q", top_k=2) + + assert len(hits) == 2 diff --git a/surfsense_backend/tests/unit/capabilities/web/discover/test_searxng_provider.py b/surfsense_backend/tests/unit/capabilities/web/discover/test_searxng_provider.py new file mode 100644 index 000000000..a184666b3 --- /dev/null +++ b/surfsense_backend/tests/unit/capabilities/web/discover/test_searxng_provider.py @@ -0,0 +1,58 @@ +"""SearxngProvider maps the SearXNG service's sources to DiscoverHits. + +Boundary mocked: the web_search_service module. NOT mocked: the source→hit mapping. +""" + +from __future__ import annotations + +from unittest.mock import AsyncMock + +import pytest + +import app.capabilities.web.discover.providers.searxng as searxng_module +from app.capabilities.web.discover.providers.searxng import SearxngProvider + +pytestmark = pytest.mark.unit + + +def _result(sources): + return ({"sources": sources}, []) + + +async def test_maps_sources_to_hits(monkeypatch): + provider = SearxngProvider() + monkeypatch.setattr( + searxng_module.web_search_service, + "search", + AsyncMock( + return_value=_result( + [ + {"title": "Acme", "url": "https://acme.com", "description": "home"}, + { + "title": "Docs", + "url": "https://acme.com/docs", + "description": "", + }, + {"title": "no url", "url": "", "description": "skip me"}, + ] + ) + ), + ) + + hits = await provider.search("acme", top_k=5) + + assert [h.url for h in hits] == ["https://acme.com", "https://acme.com/docs"] + assert hits[0].title == "Acme" + assert hits[0].snippet == "home" + assert hits[1].snippet is None # empty description normalizes to None + assert all(h.provider == "searxng" for h in hits) + + +def test_is_available_reflects_the_service(monkeypatch): + provider = SearxngProvider() + monkeypatch.setattr(searxng_module.web_search_service, "is_available", lambda: True) + assert provider.is_available() is True + monkeypatch.setattr( + searxng_module.web_search_service, "is_available", lambda: False + ) + assert provider.is_available() is False