mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-08 22:22:17 +02:00
feat(capabilities): add env-keyed searxng/linkup/baidu discover providers
This commit is contained in:
CREDO23
parent
adc39d1062
commit
329bc63b6c
6 changed files with 345 additions and 0 deletions
|
|
@ -0,0 +1,53 @@
|
|||
"""Baidu AI Search discover provider (env-keyed via ``BAIDU_API_KEY``)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import httpx
|
||||
|
||||
from app.capabilities.web.discover.schemas import DiscoverHit
|
||||
from app.config import config
|
||||
|
||||
_ENDPOINT = "https://qianfan.baidubce.com/v2/ai_search/chat/completions"
|
||||
_SNIPPET_MAX = 300
|
||||
|
||||
|
||||
class BaiduProvider:
|
||||
name = "baidu"
|
||||
|
||||
def is_available(self) -> bool:
|
||||
return bool(config.BAIDU_API_KEY)
|
||||
|
||||
async def search(self, query: str, top_k: int) -> list[DiscoverHit]:
|
||||
max_per_type = min(top_k, 20)
|
||||
payload = {
|
||||
"messages": [{"role": "user", "content": query}],
|
||||
"model": config.BAIDU_MODEL,
|
||||
"search_source": config.BAIDU_SEARCH_SOURCE,
|
||||
"resource_type_filter": [{"type": "web", "top_k": max_per_type}],
|
||||
"stream": False,
|
||||
}
|
||||
headers = {
|
||||
"X-Appbuilder-Authorization": f"Bearer {config.BAIDU_API_KEY}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
async with httpx.AsyncClient(timeout=90.0) as client:
|
||||
response = await client.post(_ENDPOINT, headers=headers, json=payload)
|
||||
response.raise_for_status()
|
||||
|
||||
hits: list[DiscoverHit] = []
|
||||
for reference in response.json().get("references", []):
|
||||
url = reference.get("url", "")
|
||||
if not url:
|
||||
continue
|
||||
content = reference.get("content", "") or ""
|
||||
hits.append(
|
||||
DiscoverHit(
|
||||
url=url,
|
||||
title=reference.get("title") or url,
|
||||
snippet=content[:_SNIPPET_MAX] or None,
|
||||
provider=self.name,
|
||||
)
|
||||
)
|
||||
if len(hits) >= top_k:
|
||||
break
|
||||
return hits
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
"""Linkup discover provider (env-keyed via ``LINKUP_API_KEY``)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
from linkup import LinkupClient
|
||||
|
||||
from app.capabilities.web.discover.schemas import DiscoverHit
|
||||
from app.config import config
|
||||
|
||||
|
||||
class LinkupProvider:
|
||||
name = "linkup"
|
||||
|
||||
def is_available(self) -> bool:
|
||||
return bool(config.LINKUP_API_KEY)
|
||||
|
||||
async def search(self, query: str, top_k: int) -> list[DiscoverHit]:
|
||||
client = LinkupClient(api_key=config.LINKUP_API_KEY)
|
||||
response = await asyncio.to_thread(
|
||||
client.search, query=query, depth="standard", output_type="searchResults"
|
||||
)
|
||||
hits: list[DiscoverHit] = []
|
||||
for result in getattr(response, "results", None) or []:
|
||||
url = getattr(result, "url", "") or ""
|
||||
if not url:
|
||||
continue
|
||||
content = getattr(result, "content", "") or ""
|
||||
hits.append(
|
||||
DiscoverHit(
|
||||
url=url,
|
||||
title=getattr(result, "name", "") or url,
|
||||
snippet=content or None,
|
||||
provider=self.name,
|
||||
)
|
||||
)
|
||||
if len(hits) >= top_k:
|
||||
break
|
||||
return hits
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
"""SearXNG discover provider, wrapping the platform-env SearXNG search service."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.capabilities.web.discover.schemas import DiscoverHit
|
||||
from app.services import web_search_service
|
||||
|
||||
|
||||
class SearxngProvider:
|
||||
"""Env-keyed via ``SEARXNG_DEFAULT_HOST`` (the platform SearXNG instance)."""
|
||||
|
||||
name = "searxng"
|
||||
|
||||
def is_available(self) -> bool:
|
||||
return web_search_service.is_available()
|
||||
|
||||
async def search(self, query: str, top_k: int) -> list[DiscoverHit]:
|
||||
result_object, _documents = await web_search_service.search(query, top_k)
|
||||
return [
|
||||
DiscoverHit(
|
||||
url=source["url"],
|
||||
title=source.get("title") or source["url"],
|
||||
snippet=source.get("description") or None,
|
||||
provider=self.name,
|
||||
)
|
||||
for source in result_object.get("sources", [])
|
||||
if source.get("url")
|
||||
]
|
||||
|
|
@ -0,0 +1,91 @@
|
|||
"""BaiduProvider maps the Baidu AI Search references to DiscoverHits, env-keyed.
|
||||
|
||||
Boundary mocked: httpx.AsyncClient + config key. NOT mocked: reference→hit mapping.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
import app.capabilities.web.discover.providers.baidu as baidu_module
|
||||
from app.capabilities.web.discover.providers.baidu import BaiduProvider
|
||||
from app.config import config
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
class _FakeResponse:
|
||||
def __init__(self, payload):
|
||||
self._payload = payload
|
||||
|
||||
def raise_for_status(self):
|
||||
return None
|
||||
|
||||
def json(self):
|
||||
return self._payload
|
||||
|
||||
|
||||
class _FakeAsyncClient:
|
||||
"""Minimal async-context httpx stand-in returning a canned payload."""
|
||||
|
||||
payload: dict = {}
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
pass
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *exc):
|
||||
return False
|
||||
|
||||
async def post(self, url, headers=None, json=None):
|
||||
return _FakeResponse(type(self).payload)
|
||||
|
||||
|
||||
def _install(monkeypatch, payload):
|
||||
monkeypatch.setattr(config, "BAIDU_API_KEY", "k")
|
||||
_FakeAsyncClient.payload = payload
|
||||
monkeypatch.setattr(baidu_module.httpx, "AsyncClient", _FakeAsyncClient)
|
||||
|
||||
|
||||
def test_is_available_reflects_the_env_key(monkeypatch):
|
||||
monkeypatch.setattr(config, "BAIDU_API_KEY", "k")
|
||||
assert BaiduProvider().is_available() is True
|
||||
monkeypatch.setattr(config, "BAIDU_API_KEY", None)
|
||||
assert BaiduProvider().is_available() is False
|
||||
|
||||
|
||||
async def test_maps_references_to_hits(monkeypatch):
|
||||
_install(
|
||||
monkeypatch,
|
||||
{
|
||||
"references": [
|
||||
{"title": "Acme", "url": "https://acme.cn", "content": "hello"},
|
||||
{"title": "No URL", "url": "", "content": "skip"},
|
||||
]
|
||||
},
|
||||
)
|
||||
|
||||
hits = await BaiduProvider().search("acme", top_k=10)
|
||||
|
||||
assert [h.url for h in hits] == ["https://acme.cn"]
|
||||
assert hits[0].title == "Acme"
|
||||
assert hits[0].snippet == "hello"
|
||||
assert hits[0].provider == "baidu"
|
||||
|
||||
|
||||
async def test_respects_top_k(monkeypatch):
|
||||
_install(
|
||||
monkeypatch,
|
||||
{
|
||||
"references": [
|
||||
{"title": f"n{i}", "url": f"https://{i}.cn", "content": "c"}
|
||||
for i in range(5)
|
||||
]
|
||||
},
|
||||
)
|
||||
|
||||
hits = await BaiduProvider().search("q", top_k=2)
|
||||
|
||||
assert len(hits) == 2
|
||||
|
|
@ -0,0 +1,75 @@
|
|||
"""LinkupProvider maps the Linkup SDK results to DiscoverHits, env-keyed.
|
||||
|
||||
Boundary mocked: the LinkupClient SDK + config key. NOT mocked: result→hit mapping.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
import app.capabilities.web.discover.providers.linkup as linkup_module
|
||||
from app.capabilities.web.discover.providers.linkup import LinkupProvider
|
||||
from app.config import config
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
class _Result:
|
||||
def __init__(self, name, url, content):
|
||||
self.name = name
|
||||
self.url = url
|
||||
self.content = content
|
||||
self.type = "text"
|
||||
|
||||
|
||||
class _Response:
|
||||
def __init__(self, results):
|
||||
self.results = results
|
||||
|
||||
|
||||
class _FakeClient:
|
||||
def __init__(self, api_key):
|
||||
self.api_key = api_key
|
||||
|
||||
def search(self, *, query, depth, output_type):
|
||||
return _Response(
|
||||
[
|
||||
_Result("Acme", "https://acme.com", "acme home"),
|
||||
_Result("No URL", "", "skip"),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def test_is_available_reflects_the_env_key(monkeypatch):
|
||||
monkeypatch.setattr(config, "LINKUP_API_KEY", "k")
|
||||
assert LinkupProvider().is_available() is True
|
||||
monkeypatch.setattr(config, "LINKUP_API_KEY", None)
|
||||
assert LinkupProvider().is_available() is False
|
||||
|
||||
|
||||
async def test_maps_results_to_hits(monkeypatch):
|
||||
monkeypatch.setattr(config, "LINKUP_API_KEY", "k")
|
||||
monkeypatch.setattr(linkup_module, "LinkupClient", _FakeClient)
|
||||
|
||||
hits = await LinkupProvider().search("acme", top_k=10)
|
||||
|
||||
assert [h.url for h in hits] == ["https://acme.com"]
|
||||
assert hits[0].title == "Acme"
|
||||
assert hits[0].snippet == "acme home"
|
||||
assert hits[0].provider == "linkup"
|
||||
|
||||
|
||||
async def test_respects_top_k(monkeypatch):
|
||||
monkeypatch.setattr(config, "LINKUP_API_KEY", "k")
|
||||
|
||||
class _ManyClient(_FakeClient):
|
||||
def search(self, *, query, depth, output_type):
|
||||
return _Response(
|
||||
[_Result(f"n{i}", f"https://{i}.com", "c") for i in range(5)]
|
||||
)
|
||||
|
||||
monkeypatch.setattr(linkup_module, "LinkupClient", _ManyClient)
|
||||
|
||||
hits = await LinkupProvider().search("q", top_k=2)
|
||||
|
||||
assert len(hits) == 2
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
"""SearxngProvider maps the SearXNG service's sources to DiscoverHits.
|
||||
|
||||
Boundary mocked: the web_search_service module. NOT mocked: the source→hit mapping.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
|
||||
import app.capabilities.web.discover.providers.searxng as searxng_module
|
||||
from app.capabilities.web.discover.providers.searxng import SearxngProvider
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
def _result(sources):
|
||||
return ({"sources": sources}, [])
|
||||
|
||||
|
||||
async def test_maps_sources_to_hits(monkeypatch):
|
||||
provider = SearxngProvider()
|
||||
monkeypatch.setattr(
|
||||
searxng_module.web_search_service,
|
||||
"search",
|
||||
AsyncMock(
|
||||
return_value=_result(
|
||||
[
|
||||
{"title": "Acme", "url": "https://acme.com", "description": "home"},
|
||||
{
|
||||
"title": "Docs",
|
||||
"url": "https://acme.com/docs",
|
||||
"description": "",
|
||||
},
|
||||
{"title": "no url", "url": "", "description": "skip me"},
|
||||
]
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
hits = await provider.search("acme", top_k=5)
|
||||
|
||||
assert [h.url for h in hits] == ["https://acme.com", "https://acme.com/docs"]
|
||||
assert hits[0].title == "Acme"
|
||||
assert hits[0].snippet == "home"
|
||||
assert hits[1].snippet is None # empty description normalizes to None
|
||||
assert all(h.provider == "searxng" for h in hits)
|
||||
|
||||
|
||||
def test_is_available_reflects_the_service(monkeypatch):
|
||||
provider = SearxngProvider()
|
||||
monkeypatch.setattr(searxng_module.web_search_service, "is_available", lambda: True)
|
||||
assert provider.is_available() is True
|
||||
monkeypatch.setattr(
|
||||
searxng_module.web_search_service, "is_available", lambda: False
|
||||
)
|
||||
assert provider.is_available() is False
|
||||
Loading…
Add table
Add a link
Reference in a new issue