mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-08 22:22:17 +02:00
feat(capabilities): replace web.scrape/web.discover with unified web.crawl
web.crawl scrapes a single URL (maxCrawlDepth=0) or spiders a whole site, backed by the proprietary site_crawler engine. Rewires the scraping subagent tools and capability tests onto the new verb.
This commit is contained in:
CREDO23
parent
f82fae3973
commit
9fe9c5b71d
31 changed files with 411 additions and 950 deletions
|
|
@ -8,8 +8,7 @@ from langchain_core.tools import BaseTool
|
|||
|
||||
from app.agents.chat.multi_agent_chat.shared.permissions import Ruleset
|
||||
from app.capabilities.core.access.agent import build_capability_tools
|
||||
from app.capabilities.web.discover.definition import WEB_DISCOVER
|
||||
from app.capabilities.web.scrape.definition import WEB_SCRAPE
|
||||
from app.capabilities.web.crawl.definition import WEB_CRAWL
|
||||
from app.capabilities.youtube.comments.definition import YOUTUBE_COMMENTS
|
||||
from app.capabilities.youtube.scrape.definition import YOUTUBE_SCRAPE
|
||||
|
||||
|
|
@ -21,7 +20,7 @@ NAME = "scraping"
|
|||
|
||||
RULESET = Ruleset(origin=NAME, rules=[])
|
||||
|
||||
_CI_VERBS = [WEB_DISCOVER, WEB_SCRAPE, YOUTUBE_SCRAPE, YOUTUBE_COMMENTS]
|
||||
_CI_VERBS = [WEB_CRAWL, YOUTUBE_SCRAPE, YOUTUBE_COMMENTS]
|
||||
|
||||
|
||||
def load_tools(
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
"""``web.*`` namespace: the generic web scraping + discovery verbs."""
|
||||
"""``web.*`` namespace: the unified web content crawler verb."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.capabilities.web.discover import definition as _discover # noqa: F401
|
||||
from app.capabilities.web.scrape import definition as _scrape # noqa: F401
|
||||
from app.capabilities.web.crawl import definition as _crawl # noqa: F401
|
||||
|
|
|
|||
3
surfsense_backend/app/capabilities/web/crawl/__init__.py
Normal file
3
surfsense_backend/app/capabilities/web/crawl/__init__.py
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
"""``web.crawl`` verb: single-URL scrape or same-site spider → one row per page."""
|
||||
|
||||
from __future__ import annotations
|
||||
25
surfsense_backend/app/capabilities/web/crawl/definition.py
Normal file
25
surfsense_backend/app/capabilities/web/crawl/definition.py
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
"""``web.crawl`` capability registration."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.capabilities.core import BillingUnit, Capability, register_capability
|
||||
from app.capabilities.web.crawl.executor import build_crawl_executor
|
||||
from app.capabilities.web.crawl.schemas import CrawlInput, CrawlOutput
|
||||
|
||||
WEB_CRAWL = Capability(
|
||||
name="web.crawl",
|
||||
description=(
|
||||
"Scrape a single web page or crawl a whole website. Give it one or more "
|
||||
"startUrls. Set maxCrawlDepth=0 to fetch just those URLs, or higher to "
|
||||
"also follow the links on each page (depth 1 = the start pages plus the "
|
||||
"pages they link to, and so on) — staying on the same site and stopping "
|
||||
"at maxCrawlPages. Returns one item per fetched page with clean markdown "
|
||||
"content, metadata (title, description), and crawl provenance."
|
||||
),
|
||||
input_schema=CrawlInput,
|
||||
output_schema=CrawlOutput,
|
||||
executor=build_crawl_executor(),
|
||||
billing_unit=BillingUnit.WEB_CRAWL,
|
||||
)
|
||||
|
||||
register_capability(WEB_CRAWL)
|
||||
65
surfsense_backend/app/capabilities/web/crawl/executor.py
Normal file
65
surfsense_backend/app/capabilities/web/crawl/executor.py
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
"""``web.crawl`` executor: seeds → site spider → one cleaned item per fetched page.
|
||||
|
||||
Boundary owned elsewhere: the crawl frontier/fetch live in the proprietary engine
|
||||
(``app.proprietary.web_crawler``). This executor only maps the engine's
|
||||
``CrawlPage`` list onto the typed ``CrawlOutput`` (status labels, truncation, and
|
||||
the captcha telemetry the billing seam reads).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.capabilities.core import Executor
|
||||
from app.capabilities.web.crawl.schemas import (
|
||||
CrawlInput,
|
||||
CrawlItem,
|
||||
CrawlMeta,
|
||||
CrawlOutput,
|
||||
)
|
||||
from app.proprietary.web_crawler import (
|
||||
CrawlOutcomeStatus,
|
||||
CrawlPage,
|
||||
WebCrawlerConnector,
|
||||
crawl_site,
|
||||
)
|
||||
|
||||
_STATUS_LABEL: dict[CrawlOutcomeStatus, str] = {
|
||||
CrawlOutcomeStatus.SUCCESS: "success",
|
||||
CrawlOutcomeStatus.EMPTY: "empty",
|
||||
CrawlOutcomeStatus.FAILED: "failed",
|
||||
}
|
||||
|
||||
|
||||
def build_crawl_executor(engine: WebCrawlerConnector | None = None) -> Executor:
|
||||
"""Build the ``web.crawl`` executor, optionally over an injected engine (tests)."""
|
||||
crawler = engine or WebCrawlerConnector()
|
||||
|
||||
async def execute(payload: CrawlInput) -> CrawlOutput:
|
||||
pages = await crawl_site(
|
||||
crawler,
|
||||
payload.startUrls,
|
||||
max_crawl_depth=payload.maxCrawlDepth,
|
||||
max_crawl_pages=payload.maxCrawlPages,
|
||||
)
|
||||
return CrawlOutput(
|
||||
items=[_to_item(page, payload.maxLength) for page in pages],
|
||||
captcha_attempts=sum(page.captcha_attempts for page in pages),
|
||||
captcha_solved=sum(1 for page in pages if page.captcha_solved),
|
||||
)
|
||||
|
||||
return execute
|
||||
|
||||
|
||||
def _to_item(page: CrawlPage, max_length: int) -> CrawlItem:
|
||||
content = page.content[:max_length] if page.content is not None else None
|
||||
return CrawlItem(
|
||||
url=page.url,
|
||||
status=_STATUS_LABEL[page.status],
|
||||
crawl=CrawlMeta(
|
||||
loadedUrl=page.loaded_url or page.url,
|
||||
depth=page.depth,
|
||||
referrerUrl=page.referrer,
|
||||
),
|
||||
markdown=content,
|
||||
metadata=page.metadata,
|
||||
error=page.error,
|
||||
)
|
||||
115
surfsense_backend/app/capabilities/web/crawl/schemas.py
Normal file
115
surfsense_backend/app/capabilities/web/crawl/schemas.py
Normal file
|
|
@ -0,0 +1,115 @@
|
|||
# ruff: noqa: N815 - public field names intentionally mirror the Apify camelCase spec
|
||||
"""``web.crawl`` I/O contracts.
|
||||
|
||||
An Apify *Website Content Crawler*-style surface (see apify.com/apify/website-
|
||||
content-crawler and docs.firecrawl.dev): one verb that either scrapes the given
|
||||
URLs (``maxCrawlDepth == 0``) or spiders their site (``maxCrawlDepth > 0``),
|
||||
bounded by ``maxCrawlPages`` and kept on the seed's site.
|
||||
|
||||
Fields are trimmed to what the proprietary engine honors today. Apify knobs the
|
||||
engine handles automatically (crawler type, proxy, dynamic-render waits) are
|
||||
intentionally omitted, as are features we haven't built (URL globs, output
|
||||
formats, click actions, PII handling).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
MAX_START_URLS = 20
|
||||
"""Per-call cap on seed URLs: bounds a synchronous request's fan-out (05)."""
|
||||
|
||||
MAX_CRAWL_DEPTH = 5
|
||||
"""Deepest link distance a spider will follow from a start URL."""
|
||||
|
||||
MAX_CRAWL_PAGES = 200
|
||||
"""Hard ceiling on pages fetched per call (protects the wallet and the run)."""
|
||||
|
||||
|
||||
class CrawlInput(BaseModel):
|
||||
startUrls: list[str] = Field(
|
||||
min_length=1,
|
||||
max_length=MAX_START_URLS,
|
||||
description=(
|
||||
"Seed URLs to crawl. With maxCrawlDepth=0 only these are fetched; "
|
||||
"with a higher depth they are also the entry points for the spider."
|
||||
),
|
||||
)
|
||||
maxCrawlDepth: int = Field(
|
||||
default=0,
|
||||
ge=0,
|
||||
le=MAX_CRAWL_DEPTH,
|
||||
description=(
|
||||
"How many link-hops to follow from each start URL. 0 = scrape only "
|
||||
"the start URLs (no spidering); 1 = also their linked pages; etc. "
|
||||
"The spider stays on the start URL's site."
|
||||
),
|
||||
)
|
||||
maxCrawlPages: int = Field(
|
||||
default=10,
|
||||
ge=1,
|
||||
le=MAX_CRAWL_PAGES,
|
||||
description=(
|
||||
"Maximum number of pages to fetch in total (start URLs included). "
|
||||
"The crawl stops once this many pages have been fetched."
|
||||
),
|
||||
)
|
||||
maxLength: int = Field(
|
||||
default=50_000,
|
||||
ge=1,
|
||||
description="Maximum characters of cleaned markdown kept per page (truncates beyond).",
|
||||
)
|
||||
|
||||
@property
|
||||
def estimated_units(self) -> int:
|
||||
"""Worst-case billable pages for the pre-flight gate (03c)."""
|
||||
if self.maxCrawlDepth == 0:
|
||||
return len(self.startUrls)
|
||||
return self.maxCrawlPages
|
||||
|
||||
|
||||
class CrawlMeta(BaseModel):
|
||||
loadedUrl: str = Field(description="The URL actually fetched for this page.")
|
||||
depth: int = Field(
|
||||
description="Link distance from a start URL (0 for a start URL itself)."
|
||||
)
|
||||
referrerUrl: str | None = Field(
|
||||
default=None,
|
||||
description="The page this URL was discovered on (null for start URLs).",
|
||||
)
|
||||
|
||||
|
||||
class CrawlItem(BaseModel):
|
||||
url: str = Field(description="The requested URL for this page.")
|
||||
status: Literal["success", "empty", "failed"] = Field(
|
||||
description="success = content returned; empty = fetched but no content; failed = could not fetch."
|
||||
)
|
||||
crawl: CrawlMeta | None = Field(
|
||||
default=None, description="Crawl provenance (loaded URL, depth, referrer)."
|
||||
)
|
||||
markdown: str | None = Field(
|
||||
default=None, description="Cleaned page content as markdown (null unless success)."
|
||||
)
|
||||
metadata: dict[str, str] | None = Field(
|
||||
default=None, description="Page metadata such as title and description."
|
||||
)
|
||||
error: str | None = Field(
|
||||
default=None, description="Failure reason when status is not success."
|
||||
)
|
||||
|
||||
|
||||
class CrawlOutput(BaseModel):
|
||||
items: list[CrawlItem] = Field(
|
||||
default_factory=list,
|
||||
description="One item per fetched page, in crawl (BFS) order.",
|
||||
)
|
||||
# Billing-only telemetry; excluded from the wire shape (mirrors web.scrape).
|
||||
captcha_attempts: int = Field(default=0, exclude=True)
|
||||
captcha_solved: int = Field(default=0, exclude=True)
|
||||
|
||||
@property
|
||||
def billable_units(self) -> int:
|
||||
"""Successful pages are the metered unit (03c)."""
|
||||
return sum(1 for item in self.items if item.status == "success")
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
"""``web.discover`` verb: a query → candidate URLs the agent can feed to ``web.scrape``."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -1,23 +0,0 @@
|
|||
"""``web.discover`` capability registration (free — see 04-capabilities open item)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.capabilities.core import Capability, register_capability
|
||||
from app.capabilities.web.discover.executor import build_discover_executor
|
||||
from app.capabilities.web.discover.schemas import DiscoverInput, DiscoverOutput
|
||||
|
||||
WEB_DISCOVER = Capability(
|
||||
name="web.discover",
|
||||
description=(
|
||||
"Search the web for a query and return ranked results. Use it to find "
|
||||
"pages when you don't already have exact URLs. Returns a list of hits "
|
||||
"(url, title, snippet, provider); pass the chosen url(s) to web.scrape "
|
||||
"to read their full content."
|
||||
),
|
||||
input_schema=DiscoverInput,
|
||||
output_schema=DiscoverOutput,
|
||||
executor=build_discover_executor(),
|
||||
billing_unit=None,
|
||||
)
|
||||
|
||||
register_capability(WEB_DISCOVER)
|
||||
|
|
@ -1,42 +0,0 @@
|
|||
"""``web.discover`` executor: route a query to the first configured search provider."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
from app.capabilities.core import Executor
|
||||
from app.capabilities.web.discover.providers.base import DiscoverProvider
|
||||
from app.capabilities.web.discover.schemas import DiscoverInput, DiscoverOutput
|
||||
|
||||
|
||||
class NoDiscoverProviderError(RuntimeError):
|
||||
"""Raised when no search provider is configured (no platform key/host set)."""
|
||||
|
||||
|
||||
def build_discover_executor(
|
||||
providers: Sequence[DiscoverProvider] | None = None,
|
||||
) -> Executor:
|
||||
"""Bind the executor to a provider set (defaults to the real env-keyed providers)."""
|
||||
registry = list(providers) if providers is not None else _default_providers()
|
||||
|
||||
async def execute(payload: DiscoverInput) -> DiscoverOutput:
|
||||
provider = next((p for p in registry if p.is_available()), None)
|
||||
if provider is None:
|
||||
raise NoDiscoverProviderError(
|
||||
"web.discover has no configured search provider "
|
||||
"(set a SearXNG host or a Linkup/Baidu key)."
|
||||
)
|
||||
hits = await provider.search(payload.query, payload.top_k)
|
||||
# Enforce the verb's documented cap here, once, for every provider:
|
||||
# some backends (e.g. SearXNG) treat `top_k` as a hint and over-return.
|
||||
return DiscoverOutput(hits=hits[: payload.top_k])
|
||||
|
||||
return execute
|
||||
|
||||
|
||||
def _default_providers() -> list[DiscoverProvider]:
|
||||
from app.capabilities.web.discover.providers.baidu import BaiduProvider
|
||||
from app.capabilities.web.discover.providers.linkup import LinkupProvider
|
||||
from app.capabilities.web.discover.providers.searxng import SearxngProvider
|
||||
|
||||
return [SearxngProvider(), LinkupProvider(), BaiduProvider()]
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
"""``web.discover`` search providers (env-keyed: SearXNG, Linkup, Baidu)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -1,53 +0,0 @@
|
|||
"""Baidu AI Search discover provider (env-keyed via ``BAIDU_API_KEY``)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import httpx
|
||||
|
||||
from app.capabilities.web.discover.schemas import DiscoverHit
|
||||
from app.config import config
|
||||
|
||||
_ENDPOINT = "https://qianfan.baidubce.com/v2/ai_search/chat/completions"
|
||||
_SNIPPET_MAX = 300
|
||||
|
||||
|
||||
class BaiduProvider:
|
||||
name = "baidu"
|
||||
|
||||
def is_available(self) -> bool:
|
||||
return bool(config.BAIDU_API_KEY)
|
||||
|
||||
async def search(self, query: str, top_k: int) -> list[DiscoverHit]:
|
||||
max_per_type = min(top_k, 20)
|
||||
payload = {
|
||||
"messages": [{"role": "user", "content": query}],
|
||||
"model": config.BAIDU_MODEL,
|
||||
"search_source": config.BAIDU_SEARCH_SOURCE,
|
||||
"resource_type_filter": [{"type": "web", "top_k": max_per_type}],
|
||||
"stream": False,
|
||||
}
|
||||
headers = {
|
||||
"X-Appbuilder-Authorization": f"Bearer {config.BAIDU_API_KEY}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
async with httpx.AsyncClient(timeout=90.0) as client:
|
||||
response = await client.post(_ENDPOINT, headers=headers, json=payload)
|
||||
response.raise_for_status()
|
||||
|
||||
hits: list[DiscoverHit] = []
|
||||
for reference in response.json().get("references", []):
|
||||
url = reference.get("url", "")
|
||||
if not url:
|
||||
continue
|
||||
content = reference.get("content", "") or ""
|
||||
hits.append(
|
||||
DiscoverHit(
|
||||
url=url,
|
||||
title=reference.get("title") or url,
|
||||
snippet=content[:_SNIPPET_MAX] or None,
|
||||
provider=self.name,
|
||||
)
|
||||
)
|
||||
if len(hits) >= top_k:
|
||||
break
|
||||
return hits
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
"""The seam every ``web.discover`` provider implements."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Protocol, runtime_checkable
|
||||
|
||||
from app.capabilities.web.discover.schemas import DiscoverHit
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class DiscoverProvider(Protocol):
|
||||
"""An env-keyed search backend that suggests candidate URLs for a query."""
|
||||
|
||||
name: str
|
||||
|
||||
def is_available(self) -> bool:
|
||||
"""True when this provider's platform key/host is configured."""
|
||||
...
|
||||
|
||||
async def search(self, query: str, top_k: int) -> list[DiscoverHit]:
|
||||
"""Return up to ``top_k`` candidate hits for ``query``."""
|
||||
...
|
||||
|
|
@ -1,40 +0,0 @@
|
|||
"""Linkup discover provider (env-keyed via ``LINKUP_API_KEY``)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
from linkup import LinkupClient
|
||||
|
||||
from app.capabilities.web.discover.schemas import DiscoverHit
|
||||
from app.config import config
|
||||
|
||||
|
||||
class LinkupProvider:
|
||||
name = "linkup"
|
||||
|
||||
def is_available(self) -> bool:
|
||||
return bool(config.LINKUP_API_KEY)
|
||||
|
||||
async def search(self, query: str, top_k: int) -> list[DiscoverHit]:
|
||||
client = LinkupClient(api_key=config.LINKUP_API_KEY)
|
||||
response = await asyncio.to_thread(
|
||||
client.search, query=query, depth="standard", output_type="searchResults"
|
||||
)
|
||||
hits: list[DiscoverHit] = []
|
||||
for result in getattr(response, "results", None) or []:
|
||||
url = getattr(result, "url", "") or ""
|
||||
if not url:
|
||||
continue
|
||||
content = getattr(result, "content", "") or ""
|
||||
hits.append(
|
||||
DiscoverHit(
|
||||
url=url,
|
||||
title=getattr(result, "name", "") or url,
|
||||
snippet=content or None,
|
||||
provider=self.name,
|
||||
)
|
||||
)
|
||||
if len(hits) >= top_k:
|
||||
break
|
||||
return hits
|
||||
|
|
@ -1,28 +0,0 @@
|
|||
"""SearXNG discover provider, wrapping the platform-env SearXNG search service."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.capabilities.web.discover.schemas import DiscoverHit
|
||||
from app.services import web_search_service
|
||||
|
||||
|
||||
class SearxngProvider:
|
||||
"""Env-keyed via ``SEARXNG_DEFAULT_HOST`` (the platform SearXNG instance)."""
|
||||
|
||||
name = "searxng"
|
||||
|
||||
def is_available(self) -> bool:
|
||||
return web_search_service.is_available()
|
||||
|
||||
async def search(self, query: str, top_k: int) -> list[DiscoverHit]:
|
||||
result_object, _documents = await web_search_service.search(query, top_k)
|
||||
return [
|
||||
DiscoverHit(
|
||||
url=source["url"],
|
||||
title=source.get("title") or source["url"],
|
||||
snippet=source.get("description") or None,
|
||||
provider=self.name,
|
||||
)
|
||||
for source in result_object.get("sources", [])
|
||||
if source.get("url")
|
||||
]
|
||||
|
|
@ -1,29 +0,0 @@
|
|||
"""``web.discover`` I/O contracts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class DiscoverInput(BaseModel):
|
||||
query: str = Field(
|
||||
description="What to search the web for, phrased in natural language."
|
||||
)
|
||||
top_k: int = Field(
|
||||
default=10, ge=1, le=50, description="Maximum number of results to return (1-50)."
|
||||
)
|
||||
|
||||
|
||||
class DiscoverHit(BaseModel):
|
||||
url: str = Field(
|
||||
description="The result's page URL; pass it to web.scrape to read it."
|
||||
)
|
||||
title: str = Field(description="The result's page title.")
|
||||
snippet: str | None = Field(
|
||||
default=None, description="A short extract summarizing the page."
|
||||
)
|
||||
provider: str = Field(description="Which search engine returned this hit.")
|
||||
|
||||
|
||||
class DiscoverOutput(BaseModel):
|
||||
hits: list[DiscoverHit] = Field(description="Ranked search results, best first.")
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
"""``web.scrape`` verb: a URL array → one cleaned row per URL."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -1,23 +0,0 @@
|
|||
"""``web.scrape`` capability registration."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.capabilities.core import BillingUnit, Capability, register_capability
|
||||
from app.capabilities.web.scrape.executor import build_scrape_executor
|
||||
from app.capabilities.web.scrape.schemas import ScrapeInput, ScrapeOutput
|
||||
|
||||
WEB_SCRAPE = Capability(
|
||||
name="web.scrape",
|
||||
description=(
|
||||
"Fetch one or more web pages and return their clean, readable content. "
|
||||
"Give it the exact URLs to read (use web.discover first if you don't have "
|
||||
"them). Returns one row per URL with a status (success/empty/failed), the "
|
||||
"page content, and metadata such as title and description."
|
||||
),
|
||||
input_schema=ScrapeInput,
|
||||
output_schema=ScrapeOutput,
|
||||
executor=build_scrape_executor(),
|
||||
billing_unit=BillingUnit.WEB_CRAWL,
|
||||
)
|
||||
|
||||
register_capability(WEB_SCRAPE)
|
||||
|
|
@ -1,45 +0,0 @@
|
|||
"""``web.scrape`` executor: fetch each URL in the array → cleaned rows."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.capabilities.core import Executor
|
||||
from app.capabilities.web.scrape.schemas import ScrapeInput, ScrapeOutput, ScrapeRow
|
||||
from app.proprietary.web_crawler import (
|
||||
CrawlOutcome,
|
||||
CrawlOutcomeStatus,
|
||||
WebCrawlerConnector,
|
||||
)
|
||||
|
||||
|
||||
def build_scrape_executor(engine: WebCrawlerConnector | None = None) -> Executor:
|
||||
"""Bind the executor to a fetch engine (defaults to the proprietary crawler)."""
|
||||
engine = engine or WebCrawlerConnector()
|
||||
|
||||
async def execute(payload: ScrapeInput) -> ScrapeOutput:
|
||||
outcomes = [await engine.crawl_url(url) for url in payload.urls]
|
||||
rows = [
|
||||
_to_row(url, outcome, payload.max_length)
|
||||
for url, outcome in zip(payload.urls, outcomes, strict=True)
|
||||
]
|
||||
return ScrapeOutput(
|
||||
rows=rows,
|
||||
captcha_attempts=sum(o.captcha_attempts for o in outcomes),
|
||||
captcha_solved=sum(1 for o in outcomes if o.captcha_solved),
|
||||
)
|
||||
|
||||
return execute
|
||||
|
||||
|
||||
def _to_row(url: str, outcome: CrawlOutcome, max_length: int) -> ScrapeRow:
|
||||
if outcome.status is CrawlOutcomeStatus.SUCCESS and outcome.result:
|
||||
content = outcome.result.get("content")
|
||||
if content is not None:
|
||||
content = content[:max_length]
|
||||
return ScrapeRow(
|
||||
url=url,
|
||||
status="success",
|
||||
content=content,
|
||||
metadata=outcome.result.get("metadata"),
|
||||
)
|
||||
status = "empty" if outcome.status is CrawlOutcomeStatus.EMPTY else "failed"
|
||||
return ScrapeRow(url=url, status=status, error=outcome.error)
|
||||
|
|
@ -1,67 +0,0 @@
|
|||
"""``web.scrape`` I/O contracts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
MAX_SCRAPE_URLS = 20
|
||||
"""Per-call batch cap: bounds a synchronous request's crawl fan-out (05)."""
|
||||
|
||||
|
||||
class ScrapeInput(BaseModel):
|
||||
urls: list[str] = Field(
|
||||
min_length=1,
|
||||
max_length=MAX_SCRAPE_URLS,
|
||||
description=(
|
||||
"Full page URLs to fetch and read (1-20), each starting with "
|
||||
"http:// or https://. Pass the exact URLs you want the content of."
|
||||
),
|
||||
)
|
||||
max_length: int = Field(
|
||||
default=50_000,
|
||||
description=(
|
||||
"Maximum characters of cleaned content returned per page; "
|
||||
"content longer than this is truncated."
|
||||
),
|
||||
)
|
||||
|
||||
@property
|
||||
def estimated_units(self) -> int:
|
||||
"""Worst-case billable crawls for pre-flight: one per requested URL."""
|
||||
return len(self.urls)
|
||||
|
||||
|
||||
class ScrapeRow(BaseModel):
|
||||
url: str = Field(description="The requested URL this result is for.")
|
||||
status: Literal["success", "empty", "failed"] = Field(
|
||||
description=(
|
||||
"'success' = content returned; 'empty' = page reached but no "
|
||||
"readable content; 'failed' = could not be fetched (see error)."
|
||||
)
|
||||
)
|
||||
content: str | None = Field(
|
||||
default=None, description="Cleaned, readable page text (present on success)."
|
||||
)
|
||||
metadata: dict[str, str] | None = Field(
|
||||
default=None, description="Page metadata such as title and description."
|
||||
)
|
||||
error: str | None = Field(
|
||||
default=None, description="Why the fetch failed (present on 'failed')."
|
||||
)
|
||||
|
||||
|
||||
class ScrapeOutput(BaseModel):
|
||||
rows: list[ScrapeRow] = Field(
|
||||
description="One result per requested URL, in the same order."
|
||||
)
|
||||
# Billing-only telemetry (Phase 3d), excluded from the client-facing result:
|
||||
# captcha solves are metered per attempt, independent of crawl success.
|
||||
captcha_attempts: int = Field(default=0, exclude=True)
|
||||
captcha_solved: int = Field(default=0, exclude=True)
|
||||
|
||||
@property
|
||||
def billable_units(self) -> int:
|
||||
"""One billable unit per successful scrape."""
|
||||
return sum(1 for row in self.rows if row.status == "success")
|
||||
|
|
@ -95,8 +95,7 @@ def test_registered_verbs_appear_on_rest():
|
|||
|
||||
router = rest.build_capabilities_router()
|
||||
paths = {route.path for route in router.routes}
|
||||
assert "/workspaces/{workspace_id}/capabilities/web.scrape" in paths
|
||||
assert "/workspaces/{workspace_id}/capabilities/web.discover" in paths
|
||||
assert "/workspaces/{workspace_id}/capabilities/web.crawl" in paths
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ import pytest
|
|||
import app.capabilities.core.billing as billing
|
||||
from app.capabilities.core.billing import charge_capability, gate_capability
|
||||
from app.capabilities.core.types import BillingUnit, CapabilityContext
|
||||
from app.capabilities.web.scrape.schemas import ScrapeInput, ScrapeOutput, ScrapeRow
|
||||
from app.capabilities.web.crawl.schemas import CrawlInput, CrawlItem, CrawlOutput
|
||||
from app.config import config
|
||||
from app.services.web_crawl_credit_service import InsufficientCreditsError
|
||||
|
||||
|
|
@ -46,10 +46,10 @@ def _make_session(owner_id, balance_micros):
|
|||
return session, fake_user
|
||||
|
||||
|
||||
def _output(*statuses: str) -> ScrapeOutput:
|
||||
return ScrapeOutput(
|
||||
rows=[
|
||||
ScrapeRow(url=f"https://{i}.com", status=status)
|
||||
def _output(*statuses: str) -> CrawlOutput:
|
||||
return CrawlOutput(
|
||||
items=[
|
||||
CrawlItem(url=f"https://{i}.com", status=status)
|
||||
for i, status in enumerate(statuses)
|
||||
]
|
||||
)
|
||||
|
|
@ -92,7 +92,7 @@ async def test_charges_workspace_owner_per_successful_crawl(monkeypatch, record_
|
|||
assert kwargs["cost_micros"] == 2000
|
||||
|
||||
|
||||
def _output_with_captcha(*statuses: str, attempts: int, solved: int) -> ScrapeOutput:
|
||||
def _output_with_captcha(*statuses: str, attempts: int, solved: int) -> CrawlOutput:
|
||||
out = _output(*statuses)
|
||||
out.captcha_attempts = attempts
|
||||
out.captcha_solved = solved
|
||||
|
|
@ -196,7 +196,7 @@ async def test_gate_blocks_when_worst_case_exceeds_balance(monkeypatch):
|
|||
|
||||
with pytest.raises(InsufficientCreditsError):
|
||||
await gate_capability(
|
||||
ScrapeInput(urls=["https://a.com", "https://b.com"]),
|
||||
CrawlInput(startUrls=["https://a.com", "https://b.com"]),
|
||||
BillingUnit.WEB_CRAWL,
|
||||
_ctx(session),
|
||||
)
|
||||
|
|
@ -208,7 +208,7 @@ async def test_gate_passes_when_balance_covers_worst_case(monkeypatch):
|
|||
session = _gate_session(_OWNER, balance_micros=100_000)
|
||||
|
||||
await gate_capability(
|
||||
ScrapeInput(urls=["https://a.com", "https://b.com"]),
|
||||
CrawlInput(startUrls=["https://a.com", "https://b.com"]),
|
||||
BillingUnit.WEB_CRAWL,
|
||||
_ctx(session),
|
||||
)
|
||||
|
|
@ -219,7 +219,7 @@ async def test_gate_is_noop_when_disabled(monkeypatch):
|
|||
session = _gate_session(_OWNER, balance_micros=0)
|
||||
|
||||
await gate_capability(
|
||||
ScrapeInput(urls=["https://a.com"]), BillingUnit.WEB_CRAWL, _ctx(session)
|
||||
CrawlInput(startUrls=["https://a.com"]), BillingUnit.WEB_CRAWL, _ctx(session)
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -233,7 +233,7 @@ async def test_gate_reserves_worst_case_captcha_when_solving_enabled(monkeypatch
|
|||
|
||||
with pytest.raises(InsufficientCreditsError):
|
||||
await gate_capability(
|
||||
ScrapeInput(urls=["https://a.com"]), BillingUnit.WEB_CRAWL, _ctx(session)
|
||||
CrawlInput(startUrls=["https://a.com"]), BillingUnit.WEB_CRAWL, _ctx(session)
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -245,7 +245,7 @@ async def test_gate_does_not_reserve_captcha_when_solving_disabled(monkeypatch):
|
|||
|
||||
# Solving off → attempts can never happen → nothing to reserve → passes.
|
||||
await gate_capability(
|
||||
ScrapeInput(urls=["https://a.com"]), BillingUnit.WEB_CRAWL, _ctx(session)
|
||||
CrawlInput(startUrls=["https://a.com"]), BillingUnit.WEB_CRAWL, _ctx(session)
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -253,6 +253,6 @@ async def test_gate_is_noop_for_free_verb(monkeypatch):
|
|||
monkeypatch.setattr(config, "WEB_CRAWL_CREDIT_BILLING_ENABLED", True)
|
||||
session = _gate_session(_OWNER, balance_micros=0)
|
||||
|
||||
await gate_capability(ScrapeInput(urls=["https://a.com"]), None, _ctx(session))
|
||||
await gate_capability(CrawlInput(startUrls=["https://a.com"]), None, _ctx(session))
|
||||
|
||||
session.execute.assert_not_called()
|
||||
|
|
|
|||
|
|
@ -9,25 +9,15 @@ from app.capabilities import (
|
|||
)
|
||||
from app.capabilities.core.store import get_capability
|
||||
from app.capabilities.core.types import BillingUnit
|
||||
from app.capabilities.web.discover.schemas import DiscoverInput, DiscoverOutput
|
||||
from app.capabilities.web.scrape.schemas import ScrapeInput, ScrapeOutput
|
||||
from app.capabilities.web.crawl.schemas import CrawlInput, CrawlOutput
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
def test_web_scrape_is_registered_with_its_schemas_and_billing_unit():
|
||||
cap = get_capability("web.scrape")
|
||||
def test_web_crawl_is_registered_with_its_schemas_and_billing_unit():
|
||||
cap = get_capability("web.crawl")
|
||||
|
||||
assert cap.name == "web.scrape"
|
||||
assert cap.input_schema is ScrapeInput
|
||||
assert cap.output_schema is ScrapeOutput
|
||||
assert cap.name == "web.crawl"
|
||||
assert cap.input_schema is CrawlInput
|
||||
assert cap.output_schema is CrawlOutput
|
||||
assert cap.billing_unit is BillingUnit.WEB_CRAWL
|
||||
|
||||
|
||||
def test_web_discover_is_registered_and_free():
|
||||
cap = get_capability("web.discover")
|
||||
|
||||
assert cap.name == "web.discover"
|
||||
assert cap.input_schema is DiscoverInput
|
||||
assert cap.output_schema is DiscoverOutput
|
||||
assert cap.billing_unit is None
|
||||
|
|
|
|||
|
|
@ -0,0 +1,113 @@
|
|||
"""``web.crawl`` executor behavior: CrawlPage list → typed CrawlOutput items.
|
||||
|
||||
Boundary mocked: the crawler engine (fake ``crawl_url`` + link graph). NOT
|
||||
mocked: the executor's page→item mapping, truncation, and captcha rollup.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from app.capabilities.web.crawl.executor import build_crawl_executor
|
||||
from app.capabilities.web.crawl.schemas import CrawlInput, CrawlOutput
|
||||
from app.proprietary.web_crawler import CrawlOutcome, CrawlOutcomeStatus
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
_SUCCESS = CrawlOutcomeStatus.SUCCESS
|
||||
|
||||
|
||||
class _FakeEngine:
|
||||
def __init__(self, graph: dict[str, tuple[CrawlOutcomeStatus, list[str]]]):
|
||||
self._graph = graph
|
||||
self.calls: list[str] = []
|
||||
|
||||
async def crawl_url(self, url: str) -> CrawlOutcome:
|
||||
self.calls.append(url)
|
||||
status, links = self._graph[url]
|
||||
if status is _SUCCESS:
|
||||
return CrawlOutcome(
|
||||
status=_SUCCESS,
|
||||
result={
|
||||
"content": f"C:{url}",
|
||||
"metadata": {"title": url},
|
||||
"links": links,
|
||||
},
|
||||
)
|
||||
return CrawlOutcome(status=status, error="boom")
|
||||
|
||||
|
||||
async def test_single_url_depth_zero_returns_one_item() -> None:
|
||||
engine = _FakeEngine({"https://e.com/": (_SUCCESS, ["https://e.com/a"])})
|
||||
execute = build_crawl_executor(engine=engine)
|
||||
|
||||
out = await execute(CrawlInput(startUrls=["https://e.com/"]))
|
||||
|
||||
assert isinstance(out, CrawlOutput)
|
||||
assert len(out.items) == 1
|
||||
item = out.items[0]
|
||||
assert item.url == "https://e.com/"
|
||||
assert item.status == "success"
|
||||
assert item.markdown == "C:https://e.com/"
|
||||
assert item.metadata == {"title": "https://e.com/"}
|
||||
assert item.crawl is not None
|
||||
assert item.crawl.depth == 0
|
||||
assert item.crawl.referrerUrl is None
|
||||
|
||||
|
||||
async def test_spider_collects_multiple_pages_with_provenance() -> None:
|
||||
engine = _FakeEngine(
|
||||
{
|
||||
"https://e.com/": (_SUCCESS, ["https://e.com/a"]),
|
||||
"https://e.com/a": (_SUCCESS, []),
|
||||
}
|
||||
)
|
||||
execute = build_crawl_executor(engine=engine)
|
||||
|
||||
out = await execute(
|
||||
CrawlInput(startUrls=["https://e.com/"], maxCrawlDepth=1, maxCrawlPages=10)
|
||||
)
|
||||
|
||||
by_url = {item.url: item for item in out.items}
|
||||
assert set(by_url) == {"https://e.com/", "https://e.com/a"}
|
||||
assert by_url["https://e.com/a"].crawl.referrerUrl == "https://e.com/"
|
||||
|
||||
|
||||
async def test_content_is_truncated_to_max_length() -> None:
|
||||
engine = _FakeEngine({"https://e.com/": (_SUCCESS, [])})
|
||||
execute = build_crawl_executor(engine=engine)
|
||||
|
||||
out = await execute(CrawlInput(startUrls=["https://e.com/"], maxLength=3))
|
||||
|
||||
assert out.items[0].markdown == "C:h"
|
||||
|
||||
|
||||
async def test_failed_page_has_no_markdown_but_keeps_error() -> None:
|
||||
engine = _FakeEngine({"https://e.com/": (CrawlOutcomeStatus.FAILED, [])})
|
||||
execute = build_crawl_executor(engine=engine)
|
||||
|
||||
out = await execute(CrawlInput(startUrls=["https://e.com/"]))
|
||||
|
||||
item = out.items[0]
|
||||
assert item.status == "failed"
|
||||
assert item.markdown is None
|
||||
assert item.error == "boom"
|
||||
|
||||
|
||||
async def test_captcha_telemetry_is_rolled_up_for_billing() -> None:
|
||||
class _CaptchaEngine:
|
||||
async def crawl_url(self, url: str) -> CrawlOutcome:
|
||||
return CrawlOutcome(
|
||||
status=_SUCCESS,
|
||||
result={"content": "ok", "metadata": {}, "links": []},
|
||||
captcha_attempts=2,
|
||||
captcha_solved=True,
|
||||
)
|
||||
|
||||
execute = build_crawl_executor(engine=_CaptchaEngine())
|
||||
|
||||
out = await execute(CrawlInput(startUrls=["https://e.com/"]))
|
||||
|
||||
assert out.captcha_attempts == 2
|
||||
assert out.captcha_solved == 1
|
||||
assert out.billable_units == 1
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
"""``web.crawl`` I/O contract: camelCase surface, bounds, and billing counters."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from app.capabilities.web.crawl.schemas import (
|
||||
CrawlInput,
|
||||
CrawlItem,
|
||||
CrawlMeta,
|
||||
CrawlOutput,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
def test_requires_at_least_one_start_url() -> None:
|
||||
with pytest.raises(ValidationError):
|
||||
CrawlInput(startUrls=[])
|
||||
|
||||
|
||||
def test_camelcase_fields_and_defaults() -> None:
|
||||
model = CrawlInput(startUrls=["https://e.com"])
|
||||
assert model.startUrls == ["https://e.com"]
|
||||
assert model.maxCrawlDepth == 0
|
||||
assert model.maxCrawlPages == 10
|
||||
assert model.maxLength == 50_000
|
||||
|
||||
|
||||
def test_depth_and_page_bounds_are_enforced() -> None:
|
||||
with pytest.raises(ValidationError):
|
||||
CrawlInput(startUrls=["https://e.com"], maxCrawlDepth=-1)
|
||||
with pytest.raises(ValidationError):
|
||||
CrawlInput(startUrls=["https://e.com"], maxCrawlDepth=99)
|
||||
with pytest.raises(ValidationError):
|
||||
CrawlInput(startUrls=["https://e.com"], maxCrawlPages=0)
|
||||
|
||||
|
||||
def test_estimated_units_for_single_url_is_seed_count() -> None:
|
||||
model = CrawlInput(startUrls=["https://a.com", "https://b.com"], maxCrawlDepth=0)
|
||||
assert model.estimated_units == 2
|
||||
|
||||
|
||||
def test_estimated_units_for_spider_is_max_pages() -> None:
|
||||
model = CrawlInput(
|
||||
startUrls=["https://a.com"], maxCrawlDepth=2, maxCrawlPages=25
|
||||
)
|
||||
assert model.estimated_units == 25
|
||||
|
||||
|
||||
def test_billable_units_counts_only_successes() -> None:
|
||||
out = CrawlOutput(
|
||||
items=[
|
||||
CrawlItem(url="a", status="success", crawl=CrawlMeta(loadedUrl="a", depth=0)),
|
||||
CrawlItem(url="b", status="empty", crawl=CrawlMeta(loadedUrl="b", depth=1)),
|
||||
CrawlItem(url="c", status="failed", crawl=CrawlMeta(loadedUrl="c", depth=1)),
|
||||
]
|
||||
)
|
||||
assert out.billable_units == 1
|
||||
|
||||
|
||||
def test_captcha_counters_are_excluded_from_the_wire_shape() -> None:
|
||||
out = CrawlOutput(items=[], captcha_attempts=3, captcha_solved=1)
|
||||
dumped = out.model_dump()
|
||||
assert "captcha_attempts" not in dumped
|
||||
assert "captcha_solved" not in dumped
|
||||
|
|
@ -1,91 +0,0 @@
|
|||
"""BaiduProvider maps the Baidu AI Search references to DiscoverHits, env-keyed.
|
||||
|
||||
Boundary mocked: httpx.AsyncClient + config key. NOT mocked: reference→hit mapping.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
import app.capabilities.web.discover.providers.baidu as baidu_module
|
||||
from app.capabilities.web.discover.providers.baidu import BaiduProvider
|
||||
from app.config import config
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
class _FakeResponse:
|
||||
def __init__(self, payload):
|
||||
self._payload = payload
|
||||
|
||||
def raise_for_status(self):
|
||||
return None
|
||||
|
||||
def json(self):
|
||||
return self._payload
|
||||
|
||||
|
||||
class _FakeAsyncClient:
|
||||
"""Minimal async-context httpx stand-in returning a canned payload."""
|
||||
|
||||
payload: dict = {}
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
pass
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *exc):
|
||||
return False
|
||||
|
||||
async def post(self, url, headers=None, json=None):
|
||||
return _FakeResponse(type(self).payload)
|
||||
|
||||
|
||||
def _install(monkeypatch, payload):
|
||||
monkeypatch.setattr(config, "BAIDU_API_KEY", "k")
|
||||
_FakeAsyncClient.payload = payload
|
||||
monkeypatch.setattr(baidu_module.httpx, "AsyncClient", _FakeAsyncClient)
|
||||
|
||||
|
||||
def test_is_available_reflects_the_env_key(monkeypatch):
|
||||
monkeypatch.setattr(config, "BAIDU_API_KEY", "k")
|
||||
assert BaiduProvider().is_available() is True
|
||||
monkeypatch.setattr(config, "BAIDU_API_KEY", None)
|
||||
assert BaiduProvider().is_available() is False
|
||||
|
||||
|
||||
async def test_maps_references_to_hits(monkeypatch):
|
||||
_install(
|
||||
monkeypatch,
|
||||
{
|
||||
"references": [
|
||||
{"title": "Acme", "url": "https://acme.cn", "content": "hello"},
|
||||
{"title": "No URL", "url": "", "content": "skip"},
|
||||
]
|
||||
},
|
||||
)
|
||||
|
||||
hits = await BaiduProvider().search("acme", top_k=10)
|
||||
|
||||
assert [h.url for h in hits] == ["https://acme.cn"]
|
||||
assert hits[0].title == "Acme"
|
||||
assert hits[0].snippet == "hello"
|
||||
assert hits[0].provider == "baidu"
|
||||
|
||||
|
||||
async def test_respects_top_k(monkeypatch):
|
||||
_install(
|
||||
monkeypatch,
|
||||
{
|
||||
"references": [
|
||||
{"title": f"n{i}", "url": f"https://{i}.cn", "content": "c"}
|
||||
for i in range(5)
|
||||
]
|
||||
},
|
||||
)
|
||||
|
||||
hits = await BaiduProvider().search("q", top_k=2)
|
||||
|
||||
assert len(hits) == 2
|
||||
|
|
@ -1,99 +0,0 @@
|
|||
"""`web.discover` executor: pick the first configured provider; self-disable when none.
|
||||
|
||||
Boundary mocked: the providers (injected fakes). NOT mocked: the executor's
|
||||
provider-selection and self-disable behavior.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from app.capabilities.web.discover.executor import (
|
||||
NoDiscoverProviderError,
|
||||
build_discover_executor,
|
||||
)
|
||||
from app.capabilities.web.discover.schemas import (
|
||||
DiscoverHit,
|
||||
DiscoverInput,
|
||||
DiscoverOutput,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
class _FakeProvider:
|
||||
def __init__(
|
||||
self, name: str, available: bool, hits: list[DiscoverHit] | None = None
|
||||
):
|
||||
self.name = name
|
||||
self._available = available
|
||||
self._hits = hits or []
|
||||
self.calls: list[tuple[str, int]] = []
|
||||
|
||||
def is_available(self) -> bool:
|
||||
return self._available
|
||||
|
||||
async def search(self, query: str, top_k: int) -> list[DiscoverHit]:
|
||||
self.calls.append((query, top_k))
|
||||
return self._hits
|
||||
|
||||
|
||||
def _hit(url: str, provider: str) -> DiscoverHit:
|
||||
return DiscoverHit(url=url, title=url, snippet="s", provider=provider)
|
||||
|
||||
|
||||
async def test_uses_the_first_available_provider():
|
||||
first = _FakeProvider(
|
||||
"searxng", available=True, hits=[_hit("https://a.com", "searxng")]
|
||||
)
|
||||
second = _FakeProvider(
|
||||
"linkup", available=True, hits=[_hit("https://b.com", "linkup")]
|
||||
)
|
||||
execute = build_discover_executor(providers=[first, second])
|
||||
|
||||
out = await execute(DiscoverInput(query="acme pricing", top_k=5))
|
||||
|
||||
assert isinstance(out, DiscoverOutput)
|
||||
assert [h.url for h in out.hits] == ["https://a.com"]
|
||||
assert first.calls == [("acme pricing", 5)]
|
||||
assert second.calls == [] # first available short-circuits
|
||||
|
||||
|
||||
async def test_skips_unavailable_providers():
|
||||
off = _FakeProvider("searxng", available=False)
|
||||
on = _FakeProvider("linkup", available=True, hits=[_hit("https://b.com", "linkup")])
|
||||
execute = build_discover_executor(providers=[off, on])
|
||||
|
||||
out = await execute(DiscoverInput(query="q"))
|
||||
|
||||
assert [h.provider for h in out.hits] == ["linkup"]
|
||||
assert off.calls == []
|
||||
|
||||
|
||||
async def test_self_disables_when_no_provider_is_configured():
|
||||
execute = build_discover_executor(
|
||||
providers=[_FakeProvider("searxng", available=False)]
|
||||
)
|
||||
|
||||
with pytest.raises(NoDiscoverProviderError):
|
||||
await execute(DiscoverInput(query="q"))
|
||||
|
||||
|
||||
async def test_caps_hits_to_top_k_when_provider_over_returns():
|
||||
# SearXNG treats `limit` as a hint and can return more rows than asked; the
|
||||
# verb must honor its own documented `top_k` cap regardless of the provider.
|
||||
over = _FakeProvider(
|
||||
"searxng",
|
||||
available=True,
|
||||
hits=[_hit(f"https://{i}.com", "searxng") for i in range(5)],
|
||||
)
|
||||
execute = build_discover_executor(providers=[over])
|
||||
|
||||
out = await execute(DiscoverInput(query="q", top_k=3))
|
||||
|
||||
assert len(out.hits) == 3
|
||||
assert [h.url for h in out.hits] == [
|
||||
"https://0.com",
|
||||
"https://1.com",
|
||||
"https://2.com",
|
||||
]
|
||||
|
|
@ -1,75 +0,0 @@
|
|||
"""LinkupProvider maps the Linkup SDK results to DiscoverHits, env-keyed.
|
||||
|
||||
Boundary mocked: the LinkupClient SDK + config key. NOT mocked: result→hit mapping.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
import app.capabilities.web.discover.providers.linkup as linkup_module
|
||||
from app.capabilities.web.discover.providers.linkup import LinkupProvider
|
||||
from app.config import config
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
class _Result:
|
||||
def __init__(self, name, url, content):
|
||||
self.name = name
|
||||
self.url = url
|
||||
self.content = content
|
||||
self.type = "text"
|
||||
|
||||
|
||||
class _Response:
|
||||
def __init__(self, results):
|
||||
self.results = results
|
||||
|
||||
|
||||
class _FakeClient:
|
||||
def __init__(self, api_key):
|
||||
self.api_key = api_key
|
||||
|
||||
def search(self, *, query, depth, output_type):
|
||||
return _Response(
|
||||
[
|
||||
_Result("Acme", "https://acme.com", "acme home"),
|
||||
_Result("No URL", "", "skip"),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def test_is_available_reflects_the_env_key(monkeypatch):
|
||||
monkeypatch.setattr(config, "LINKUP_API_KEY", "k")
|
||||
assert LinkupProvider().is_available() is True
|
||||
monkeypatch.setattr(config, "LINKUP_API_KEY", None)
|
||||
assert LinkupProvider().is_available() is False
|
||||
|
||||
|
||||
async def test_maps_results_to_hits(monkeypatch):
|
||||
monkeypatch.setattr(config, "LINKUP_API_KEY", "k")
|
||||
monkeypatch.setattr(linkup_module, "LinkupClient", _FakeClient)
|
||||
|
||||
hits = await LinkupProvider().search("acme", top_k=10)
|
||||
|
||||
assert [h.url for h in hits] == ["https://acme.com"]
|
||||
assert hits[0].title == "Acme"
|
||||
assert hits[0].snippet == "acme home"
|
||||
assert hits[0].provider == "linkup"
|
||||
|
||||
|
||||
async def test_respects_top_k(monkeypatch):
|
||||
monkeypatch.setattr(config, "LINKUP_API_KEY", "k")
|
||||
|
||||
class _ManyClient(_FakeClient):
|
||||
def search(self, *, query, depth, output_type):
|
||||
return _Response(
|
||||
[_Result(f"n{i}", f"https://{i}.com", "c") for i in range(5)]
|
||||
)
|
||||
|
||||
monkeypatch.setattr(linkup_module, "LinkupClient", _ManyClient)
|
||||
|
||||
hits = await LinkupProvider().search("q", top_k=2)
|
||||
|
||||
assert len(hits) == 2
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
"""``web.discover`` I/O contract bounds."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from app.capabilities.web.discover.schemas import DiscoverInput
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
def test_top_k_defaults_and_accepts_the_valid_range():
|
||||
assert DiscoverInput(query="q").top_k == 10
|
||||
assert DiscoverInput(query="q", top_k=1).top_k == 1
|
||||
assert DiscoverInput(query="q", top_k=50).top_k == 50
|
||||
|
||||
|
||||
@pytest.mark.parametrize("bad", [0, -1, 51, 100])
|
||||
def test_top_k_outside_1_to_50_is_rejected(bad):
|
||||
with pytest.raises(ValidationError):
|
||||
DiscoverInput(query="q", top_k=bad)
|
||||
|
|
@ -1,58 +0,0 @@
|
|||
"""SearxngProvider maps the SearXNG service's sources to DiscoverHits.
|
||||
|
||||
Boundary mocked: the web_search_service module. NOT mocked: the source→hit mapping.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
|
||||
import app.capabilities.web.discover.providers.searxng as searxng_module
|
||||
from app.capabilities.web.discover.providers.searxng import SearxngProvider
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
def _result(sources):
|
||||
return ({"sources": sources}, [])
|
||||
|
||||
|
||||
async def test_maps_sources_to_hits(monkeypatch):
|
||||
provider = SearxngProvider()
|
||||
monkeypatch.setattr(
|
||||
searxng_module.web_search_service,
|
||||
"search",
|
||||
AsyncMock(
|
||||
return_value=_result(
|
||||
[
|
||||
{"title": "Acme", "url": "https://acme.com", "description": "home"},
|
||||
{
|
||||
"title": "Docs",
|
||||
"url": "https://acme.com/docs",
|
||||
"description": "",
|
||||
},
|
||||
{"title": "no url", "url": "", "description": "skip me"},
|
||||
]
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
hits = await provider.search("acme", top_k=5)
|
||||
|
||||
assert [h.url for h in hits] == ["https://acme.com", "https://acme.com/docs"]
|
||||
assert hits[0].title == "Acme"
|
||||
assert hits[0].snippet == "home"
|
||||
assert hits[1].snippet is None # empty description normalizes to None
|
||||
assert all(h.provider == "searxng" for h in hits)
|
||||
|
||||
|
||||
def test_is_available_reflects_the_service(monkeypatch):
|
||||
provider = SearxngProvider()
|
||||
monkeypatch.setattr(searxng_module.web_search_service, "is_available", lambda: True)
|
||||
assert provider.is_available() is True
|
||||
monkeypatch.setattr(
|
||||
searxng_module.web_search_service, "is_available", lambda: False
|
||||
)
|
||||
assert provider.is_available() is False
|
||||
|
|
@ -1,141 +0,0 @@
|
|||
"""`web.scrape` executor behavior: URLs in → cleaned rows out.
|
||||
|
||||
Boundary mocked: the crawler (injected fake). NOT mocked: the executor's own
|
||||
CrawlOutcome → ScrapeRow mapping.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from app.capabilities.web.scrape.executor import build_scrape_executor
|
||||
from app.capabilities.web.scrape.schemas import ScrapeInput, ScrapeOutput
|
||||
from app.proprietary.web_crawler import CrawlOutcome, CrawlOutcomeStatus
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
class _FakeCrawler:
|
||||
"""Stand-in for WebCrawlerConnector: serves a canned outcome per URL."""
|
||||
|
||||
def __init__(self, outcomes: dict[str, CrawlOutcome]):
|
||||
self._outcomes = outcomes
|
||||
self.calls: list[str] = []
|
||||
|
||||
async def crawl_url(self, url: str) -> CrawlOutcome:
|
||||
self.calls.append(url)
|
||||
return self._outcomes[url]
|
||||
|
||||
|
||||
def _success(content: str, metadata: dict[str, str]) -> CrawlOutcome:
|
||||
return CrawlOutcome(
|
||||
status=CrawlOutcomeStatus.SUCCESS,
|
||||
result={
|
||||
"content": content,
|
||||
"metadata": metadata,
|
||||
"crawler_type": "scrapling-static",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
async def test_scrape_returns_one_cleaned_row_for_a_successful_url():
|
||||
url = "https://example.com"
|
||||
crawler = _FakeCrawler({url: _success("# Hello", {"title": "Hello"})})
|
||||
execute = build_scrape_executor(engine=crawler)
|
||||
|
||||
out = await execute(ScrapeInput(urls=[url]))
|
||||
|
||||
assert isinstance(out, ScrapeOutput)
|
||||
assert len(out.rows) == 1
|
||||
row = out.rows[0]
|
||||
assert row.url == url
|
||||
assert row.status == "success"
|
||||
assert row.content == "# Hello"
|
||||
assert row.metadata == {"title": "Hello"}
|
||||
|
||||
|
||||
async def test_scrape_returns_one_row_per_url_in_input_order():
|
||||
a, b, c = "https://a.com", "https://b.com", "https://c.com"
|
||||
crawler = _FakeCrawler(
|
||||
{
|
||||
a: _success("A", {"title": "A"}),
|
||||
b: _success("B", {"title": "B"}),
|
||||
c: _success("C", {"title": "C"}),
|
||||
}
|
||||
)
|
||||
execute = build_scrape_executor(engine=crawler)
|
||||
|
||||
out = await execute(ScrapeInput(urls=[a, b, c]))
|
||||
|
||||
assert [row.url for row in out.rows] == [a, b, c]
|
||||
assert [row.content for row in out.rows] == ["A", "B", "C"]
|
||||
|
||||
|
||||
async def test_content_longer_than_max_length_is_truncated():
|
||||
url = "https://long.com"
|
||||
crawler = _FakeCrawler({url: _success("A" * 100, {"title": "Long"})})
|
||||
execute = build_scrape_executor(engine=crawler)
|
||||
|
||||
out = await execute(ScrapeInput(urls=[url], max_length=10))
|
||||
|
||||
assert out.rows[0].content == "A" * 10
|
||||
|
||||
|
||||
async def test_content_within_max_length_is_untouched():
|
||||
url = "https://short.com"
|
||||
crawler = _FakeCrawler({url: _success("hello", {"title": "Short"})})
|
||||
execute = build_scrape_executor(engine=crawler)
|
||||
|
||||
out = await execute(ScrapeInput(urls=[url], max_length=10))
|
||||
|
||||
assert out.rows[0].content == "hello"
|
||||
|
||||
|
||||
async def test_scrape_surfaces_total_captcha_attempts_for_billing():
|
||||
ok, blocked = "https://ok.com", "https://blocked.com"
|
||||
crawler = _FakeCrawler(
|
||||
{
|
||||
ok: CrawlOutcome(
|
||||
status=CrawlOutcomeStatus.SUCCESS,
|
||||
result={"content": "OK", "metadata": {}},
|
||||
captcha_attempts=2,
|
||||
captcha_solved=True,
|
||||
),
|
||||
blocked: CrawlOutcome(
|
||||
status=CrawlOutcomeStatus.FAILED,
|
||||
error="blocked",
|
||||
captcha_attempts=1,
|
||||
captcha_solved=False,
|
||||
),
|
||||
}
|
||||
)
|
||||
execute = build_scrape_executor(engine=crawler)
|
||||
|
||||
out = await execute(ScrapeInput(urls=[ok, blocked]))
|
||||
|
||||
# Attempts bill even when the crawl ultimately failed (Phase 3d).
|
||||
assert out.captcha_attempts == 3
|
||||
assert out.captcha_solved == 1
|
||||
|
||||
|
||||
async def test_partial_failure_keeps_the_batch_and_labels_each_url():
|
||||
ok, empty, failed = "https://ok.com", "https://empty.com", "https://failed.com"
|
||||
crawler = _FakeCrawler(
|
||||
{
|
||||
ok: _success("OK", {"title": "OK"}),
|
||||
empty: CrawlOutcome(status=CrawlOutcomeStatus.EMPTY, error="no content"),
|
||||
failed: CrawlOutcome(status=CrawlOutcomeStatus.FAILED, error="blocked"),
|
||||
}
|
||||
)
|
||||
execute = build_scrape_executor(engine=crawler)
|
||||
|
||||
out = await execute(ScrapeInput(urls=[ok, empty, failed]))
|
||||
|
||||
by_url = {row.url: row for row in out.rows}
|
||||
assert {u: r.status for u, r in by_url.items()} == {
|
||||
ok: "success",
|
||||
empty: "empty",
|
||||
failed: "failed",
|
||||
}
|
||||
assert by_url[failed].content is None
|
||||
assert by_url[failed].error == "blocked"
|
||||
|
|
@ -1,47 +0,0 @@
|
|||
"""ScrapeOutput reports its own billable count; ScrapeInput bounds its batch size."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from app.capabilities.web.scrape.schemas import (
|
||||
MAX_SCRAPE_URLS,
|
||||
ScrapeInput,
|
||||
ScrapeOutput,
|
||||
ScrapeRow,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
def _output(*statuses: str) -> ScrapeOutput:
|
||||
return ScrapeOutput(
|
||||
rows=[
|
||||
ScrapeRow(url=f"https://{i}.com", status=status)
|
||||
for i, status in enumerate(statuses)
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def test_billable_units_counts_successful_rows():
|
||||
assert _output("success", "empty", "success", "failed").billable_units == 2
|
||||
|
||||
|
||||
def test_billable_units_is_zero_without_successes():
|
||||
assert _output("empty", "failed").billable_units == 0
|
||||
|
||||
|
||||
def test_rejects_empty_url_batch():
|
||||
with pytest.raises(ValidationError):
|
||||
ScrapeInput(urls=[])
|
||||
|
||||
|
||||
def test_rejects_batch_over_the_cap():
|
||||
with pytest.raises(ValidationError):
|
||||
ScrapeInput(urls=[f"https://{i}.com" for i in range(MAX_SCRAPE_URLS + 1)])
|
||||
|
||||
|
||||
def test_accepts_batch_at_the_cap():
|
||||
payload = ScrapeInput(urls=[f"https://{i}.com" for i in range(MAX_SCRAPE_URLS)])
|
||||
assert payload.estimated_units == MAX_SCRAPE_URLS
|
||||
Loading…
Add table
Add a link
Reference in a new issue