mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-14 22:52:15 +02:00
feat(capabilities): add web.scrape schemas
This commit is contained in:
parent
bfe3117302
commit
bbd81bcc99
3 changed files with 37 additions and 0 deletions
5
surfsense_backend/app/capabilities/web/__init__.py
Normal file
5
surfsense_backend/app/capabilities/web/__init__.py
Normal file
|
|
@ -0,0 +1,5 @@
|
||||||
|
"""``web.*`` namespace: the generic web scraping verbs."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from app.capabilities.web.scrape import definition # noqa: F401 — registers web.scrape
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
"""``web.scrape`` verb: a URL array → one cleaned row per URL."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
29
surfsense_backend/app/capabilities/web/scrape/schemas.py
Normal file
29
surfsense_backend/app/capabilities/web/scrape/schemas.py
Normal file
|
|
@ -0,0 +1,29 @@
|
||||||
|
"""``web.scrape`` I/O contracts."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Literal
|
||||||
|
|
||||||
|
from pydantic import BaseModel
|
||||||
|
|
||||||
|
|
||||||
|
class ScrapeInput(BaseModel):
|
||||||
|
urls: list[str]
|
||||||
|
max_length: int = 50_000
|
||||||
|
|
||||||
|
|
||||||
|
class ScrapeRow(BaseModel):
|
||||||
|
url: str
|
||||||
|
status: Literal["success", "empty", "failed"]
|
||||||
|
content: str | None = None
|
||||||
|
metadata: dict[str, str] | None = None
|
||||||
|
error: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class ScrapeOutput(BaseModel):
|
||||||
|
rows: list[ScrapeRow]
|
||||||
|
|
||||||
|
@property
|
||||||
|
def billable_units(self) -> int:
|
||||||
|
"""One billable unit per successful scrape."""
|
||||||
|
return sum(1 for row in self.rows if row.status == "success")
|
||||||
Loading…
Add table
Add a link
Reference in a new issue