mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-06 22:12:12 +02:00
feat(capabilities): add web.scrape schemas
This commit is contained in:
parent
bfe3117302
commit
bbd81bcc99
3 changed files with 37 additions and 0 deletions
5
surfsense_backend/app/capabilities/web/__init__.py
Normal file
5
surfsense_backend/app/capabilities/web/__init__.py
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
"""``web.*`` namespace: the generic web scraping verbs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.capabilities.web.scrape import definition # noqa: F401 — registers web.scrape
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
"""``web.scrape`` verb: a URL array → one cleaned row per URL."""
|
||||
|
||||
from __future__ import annotations
|
||||
29
surfsense_backend/app/capabilities/web/scrape/schemas.py
Normal file
29
surfsense_backend/app/capabilities/web/scrape/schemas.py
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
"""``web.scrape`` I/O contracts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class ScrapeInput(BaseModel):
|
||||
urls: list[str]
|
||||
max_length: int = 50_000
|
||||
|
||||
|
||||
class ScrapeRow(BaseModel):
|
||||
url: str
|
||||
status: Literal["success", "empty", "failed"]
|
||||
content: str | None = None
|
||||
metadata: dict[str, str] | None = None
|
||||
error: str | None = None
|
||||
|
||||
|
||||
class ScrapeOutput(BaseModel):
|
||||
rows: list[ScrapeRow]
|
||||
|
||||
@property
|
||||
def billable_units(self) -> int:
|
||||
"""One billable unit per successful scrape."""
|
||||
return sum(1 for row in self.rows if row.status == "success")
|
||||
Loading…
Add table
Add a link
Reference in a new issue