apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-07-22 23:31:12 +02:00
Code Issues Projects Releases Packages Wiki Activity

feat(capabilities): meter captcha, truncate scrape, bound discover

Browse source
This commit is contained in:
CREDO23 2026-07-02 18:32:42 +02:00
parent 88a94611c9
commit 790507d107
7 changed files with 212 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

19
surfsense_backend/app/capabilities/web/scrape/executor.py
View file

@ -16,18 +16,29 @@ def build_scrape_executor(engine: WebCrawlerConnector | None = None) -> Executor
engine = engine or WebCrawlerConnector()
async def execute(payload: ScrapeInput) -> ScrapeOutput:
rows = [_to_row(url, await engine.crawl_url(url)) for url in payload.urls]
return ScrapeOutput(rows=rows)
outcomes = [await engine.crawl_url(url) for url in payload.urls]
rows = [
_to_row(url, outcome, payload.max_length)
for url, outcome in zip(payload.urls, outcomes, strict=True)
]
return ScrapeOutput(
rows=rows,
captcha_attempts=sum(o.captcha_attempts for o in outcomes),
captcha_solved=sum(1 for o in outcomes if o.captcha_solved),
)
return execute
def _to_row(url: str, outcome: CrawlOutcome) -> ScrapeRow:
def _to_row(url: str, outcome: CrawlOutcome, max_length: int) -> ScrapeRow:
if outcome.status is CrawlOutcomeStatus.SUCCESS and outcome.result:
content = outcome.result.get("content")
if content is not None:
content = content[:max_length]
return ScrapeRow(
url=url,
status="success",
content=outcome.result.get("content"),
content=content,
metadata=outcome.result.get("metadata"),
)
status = "empty" if outcome.status is CrawlOutcomeStatus.EMPTY else "failed"