From 6679fda6624a3b22e80dbbc4644b100ea96d131e Mon Sep 17 00:00:00 2001
From: CREDO23 <bakerathierry@gmail.com>
Date: Wed, 1 Jul 2026 16:42:47 +0200
Subject: [PATCH] feat(capabilities): wire web.scrape to proprietary crawler

---
 .../app/capabilities/web/scrape/executor.py   | 34 +++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100644 surfsense_backend/app/capabilities/web/scrape/executor.py

diff --git a/surfsense_backend/app/capabilities/web/scrape/executor.py b/surfsense_backend/app/capabilities/web/scrape/executor.py
new file mode 100644
index 000000000..46f067b56
--- /dev/null
+++ b/surfsense_backend/app/capabilities/web/scrape/executor.py
@@ -0,0 +1,34 @@
+"""``web.scrape`` executor: fetch each URL in the array → cleaned rows."""
+
+from __future__ import annotations
+
+from app.capabilities.types import Executor
+from app.capabilities.web.scrape.schemas import ScrapeInput, ScrapeOutput, ScrapeRow
+from app.proprietary.web_crawler import (
+    CrawlOutcome,
+    CrawlOutcomeStatus,
+    WebCrawlerConnector,
+)
+
+
+def build_scrape_executor(engine: WebCrawlerConnector | None = None) -> Executor:
+    """Bind the executor to a fetch engine (defaults to the proprietary crawler)."""
+    engine = engine or WebCrawlerConnector()
+
+    async def execute(payload: ScrapeInput) -> ScrapeOutput:
+        rows = [_to_row(url, await engine.crawl_url(url)) for url in payload.urls]
+        return ScrapeOutput(rows=rows)
+
+    return execute
+
+
+def _to_row(url: str, outcome: CrawlOutcome) -> ScrapeRow:
+    if outcome.status is CrawlOutcomeStatus.SUCCESS and outcome.result:
+        return ScrapeRow(
+            url=url,
+            status="success",
+            content=outcome.result.get("content"),
+            metadata=outcome.result.get("metadata"),
+        )
+    status = "empty" if outcome.status is CrawlOutcomeStatus.EMPTY else "failed"
+    return ScrapeRow(url=url, status=status, error=outcome.error)