feat(03a): implement Scrapling-only web crawler and remove Firecrawl integration

- Standardized the web crawler to use Scrapling exclusively, removing Firecrawl entirely.
- Updated the crawler's location to `app/proprietary/web_crawler/connector.py` under a non-Apache-2 license boundary.
- Refactored the `WebCrawlerConnector` to eliminate the Firecrawl API key dependency, simplifying the interface for crawling URLs.
- Adjusted related components to accommodate the new structure and ensure successful crawl outcomes are properly handled.
- Updated documentation to reflect these changes and the new implementation status.

surfsense_backend/tests/unit/automations/actions/builtin/agent_task/test_dependencies.py

@@ -39,9 +39,9 @@ def patched_side_effects(monkeypatch: pytest.MonkeyPatch):
     """Stub the connector setup + checkpointer so only policy/LLM logic runs."""
 
     async def _fake_setup(_session, *, workspace_id):
-        return (SimpleNamespace(name="connector"), "fc-key")
+        return SimpleNamespace(name="connector")
 
-    monkeypatch.setattr(deps_mod, "setup_connector_and_firecrawl", _fake_setup)
+    monkeypatch.setattr(deps_mod, "setup_connector_service", _fake_setup)
     return None
 
 
@@ -78,7 +78,7 @@ async def test_build_dependencies_resolves_captured_chat_model_id(
 
     assert captured == {"config_id": -7, "workspace_id": 42}
     assert result.llm.name == "llm"
-    assert result.firecrawl_api_key == "fc-key"
+    assert result.connector_service.name == "connector"
 
 
 async def test_build_dependencies_validates_captured_ids(

surfsense_backend/tests/unit/proprietary/web_crawler/test_connector.py

@@ -0,0 +1,118 @@
+"""Unit tests for ``WebCrawlerConnector.crawl_url`` outcome semantics (Phase 3a).
+
+These exercise the Scrapling tier ladder and the explicit ``CrawlOutcome``
+contract that Phase 3c bills on, by stubbing the per-tier fetch helpers so the
+ladder logic is tested deterministically without launching browsers/HTTP.
+"""
+
+import pytest
+
+from app.proprietary.web_crawler import (
+    CrawlOutcomeStatus,
+    WebCrawlerConnector,
+)
+
+pytestmark = pytest.mark.unit
+
+
+def _result(tier: str) -> dict:
+    return {
+        "content": "hello world",
+        "metadata": {"source": "https://example.com", "title": "Example"},
+        "crawler_type": tier,
+    }
+
+
+async def test_invalid_url_is_failed() -> None:
+    """A URL that fails validation never reaches a tier and is FAILED."""
+    outcome = await WebCrawlerConnector().crawl_url("not a url")
+
+    assert outcome.status is CrawlOutcomeStatus.FAILED
+    assert outcome.result is None
+    assert "Invalid URL" in (outcome.error or "")
+
+
+async def test_static_tier_success_short_circuits(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Static success returns SUCCESS and never touches the browser tiers."""
+    crawler = WebCrawlerConnector()
+    later_calls: list[str] = []
+
+    async def _static(_url: str) -> dict:
+        return _result("scrapling-static")
+
+    async def _record_dynamic(_url: str) -> None:
+        later_calls.append("dynamic")
+        return None
+
+    async def _record_stealthy(_url: str) -> None:
+        later_calls.append("stealthy")
+        return None
+
+    monkeypatch.setattr(crawler, "_crawl_with_async_fetcher", _static)
+    monkeypatch.setattr(crawler, "_crawl_with_dynamic", _record_dynamic)
+    monkeypatch.setattr(crawler, "_crawl_with_stealthy", _record_stealthy)
+
+    outcome = await crawler.crawl_url("https://example.com")
+
+    assert outcome.status is CrawlOutcomeStatus.SUCCESS
+    assert outcome.tier == "scrapling-static"
+    assert outcome.result is not None
+    assert outcome.result["crawler_type"] == "scrapling-static"
+    assert later_calls == []
+
+
+async def test_escalates_to_dynamic_on_static_miss(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Static empty extraction escalates to the dynamic tier."""
+    crawler = WebCrawlerConnector()
+
+    async def _empty(_url: str) -> None:
+        return None
+
+    async def _dynamic(_url: str) -> dict:
+        return _result("scrapling-dynamic")
+
+    monkeypatch.setattr(crawler, "_crawl_with_async_fetcher", _empty)
+    monkeypatch.setattr(crawler, "_crawl_with_dynamic", _dynamic)
+
+    outcome = await crawler.crawl_url("https://example.com")
+
+    assert outcome.status is CrawlOutcomeStatus.SUCCESS
+    assert outcome.tier == "scrapling-dynamic"
+
+
+async def test_all_tiers_empty_is_empty(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Every tier fetched but extracted nothing -> EMPTY (not billable)."""
+    crawler = WebCrawlerConnector()
+
+    async def _empty(_url: str) -> None:
+        return None
+
+    monkeypatch.setattr(crawler, "_crawl_with_async_fetcher", _empty)
+    monkeypatch.setattr(crawler, "_crawl_with_dynamic", _empty)
+    monkeypatch.setattr(crawler, "_crawl_with_stealthy", _empty)
+
+    outcome = await crawler.crawl_url("https://example.com")
+
+    assert outcome.status is CrawlOutcomeStatus.EMPTY
+    assert outcome.result is None
+
+
+async def test_all_tiers_raise_is_failed(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Every tier raising (none reachable) -> FAILED with aggregated errors."""
+    crawler = WebCrawlerConnector()
+
+    async def _boom(_url: str) -> None:
+        raise RuntimeError("fetch exploded")
+
+    monkeypatch.setattr(crawler, "_crawl_with_async_fetcher", _boom)
+    monkeypatch.setattr(crawler, "_crawl_with_dynamic", _boom)
+    monkeypatch.setattr(crawler, "_crawl_with_stealthy", _boom)
+
+    outcome = await crawler.crawl_url("https://example.com")
+
+    assert outcome.status is CrawlOutcomeStatus.FAILED
+    assert "fetch exploded" in (outcome.error or "")

surfsense_backend/tests/unit/utils/test_validators.py

@@ -332,9 +332,8 @@ def test_validate_connector_config_invalid():
     with pytest.raises(ValueError):
         validate_connector_config("SEARXNG_API", {"SEARXNG_HOST": "not-a-url"})
 
-    # Invalid email format (if JIRA was enabled, etc. We test with WEBCRAWLER's custom validation)
-    # Firecrawl key format error:
+    # WEBCRAWLER_CONNECTOR custom validation: malformed INITIAL_URLS rejected.
     with pytest.raises(ValueError):
         validate_connector_config(
-            "WEBCRAWLER_CONNECTOR", {"FIRECRAWL_API_KEY": "invalid-prefix-key"}
+            "WEBCRAWLER_CONNECTOR", {"INITIAL_URLS": "not-a-url"}
         )