feat(03a): implement Scrapling-only web crawler and remove Firecrawl integration

- Standardized the web crawler to use Scrapling exclusively, removing Firecrawl entirely.
- Updated the crawler's location to `app/proprietary/web_crawler/connector.py` under a non-Apache-2 license boundary.
- Refactored the `WebCrawlerConnector` to eliminate the Firecrawl API key dependency, simplifying the interface for crawling URLs.
- Adjusted related components to accommodate the new structure and ensure successful crawl outcomes are properly handled.
- Updated documentation to reflect these changes and the new implementation status.

surfsense_backend/app/utils/validators.py

@@ -469,14 +469,6 @@ def validate_connector_config(
         if not isinstance(value, list) or not value:
             raise ValueError(f"{field_name} must be a non-empty list of strings")
 
-    def validate_firecrawl_api_key_format() -> None:
-        """Validate Firecrawl API key format if provided."""
-        api_key = config.get("FIRECRAWL_API_KEY", "")
-        if api_key and api_key.strip() and not api_key.strip().startswith("fc-"):
-            raise ValueError(
-                "Firecrawl API key should start with 'fc-'. Please verify your API key."
-            )
-
     def validate_initial_urls() -> None:
         initial_urls = config.get("INITIAL_URLS", "")
         if initial_urls and initial_urls.strip():
@@ -571,10 +563,9 @@ def validate_connector_config(
         # },
         "LUMA_CONNECTOR": {"required": ["LUMA_API_KEY"], "validators": {}},
         "WEBCRAWLER_CONNECTOR": {
-            "required": [],  # No required fields - API key is optional
-            "optional": ["FIRECRAWL_API_KEY", "INITIAL_URLS"],
+            "required": [],  # No required fields - URLs are optional
+            "optional": ["INITIAL_URLS"],
             "validators": {
-                "FIRECRAWL_API_KEY": lambda: validate_firecrawl_api_key_format(),
                 "INITIAL_URLS": lambda: validate_initial_urls(),
             },
         },