mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-06 22:12:12 +02:00
feat(03a): implement Scrapling-only web crawler and remove Firecrawl integration
- Standardized the web crawler to use Scrapling exclusively, removing Firecrawl entirely. - Updated the crawler's location to `app/proprietary/web_crawler/connector.py` under a non-Apache-2 license boundary. - Refactored the `WebCrawlerConnector` to eliminate the Firecrawl API key dependency, simplifying the interface for crawling URLs. - Adjusted related components to accommodate the new structure and ensure successful crawl outcomes are properly handled. - Updated documentation to reflect these changes and the new implementation status.
This commit is contained in:
DESKTOP-RTLN3BA\$punk
parent
34ab23f5d8
commit
5c36cd3071
32 changed files with 506 additions and 257 deletions
|
|
@ -109,7 +109,7 @@ RUN --mount=type=secret,id=HF_TOKEN \
|
|||
HF_TOKEN="$(cat /run/secrets/HF_TOKEN 2>/dev/null || true)" \
|
||||
python -c "from chonkie import AutoEmbeddings; AutoEmbeddings.get_embeddings('${EMBEDDING_MODEL}')"
|
||||
|
||||
# Install Scrapling's browser engines (patchright Chromium + Camoufox).
|
||||
# Install Scrapling's browser engines (patchright Chromium).
|
||||
# Scrapling pulls playwright/patchright via the `fetchers` extra; `scrapling install`
|
||||
# downloads the matching browser binaries used by DynamicFetcher/StealthyFetcher.
|
||||
RUN scrapling install
|
||||
|
|
|
|||
|
|
@ -68,7 +68,6 @@ async def create_multi_agent_chat_deep_agent(
|
|||
enabled_tools: list[str] | None = None,
|
||||
disabled_tools: list[str] | None = None,
|
||||
additional_tools: Sequence[BaseTool] | None = None,
|
||||
firecrawl_api_key: str | None = None,
|
||||
thread_visibility: ChatVisibility | None = None,
|
||||
mentioned_document_ids: list[int] | None = None,
|
||||
anon_session_id: str | None = None,
|
||||
|
|
@ -139,7 +138,6 @@ async def create_multi_agent_chat_deep_agent(
|
|||
"workspace_id": workspace_id,
|
||||
"db_session": db_session,
|
||||
"connector_service": connector_service,
|
||||
"firecrawl_api_key": firecrawl_api_key,
|
||||
"user_id": user_id,
|
||||
"auth_context": auth_context,
|
||||
"thread_id": thread_id,
|
||||
|
|
|
|||
|
|
@ -31,8 +31,8 @@ from .update_memory import (
|
|||
)
|
||||
|
||||
|
||||
def _build_scrape_webpage_tool(deps: dict[str, Any]) -> BaseTool:
|
||||
return create_scrape_webpage_tool(firecrawl_api_key=deps.get("firecrawl_api_key"))
|
||||
def _build_scrape_webpage_tool(_deps: dict[str, Any]) -> BaseTool:
|
||||
return create_scrape_webpage_tool()
|
||||
|
||||
|
||||
def _build_web_search_tool(deps: dict[str, Any]) -> BaseTool:
|
||||
|
|
|
|||
|
|
@ -18,7 +18,10 @@ from requests import Session
|
|||
from scrapling.fetchers import AsyncFetcher
|
||||
from youtube_transcript_api import YouTubeTranscriptApi
|
||||
|
||||
from app.connectors.webcrawler_connector import WebCrawlerConnector
|
||||
from app.proprietary.web_crawler import (
|
||||
CrawlOutcomeStatus,
|
||||
WebCrawlerConnector,
|
||||
)
|
||||
from app.tasks.document_processors.youtube_processor import get_youtube_video_id
|
||||
from app.utils.proxy import get_proxy_url, get_requests_proxies
|
||||
|
||||
|
|
@ -167,14 +170,10 @@ async def _scrape_youtube_video(
|
|||
}
|
||||
|
||||
|
||||
def create_scrape_webpage_tool(firecrawl_api_key: str | None = None):
|
||||
def create_scrape_webpage_tool():
|
||||
"""
|
||||
Factory function to create the scrape_webpage tool.
|
||||
|
||||
Args:
|
||||
firecrawl_api_key: Optional Firecrawl API key for premium web scraping.
|
||||
Falls back to Chromium/Trafilatura if not provided.
|
||||
|
||||
Returns:
|
||||
A configured tool function for scraping webpages.
|
||||
"""
|
||||
|
|
@ -229,10 +228,10 @@ def create_scrape_webpage_tool(firecrawl_api_key: str | None = None):
|
|||
if video_id:
|
||||
return await _scrape_youtube_video(url, video_id, max_length)
|
||||
|
||||
connector = WebCrawlerConnector(firecrawl_api_key=firecrawl_api_key)
|
||||
result, error = await connector.crawl_url(url, formats=["markdown"])
|
||||
connector = WebCrawlerConnector()
|
||||
outcome = await connector.crawl_url(url)
|
||||
|
||||
if error:
|
||||
if outcome.status is not CrawlOutcomeStatus.SUCCESS or not outcome.result:
|
||||
return {
|
||||
"id": scrape_id,
|
||||
"assetId": url,
|
||||
|
|
@ -240,20 +239,10 @@ def create_scrape_webpage_tool(firecrawl_api_key: str | None = None):
|
|||
"href": url,
|
||||
"title": domain or "Webpage",
|
||||
"domain": domain,
|
||||
"error": error,
|
||||
}
|
||||
|
||||
if not result:
|
||||
return {
|
||||
"id": scrape_id,
|
||||
"assetId": url,
|
||||
"kind": "article",
|
||||
"href": url,
|
||||
"title": domain or "Webpage",
|
||||
"domain": domain,
|
||||
"error": "No content returned from crawler",
|
||||
"error": outcome.error or "No content returned from crawler",
|
||||
}
|
||||
|
||||
result = outcome.result
|
||||
content = result.get("content", "")
|
||||
metadata = result.get("metadata", {})
|
||||
|
||||
|
|
|
|||
|
|
@ -25,5 +25,5 @@ def load_tools(
|
|||
workspace_id=d.get("workspace_id"),
|
||||
available_connectors=d.get("available_connectors"),
|
||||
),
|
||||
create_scrape_webpage_tool(firecrawl_api_key=d.get("firecrawl_api_key")),
|
||||
create_scrape_webpage_tool(),
|
||||
]
|
||||
|
|
|
|||
|
|
@ -12,7 +12,10 @@ from requests import Session
|
|||
from scrapling.fetchers import AsyncFetcher
|
||||
from youtube_transcript_api import YouTubeTranscriptApi
|
||||
|
||||
from app.connectors.webcrawler_connector import WebCrawlerConnector
|
||||
from app.proprietary.web_crawler import (
|
||||
CrawlOutcomeStatus,
|
||||
WebCrawlerConnector,
|
||||
)
|
||||
from app.tasks.document_processors.youtube_processor import get_youtube_video_id
|
||||
from app.utils.proxy import get_proxy_url, get_requests_proxies
|
||||
|
||||
|
|
@ -161,14 +164,10 @@ async def _scrape_youtube_video(
|
|||
}
|
||||
|
||||
|
||||
def create_scrape_webpage_tool(firecrawl_api_key: str | None = None):
|
||||
def create_scrape_webpage_tool():
|
||||
"""
|
||||
Factory function to create the scrape_webpage tool.
|
||||
|
||||
Args:
|
||||
firecrawl_api_key: Optional Firecrawl API key for premium web scraping.
|
||||
Falls back to Chromium/Trafilatura if not provided.
|
||||
|
||||
Returns:
|
||||
A configured tool function for scraping webpages.
|
||||
"""
|
||||
|
|
@ -223,10 +222,10 @@ def create_scrape_webpage_tool(firecrawl_api_key: str | None = None):
|
|||
if video_id:
|
||||
return await _scrape_youtube_video(url, video_id, max_length)
|
||||
|
||||
connector = WebCrawlerConnector(firecrawl_api_key=firecrawl_api_key)
|
||||
result, error = await connector.crawl_url(url, formats=["markdown"])
|
||||
connector = WebCrawlerConnector()
|
||||
outcome = await connector.crawl_url(url)
|
||||
|
||||
if error:
|
||||
if outcome.status is not CrawlOutcomeStatus.SUCCESS or not outcome.result:
|
||||
return {
|
||||
"id": scrape_id,
|
||||
"assetId": url,
|
||||
|
|
@ -234,20 +233,10 @@ def create_scrape_webpage_tool(firecrawl_api_key: str | None = None):
|
|||
"href": url,
|
||||
"title": domain or "Webpage",
|
||||
"domain": domain,
|
||||
"error": error,
|
||||
}
|
||||
|
||||
if not result:
|
||||
return {
|
||||
"id": scrape_id,
|
||||
"assetId": url,
|
||||
"kind": "article",
|
||||
"href": url,
|
||||
"title": domain or "Webpage",
|
||||
"domain": domain,
|
||||
"error": "No content returned from crawler",
|
||||
"error": outcome.error or "No content returned from crawler",
|
||||
}
|
||||
|
||||
result = outcome.result
|
||||
content = result.get("content", "")
|
||||
metadata = result.get("metadata", {})
|
||||
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ from app.automations.services.model_policy import (
|
|||
from app.db import Workspace
|
||||
from app.tasks.chat.streaming.flows.shared.llm_bundle import load_llm_bundle
|
||||
from app.tasks.chat.streaming.flows.shared.pre_stream_setup import (
|
||||
setup_connector_and_firecrawl,
|
||||
setup_connector_service,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -31,7 +31,6 @@ class AgentDependencies:
|
|||
llm: Any
|
||||
agent_config: Any
|
||||
connector_service: Any
|
||||
firecrawl_api_key: str | None
|
||||
checkpointer: Any
|
||||
|
||||
|
||||
|
|
@ -82,7 +81,7 @@ async def build_dependencies(
|
|||
if err is not None or llm is None:
|
||||
raise DependencyError(err or "failed to load chat model config")
|
||||
|
||||
connector_service, firecrawl_api_key = await setup_connector_and_firecrawl(
|
||||
connector_service = await setup_connector_service(
|
||||
session, workspace_id=workspace_id
|
||||
)
|
||||
# Per-task InMemorySaver: the shared Postgres checkpointer's connection
|
||||
|
|
@ -100,6 +99,5 @@ async def build_dependencies(
|
|||
llm=llm,
|
||||
agent_config=agent_config,
|
||||
connector_service=connector_service,
|
||||
firecrawl_api_key=firecrawl_api_key,
|
||||
checkpointer=checkpointer,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -171,7 +171,6 @@ async def run_agent_task(
|
|||
user_id=user_id,
|
||||
thread_id=None,
|
||||
agent_config=deps.agent_config,
|
||||
firecrawl_api_key=deps.firecrawl_api_key,
|
||||
thread_visibility=ChatVisibility.PRIVATE,
|
||||
mentioned_document_ids=mentioned_document_ids,
|
||||
image_gen_model_id=ctx.image_gen_model_id,
|
||||
|
|
|
|||
11
surfsense_backend/app/proprietary/LICENSE
Normal file
11
surfsense_backend/app/proprietary/LICENSE
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
SurfSense Proprietary / Source-Available License (PLACEHOLDER)
|
||||
|
||||
All files in this directory tree (app/proprietary/**) are NOT covered by the
|
||||
Apache License 2.0 that governs the rest of this repository.
|
||||
|
||||
The final license text is TBD. Until it is finalized, ALL RIGHTS ARE RESERVED:
|
||||
the code in this directory may not be copied, modified, redistributed, sublicensed,
|
||||
or relicensed (including under Apache-2.0) without the explicit prior written
|
||||
permission of the copyright holder.
|
||||
|
||||
Copyright (c) 2026 SurfSense. All rights reserved.
|
||||
32
surfsense_backend/app/proprietary/README.md
Normal file
32
surfsense_backend/app/proprietary/README.md
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
# `app.proprietary` — non-Apache-2 license boundary
|
||||
|
||||
Everything in this directory tree is licensed **separately** from the rest of
|
||||
SurfSense (which is Apache-2.0). See [`LICENSE`](./LICENSE).
|
||||
|
||||
## Why this exists
|
||||
|
||||
This package holds the product moat:
|
||||
|
||||
- the in-house **undetectable web crawler** (Scrapling tiers + stealth/captcha
|
||||
hardening), and
|
||||
- (future) **platform-specific actors** that scrape/extract structured data from
|
||||
individual platforms.
|
||||
|
||||
Keeping it in one clearly-named directory makes the license boundary
|
||||
unambiguous: a single rule — *everything under `app/proprietary/**` is not
|
||||
Apache-2.0* — instead of per-file headers scattered across the tree.
|
||||
|
||||
## Layout
|
||||
|
||||
- `web_crawler/` — the Scrapling-based crawler engine. Public API:
|
||||
`WebCrawlerConnector`, `CrawlOutcome`, `CrawlOutcomeStatus`
|
||||
(`from app.proprietary.web_crawler import ...`).
|
||||
- `platforms/` — (future, Phase 8) platform-specific actors; scaffolded/empty.
|
||||
|
||||
## Rules
|
||||
|
||||
- **Do not** add Apache-2.0-intended code here.
|
||||
- Apache-2.0 code elsewhere **may import from** this package (the indexer and the
|
||||
chat `scrape_webpage` tools do); that does not move them under this license.
|
||||
- Depend only on the public API exported from each subpackage's `__init__`, not
|
||||
on internal modules, so the boundary stays clean and swappable.
|
||||
10
surfsense_backend/app/proprietary/__init__.py
Normal file
10
surfsense_backend/app/proprietary/__init__.py
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
"""SurfSense proprietary packages (non-Apache-2 license boundary).
|
||||
|
||||
Everything under ``app.proprietary`` is licensed **separately** from the
|
||||
Apache-2.0 project root — see ``app/proprietary/LICENSE``. This package holds
|
||||
the in-house undetectable crawler engine and (future) platform-specific actors
|
||||
that form the product's moat.
|
||||
|
||||
Apache-2.0 code elsewhere in the app may *import from* this package, but code
|
||||
placed *inside* it is not covered by the repository's Apache-2.0 license.
|
||||
"""
|
||||
7
surfsense_backend/app/proprietary/platforms/__init__.py
Normal file
7
surfsense_backend/app/proprietary/platforms/__init__.py
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
"""Platform-specific crawl/extraction actors (non-Apache-2; see ../LICENSE).
|
||||
|
||||
Scaffolded for future work (Phase 8 — Platform Actors). Each platform (e.g.
|
||||
maps, professional networks) will get its own subpackage that reuses the shared
|
||||
fetch strategies in ``app.proprietary.web_crawler`` under a platform-specific
|
||||
structured extractor. Empty for now by design.
|
||||
"""
|
||||
13
surfsense_backend/app/proprietary/web_crawler/__init__.py
Normal file
13
surfsense_backend/app/proprietary/web_crawler/__init__.py
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
"""Proprietary web crawler engine (non-Apache-2; see ``app/proprietary/LICENSE``).
|
||||
|
||||
Public API for the single-framework (Scrapling) undetectable crawler. Callers
|
||||
depend only on these symbols, never on internal tier/strategy details.
|
||||
"""
|
||||
|
||||
from app.proprietary.web_crawler.connector import (
|
||||
CrawlOutcome,
|
||||
CrawlOutcomeStatus,
|
||||
WebCrawlerConnector,
|
||||
)
|
||||
|
||||
__all__ = ["CrawlOutcome", "CrawlOutcomeStatus", "WebCrawlerConnector"]
|
||||
|
|
@ -1,25 +1,34 @@
|
|||
# SurfSense proprietary crawler engine.
|
||||
#
|
||||
# This module is part of the ``app.proprietary`` package and is licensed
|
||||
# SEPARATELY from the Apache-2.0 project root. See ``app/proprietary/LICENSE``.
|
||||
# Do not relicense or redistribute this file under Apache-2.0.
|
||||
"""
|
||||
WebCrawler Connector Module
|
||||
|
||||
A module for crawling web pages and extracting content using Firecrawl or
|
||||
Scrapling's tiered fetchers, with Trafilatura for HTML -> markdown extraction.
|
||||
Provides a unified interface for web scraping.
|
||||
A single-framework (Scrapling) web crawler with Trafilatura for HTML -> markdown
|
||||
extraction. Provides a unified interface for web scraping.
|
||||
|
||||
Fallback order:
|
||||
1. Firecrawl (if API key is configured)
|
||||
2. Scrapling AsyncFetcher (fast static HTTP, no browser subprocess)
|
||||
3. Scrapling DynamicFetcher (full browser, run in a thread)
|
||||
4. Scrapling StealthyFetcher (anti-bot stealth browser, run in a thread)
|
||||
Fallback ladder (the ``FetchStrategy`` seam — see ``plans/backend/03a-crawler-core.md``):
|
||||
1. Scrapling AsyncFetcher (fast static HTTP, TLS-impersonated, no subprocess)
|
||||
2. Scrapling DynamicFetcher (full browser, run in a thread)
|
||||
3. Scrapling StealthyFetcher (patchright-Chromium anti-bot + Cloudflare solving,
|
||||
run in a thread)
|
||||
|
||||
Every tier returns extracted content via the same ``CrawlOutcome`` contract, so
|
||||
callers (indexer, chat tool, crawl billing) depend only on the outcome, never on
|
||||
which tier produced it.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from typing import Any
|
||||
|
||||
import trafilatura
|
||||
import validators
|
||||
from firecrawl import AsyncFirecrawlApp
|
||||
from scrapling.fetchers import AsyncFetcher, DynamicFetcher, StealthyFetcher
|
||||
|
||||
from app.utils.proxy import get_proxy_url
|
||||
|
|
@ -30,76 +39,68 @@ logger = logging.getLogger(__name__)
|
|||
_PERF = "[webcrawler][perf]"
|
||||
|
||||
|
||||
class CrawlOutcomeStatus(str, Enum):
|
||||
"""Deterministic per-URL crawl result, single-sourcing the billable signal."""
|
||||
|
||||
SUCCESS = "success" # a tier returned usable extracted content
|
||||
EMPTY = "empty" # fetched, but no usable content after ALL tiers
|
||||
FAILED = "failed" # invalid URL or every tier errored / was unavailable
|
||||
|
||||
|
||||
@dataclass
|
||||
class CrawlOutcome:
|
||||
"""Explicit ``crawl_url`` result shared by every caller.
|
||||
|
||||
The **billable success predicate is single-sourced**:
|
||||
``status == CrawlOutcomeStatus.SUCCESS`` (Phase 3c meters on it). Picking a
|
||||
dataclass over a tuple lets later subplans append fields without breaking
|
||||
callers (03d adds ``captcha_attempts``/``captcha_solved`` for per-attempt
|
||||
billing; 03e's block classifier can attach a ``block_type``).
|
||||
"""
|
||||
|
||||
status: CrawlOutcomeStatus
|
||||
result: dict[str, Any] | None = None
|
||||
error: str | None = None
|
||||
tier: str | None = None
|
||||
|
||||
|
||||
class WebCrawlerConnector:
|
||||
"""Class for crawling web pages and extracting content."""
|
||||
|
||||
def __init__(self, firecrawl_api_key: str | None = None):
|
||||
"""
|
||||
Initialize the WebCrawlerConnector class.
|
||||
|
||||
Args:
|
||||
firecrawl_api_key: Firecrawl API key (optional). If provided, Firecrawl will be tried first
|
||||
and Scrapling will be used as fallback if Firecrawl fails. If not provided,
|
||||
Scrapling fetchers are used directly.
|
||||
"""
|
||||
self.firecrawl_api_key = firecrawl_api_key
|
||||
self.use_firecrawl = bool(firecrawl_api_key)
|
||||
|
||||
def set_api_key(self, api_key: str) -> None:
|
||||
"""
|
||||
Set the Firecrawl API key and enable Firecrawl usage.
|
||||
|
||||
Args:
|
||||
api_key: Firecrawl API key
|
||||
"""
|
||||
self.firecrawl_api_key = api_key
|
||||
self.use_firecrawl = True
|
||||
|
||||
async def crawl_url(
|
||||
self, url: str, formats: list[str] | None = None
|
||||
) -> tuple[dict[str, Any] | None, str | None]:
|
||||
async def crawl_url(self, url: str) -> CrawlOutcome:
|
||||
"""
|
||||
Crawl a single URL and extract its content.
|
||||
|
||||
Fallback order:
|
||||
1. Firecrawl (if API key configured)
|
||||
2. Scrapling AsyncFetcher (fast static HTTP, no subprocess)
|
||||
3. Scrapling DynamicFetcher (full browser, run in a thread)
|
||||
4. Scrapling StealthyFetcher (anti-bot stealth browser, run in a thread)
|
||||
Fallback ladder:
|
||||
1. Scrapling AsyncFetcher (fast static HTTP, TLS-impersonated)
|
||||
2. Scrapling DynamicFetcher (full browser, run in a thread)
|
||||
3. Scrapling StealthyFetcher (anti-bot stealth browser + Cloudflare
|
||||
solving, run in a thread)
|
||||
|
||||
Args:
|
||||
url: URL to crawl
|
||||
formats: List of formats to extract (e.g., ["markdown", "html"]) - only for Firecrawl
|
||||
|
||||
Returns:
|
||||
Tuple containing (crawl result dict, error message or None)
|
||||
Result dict contains:
|
||||
- content: Extracted content (markdown or HTML)
|
||||
A ``CrawlOutcome``. On ``SUCCESS``, ``result`` is a dict containing:
|
||||
- content: Extracted content (markdown)
|
||||
- metadata: Page metadata (title, description, etc.)
|
||||
- source: Original URL
|
||||
- crawler_type: Type of crawler used
|
||||
- crawler_type: Identifier of the tier that produced the content
|
||||
"""
|
||||
total_start = time.perf_counter()
|
||||
try:
|
||||
if not validators.url(url):
|
||||
return None, f"Invalid URL: {url}"
|
||||
return CrawlOutcome(
|
||||
status=CrawlOutcomeStatus.FAILED,
|
||||
error=f"Invalid URL: {url}",
|
||||
)
|
||||
|
||||
errors: list[str] = []
|
||||
# True once any tier fetched the page but extraction yielded nothing
|
||||
# (distinguishes EMPTY from FAILED, where every tier raised/was
|
||||
# unavailable).
|
||||
reached_without_content = False
|
||||
|
||||
# --- 1. Firecrawl (premium, if configured) ---
|
||||
if self.use_firecrawl:
|
||||
tier_start = time.perf_counter()
|
||||
try:
|
||||
logger.info(f"[webcrawler] Using Firecrawl for: {url}")
|
||||
result = await self._crawl_with_firecrawl(url, formats)
|
||||
self._log_tier_outcome("firecrawl", url, tier_start, "success")
|
||||
self._log_total(url, "firecrawl", total_start)
|
||||
return result, None
|
||||
except Exception as exc:
|
||||
errors.append(f"Firecrawl: {exc!s}")
|
||||
self._log_tier_outcome("firecrawl", url, tier_start, "error", exc)
|
||||
|
||||
# --- 2. Scrapling AsyncFetcher (fast static HTTP) ---
|
||||
# --- 1. Scrapling AsyncFetcher (fast static HTTP) ---
|
||||
tier_start = time.perf_counter()
|
||||
try:
|
||||
logger.info(f"[webcrawler] Using Scrapling AsyncFetcher for: {url}")
|
||||
|
|
@ -109,7 +110,12 @@ class WebCrawlerConnector:
|
|||
"scrapling-static", url, tier_start, "success"
|
||||
)
|
||||
self._log_total(url, "scrapling-static", total_start)
|
||||
return result, None
|
||||
return CrawlOutcome(
|
||||
status=CrawlOutcomeStatus.SUCCESS,
|
||||
result=result,
|
||||
tier="scrapling-static",
|
||||
)
|
||||
reached_without_content = True
|
||||
errors.append("Scrapling static: empty extraction")
|
||||
self._log_tier_outcome("scrapling-static", url, tier_start, "empty")
|
||||
except Exception as exc:
|
||||
|
|
@ -118,7 +124,7 @@ class WebCrawlerConnector:
|
|||
"scrapling-static", url, tier_start, "error", exc
|
||||
)
|
||||
|
||||
# --- 3. Scrapling DynamicFetcher (full browser) ---
|
||||
# --- 2. Scrapling DynamicFetcher (full browser) ---
|
||||
tier_start = time.perf_counter()
|
||||
try:
|
||||
logger.info(f"[webcrawler] Using Scrapling DynamicFetcher for: {url}")
|
||||
|
|
@ -128,7 +134,12 @@ class WebCrawlerConnector:
|
|||
"scrapling-dynamic", url, tier_start, "success"
|
||||
)
|
||||
self._log_total(url, "scrapling-dynamic", total_start)
|
||||
return result, None
|
||||
return CrawlOutcome(
|
||||
status=CrawlOutcomeStatus.SUCCESS,
|
||||
result=result,
|
||||
tier="scrapling-dynamic",
|
||||
)
|
||||
reached_without_content = True
|
||||
errors.append("Scrapling dynamic: empty extraction")
|
||||
self._log_tier_outcome("scrapling-dynamic", url, tier_start, "empty")
|
||||
except NotImplementedError:
|
||||
|
|
@ -145,7 +156,7 @@ class WebCrawlerConnector:
|
|||
"scrapling-dynamic", url, tier_start, "error", exc
|
||||
)
|
||||
|
||||
# --- 4. Scrapling StealthyFetcher (anti-bot, last resort) ---
|
||||
# --- 3. Scrapling StealthyFetcher (anti-bot, last resort) ---
|
||||
tier_start = time.perf_counter()
|
||||
try:
|
||||
logger.info(f"[webcrawler] Using Scrapling StealthyFetcher for: {url}")
|
||||
|
|
@ -155,7 +166,12 @@ class WebCrawlerConnector:
|
|||
"scrapling-stealthy", url, tier_start, "success"
|
||||
)
|
||||
self._log_total(url, "scrapling-stealthy", total_start)
|
||||
return result, None
|
||||
return CrawlOutcome(
|
||||
status=CrawlOutcomeStatus.SUCCESS,
|
||||
result=result,
|
||||
tier="scrapling-stealthy",
|
||||
)
|
||||
reached_without_content = True
|
||||
errors.append("Scrapling stealthy: empty extraction")
|
||||
self._log_tier_outcome("scrapling-stealthy", url, tier_start, "empty")
|
||||
except NotImplementedError:
|
||||
|
|
@ -173,11 +189,22 @@ class WebCrawlerConnector:
|
|||
)
|
||||
|
||||
self._log_total(url, "none", total_start)
|
||||
return None, f"All crawl methods failed for {url}. {'; '.join(errors)}"
|
||||
if reached_without_content:
|
||||
return CrawlOutcome(
|
||||
status=CrawlOutcomeStatus.EMPTY,
|
||||
error=f"No content extracted for {url}. {'; '.join(errors)}",
|
||||
)
|
||||
return CrawlOutcome(
|
||||
status=CrawlOutcomeStatus.FAILED,
|
||||
error=f"All crawl methods failed for {url}. {'; '.join(errors)}",
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
self._log_total(url, "error", total_start)
|
||||
return None, f"Error crawling URL {url}: {e!s}"
|
||||
return CrawlOutcome(
|
||||
status=CrawlOutcomeStatus.FAILED,
|
||||
error=f"Error crawling URL {url}: {e!s}",
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _log_tier_outcome(
|
||||
|
|
@ -220,57 +247,6 @@ class WebCrawlerConnector:
|
|||
total_ms,
|
||||
)
|
||||
|
||||
async def _crawl_with_firecrawl(
|
||||
self, url: str, formats: list[str] | None = None
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Crawl URL using Firecrawl.
|
||||
|
||||
Args:
|
||||
url: URL to crawl
|
||||
formats: List of formats to extract
|
||||
|
||||
Returns:
|
||||
Dict containing crawled content and metadata
|
||||
|
||||
Raises:
|
||||
ValueError: If Firecrawl scraping fails
|
||||
"""
|
||||
if not self.firecrawl_api_key:
|
||||
raise ValueError("Firecrawl API key not set. Call set_api_key() first.")
|
||||
|
||||
firecrawl_app = AsyncFirecrawlApp(api_key=self.firecrawl_api_key)
|
||||
|
||||
# Default to markdown format
|
||||
if formats is None:
|
||||
formats = ["markdown"]
|
||||
|
||||
# v2 API returns Document directly and raises an exception on failure
|
||||
scrape_result = await firecrawl_app.scrape(url, formats=formats)
|
||||
|
||||
if not scrape_result:
|
||||
raise ValueError("Firecrawl returned no result")
|
||||
|
||||
# Extract content based on format
|
||||
content = scrape_result.markdown or scrape_result.html or ""
|
||||
|
||||
# Extract metadata - v2 returns DocumentMetadata object
|
||||
metadata_obj = scrape_result.metadata
|
||||
metadata = metadata_obj.model_dump() if metadata_obj else {}
|
||||
|
||||
return {
|
||||
"content": content,
|
||||
"metadata": {
|
||||
"source": url,
|
||||
"title": metadata.get("title", url),
|
||||
"description": metadata.get("description", ""),
|
||||
"language": metadata.get("language", ""),
|
||||
"sourceURL": metadata.get("source_url", url),
|
||||
**metadata,
|
||||
},
|
||||
"crawler_type": "firecrawl",
|
||||
}
|
||||
|
||||
async def _crawl_with_async_fetcher(self, url: str) -> dict[str, Any] | None:
|
||||
"""
|
||||
Crawl URL using Scrapling's AsyncFetcher (static HTTP) + Trafilatura.
|
||||
|
|
@ -281,9 +257,13 @@ class WebCrawlerConnector:
|
|||
rendered SPAs) so the caller can fall through to the browser tiers.
|
||||
"""
|
||||
fetch_start = time.perf_counter()
|
||||
# ``impersonate="chrome"`` makes curl_cffi present a real Chrome TLS
|
||||
# ClientHello (JA3/JA4) instead of its default fingerprint, keeping the
|
||||
# static tier coherent with the browser tiers' UA (see 03e §2b).
|
||||
page = await AsyncFetcher.get(
|
||||
url,
|
||||
stealthy_headers=True,
|
||||
impersonate="chrome",
|
||||
proxy=get_proxy_url(),
|
||||
timeout=20,
|
||||
)
|
||||
|
|
@ -340,7 +320,7 @@ class WebCrawlerConnector:
|
|||
|
||||
async def _crawl_with_stealthy(self, url: str) -> dict[str, Any] | None:
|
||||
"""
|
||||
Crawl URL using Scrapling's StealthyFetcher (Camoufox) + Trafilatura.
|
||||
Crawl URL using Scrapling's StealthyFetcher (patchright-Chromium) + Trafilatura.
|
||||
|
||||
Last-resort tier with anti-bot features. Runs the sync fetch in a worker
|
||||
thread for the same event-loop-safety reasons as DynamicFetcher. Falls
|
||||
|
|
@ -351,11 +331,14 @@ class WebCrawlerConnector:
|
|||
def _crawl_with_stealthy_sync(self, url: str) -> dict[str, Any] | None:
|
||||
"""Synchronous StealthyFetcher crawl executed in a worker thread."""
|
||||
fetch_start = time.perf_counter()
|
||||
# ``solve_cloudflare=True`` runs the full Turnstile/Interstitial challenge
|
||||
# loop; scoped to this last-resort tier only (it spins up the browser).
|
||||
page = StealthyFetcher.fetch(
|
||||
url,
|
||||
headless=True,
|
||||
network_idle=True,
|
||||
block_ads=True,
|
||||
solve_cloudflare=True,
|
||||
proxy=get_proxy_url(),
|
||||
)
|
||||
fetch_ms = (time.perf_counter() - fetch_start) * 1000
|
||||
|
|
@ -29,7 +29,6 @@ async def build_main_agent_for_thread(
|
|||
user_id: str | None,
|
||||
thread_id: int | None,
|
||||
agent_config: AgentConfig | None,
|
||||
firecrawl_api_key: str | None,
|
||||
thread_visibility: ChatVisibility | None,
|
||||
filesystem_selection: FilesystemSelection | None,
|
||||
disabled_tools: list[str] | None = None,
|
||||
|
|
@ -45,7 +44,6 @@ async def build_main_agent_for_thread(
|
|||
user_id=user_id,
|
||||
thread_id=thread_id,
|
||||
agent_config=agent_config,
|
||||
firecrawl_api_key=firecrawl_api_key,
|
||||
thread_visibility=thread_visibility,
|
||||
filesystem_selection=filesystem_selection,
|
||||
disabled_tools=disabled_tools,
|
||||
|
|
|
|||
|
|
@ -83,7 +83,7 @@ from app.tasks.chat.streaming.flows.shared.first_frames import (
|
|||
from app.tasks.chat.streaming.flows.shared.llm_bundle import load_llm_bundle
|
||||
from app.tasks.chat.streaming.flows.shared.pre_stream_setup import (
|
||||
get_chat_checkpointer,
|
||||
setup_connector_and_firecrawl,
|
||||
setup_connector_service,
|
||||
)
|
||||
from app.tasks.chat.streaming.flows.shared.premium_quota import (
|
||||
CreditReservation,
|
||||
|
|
@ -376,11 +376,11 @@ async def stream_new_chat(
|
|||
)
|
||||
|
||||
_t0 = time.perf_counter()
|
||||
connector_service, firecrawl_api_key = await setup_connector_and_firecrawl(
|
||||
connector_service = await setup_connector_service(
|
||||
session, workspace_id=workspace_id
|
||||
)
|
||||
_perf_log.info(
|
||||
"[stream_new_chat] Connector service + firecrawl key in %.3fs",
|
||||
"[stream_new_chat] Connector service in %.3fs",
|
||||
time.perf_counter() - _t0,
|
||||
)
|
||||
|
||||
|
|
@ -410,7 +410,6 @@ async def stream_new_chat(
|
|||
user_id=user_id,
|
||||
thread_id=chat_id,
|
||||
agent_config=agent_config,
|
||||
firecrawl_api_key=firecrawl_api_key,
|
||||
thread_visibility=visibility,
|
||||
filesystem_selection=filesystem_selection,
|
||||
disabled_tools=disabled_tools,
|
||||
|
|
@ -665,7 +664,6 @@ async def stream_new_chat(
|
|||
user_id=user_id,
|
||||
thread_id=chat_id,
|
||||
agent_config=agent_config,
|
||||
firecrawl_api_key=firecrawl_api_key,
|
||||
thread_visibility=visibility,
|
||||
filesystem_selection=filesystem_selection,
|
||||
disabled_tools=disabled_tools,
|
||||
|
|
|
|||
|
|
@ -62,7 +62,7 @@ from app.tasks.chat.streaming.flows.shared.first_frames import (
|
|||
from app.tasks.chat.streaming.flows.shared.llm_bundle import load_llm_bundle
|
||||
from app.tasks.chat.streaming.flows.shared.pre_stream_setup import (
|
||||
get_chat_checkpointer,
|
||||
setup_connector_and_firecrawl,
|
||||
setup_connector_service,
|
||||
)
|
||||
from app.tasks.chat.streaming.flows.shared.premium_quota import (
|
||||
CreditReservation,
|
||||
|
|
@ -314,11 +314,11 @@ async def stream_resume_chat(
|
|||
# --- Pre-stream setup ---
|
||||
|
||||
_t0 = time.perf_counter()
|
||||
connector_service, firecrawl_api_key = await setup_connector_and_firecrawl(
|
||||
connector_service = await setup_connector_service(
|
||||
session, workspace_id=workspace_id
|
||||
)
|
||||
_perf_log.info(
|
||||
"[stream_resume] Connector service + firecrawl key in %.3fs",
|
||||
"[stream_resume] Connector service in %.3fs",
|
||||
time.perf_counter() - _t0,
|
||||
)
|
||||
|
||||
|
|
@ -344,7 +344,6 @@ async def stream_resume_chat(
|
|||
user_id=user_id,
|
||||
thread_id=chat_id,
|
||||
agent_config=agent_config,
|
||||
firecrawl_api_key=firecrawl_api_key,
|
||||
thread_visibility=visibility,
|
||||
filesystem_selection=filesystem_selection,
|
||||
disabled_tools=disabled_tools,
|
||||
|
|
@ -480,7 +479,6 @@ async def stream_resume_chat(
|
|||
user_id=user_id,
|
||||
thread_id=chat_id,
|
||||
agent_config=agent_config,
|
||||
firecrawl_api_key=firecrawl_api_key,
|
||||
thread_visibility=visibility,
|
||||
filesystem_selection=filesystem_selection,
|
||||
disabled_tools=disabled_tools,
|
||||
|
|
|
|||
|
|
@ -1,33 +1,20 @@
|
|||
"""Pre-stream setup: connector service, firecrawl key, checkpointer."""
|
||||
"""Pre-stream setup: connector service, checkpointer."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.agents.chat.runtime.checkpointer import get_checkpointer
|
||||
from app.db import SearchSourceConnectorType
|
||||
from app.services.connector_service import ConnectorService
|
||||
|
||||
|
||||
async def setup_connector_and_firecrawl(
|
||||
async def setup_connector_service(
|
||||
session: AsyncSession,
|
||||
*,
|
||||
workspace_id: int,
|
||||
) -> tuple[ConnectorService, str | None]:
|
||||
"""Build the per-turn connector service and pull the firecrawl API key.
|
||||
|
||||
Returns ``(connector_service, firecrawl_api_key)``. ``firecrawl_api_key`` is
|
||||
``None`` when no web-crawler connector is configured (the agent simply
|
||||
skips firecrawl-backed tools in that case).
|
||||
"""
|
||||
connector_service = ConnectorService(session, workspace_id=workspace_id)
|
||||
firecrawl_api_key: str | None = None
|
||||
webcrawler_connector = await connector_service.get_connector_by_type(
|
||||
SearchSourceConnectorType.WEBCRAWLER_CONNECTOR, workspace_id
|
||||
)
|
||||
if webcrawler_connector and webcrawler_connector.config:
|
||||
firecrawl_api_key = webcrawler_connector.config.get("FIRECRAWL_API_KEY")
|
||||
return connector_service, firecrawl_api_key
|
||||
) -> ConnectorService:
|
||||
"""Build the per-turn connector service for the workspace."""
|
||||
return ConnectorService(session, workspace_id=workspace_id)
|
||||
|
||||
|
||||
async def get_chat_checkpointer():
|
||||
|
|
|
|||
|
|
@ -13,7 +13,10 @@ from datetime import datetime
|
|||
from sqlalchemy.exc import SQLAlchemyError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.connectors.webcrawler_connector import WebCrawlerConnector
|
||||
from app.proprietary.web_crawler import (
|
||||
CrawlOutcomeStatus,
|
||||
WebCrawlerConnector,
|
||||
)
|
||||
from app.db import Document, DocumentStatus, DocumentType, SearchSourceConnectorType
|
||||
from app.services.task_logging_service import TaskLoggingService
|
||||
from app.utils.document_converters import (
|
||||
|
|
@ -111,9 +114,6 @@ async def index_crawled_urls(
|
|||
f"Connector with ID {connector_id} not found or is not a webcrawler connector",
|
||||
)
|
||||
|
||||
# Get the Firecrawl API key from the connector config (optional)
|
||||
api_key = connector.config.get("FIRECRAWL_API_KEY")
|
||||
|
||||
# Get URLs from connector config
|
||||
raw_initial_urls = connector.config.get("INITIAL_URLS")
|
||||
urls = parse_webcrawler_urls(raw_initial_urls)
|
||||
|
|
@ -132,11 +132,10 @@ async def index_crawled_urls(
|
|||
f"Initializing webcrawler client for connector {connector_id}",
|
||||
{
|
||||
"stage": "client_initialization",
|
||||
"use_firecrawl": bool(api_key),
|
||||
},
|
||||
)
|
||||
|
||||
crawler = WebCrawlerConnector(firecrawl_api_key=api_key)
|
||||
crawler = WebCrawlerConnector()
|
||||
|
||||
# Validate URLs
|
||||
if not urls:
|
||||
|
|
@ -169,6 +168,10 @@ async def index_crawled_urls(
|
|||
documents_skipped = 0
|
||||
documents_failed = 0
|
||||
duplicate_content_count = 0
|
||||
# Explicit "crawl succeeded" count: one per URL that yielded usable
|
||||
# extracted content, independent of downstream KB dedupe/unchanged
|
||||
# bucketing. This is the billable unit Phase 3c meters on.
|
||||
crawls_succeeded = 0
|
||||
|
||||
# Heartbeat tracking - update notification periodically to prevent appearing stuck
|
||||
last_heartbeat_time = time.time()
|
||||
|
|
@ -294,16 +297,27 @@ async def index_crawled_urls(
|
|||
)
|
||||
|
||||
# Crawl the URL
|
||||
crawl_result, error = await crawler.crawl_url(url)
|
||||
outcome = await crawler.crawl_url(url)
|
||||
|
||||
if error or not crawl_result:
|
||||
logger.warning(f"Failed to crawl URL {url}: {error}")
|
||||
document.status = DocumentStatus.failed(error or "Crawl failed")
|
||||
if (
|
||||
outcome.status is not CrawlOutcomeStatus.SUCCESS
|
||||
or not outcome.result
|
||||
):
|
||||
logger.warning(f"Failed to crawl URL {url}: {outcome.error}")
|
||||
document.status = DocumentStatus.failed(
|
||||
outcome.error or "Crawl failed"
|
||||
)
|
||||
document.updated_at = get_current_timestamp()
|
||||
await session.commit()
|
||||
documents_failed += 1
|
||||
continue
|
||||
|
||||
# A tier extracted usable content: count the successful crawl now,
|
||||
# before the dedupe/unchanged branches (those still represent a
|
||||
# successful crawl and must bill — see 03c).
|
||||
crawls_succeeded += 1
|
||||
crawl_result = outcome.result
|
||||
|
||||
# Extract content and metadata
|
||||
content = crawl_result.get("content", "")
|
||||
metadata = crawl_result.get("metadata", {})
|
||||
|
|
@ -457,6 +471,7 @@ async def index_crawled_urls(
|
|||
f"Successfully completed crawled web page indexing for connector {connector_id}",
|
||||
{
|
||||
"urls_processed": total_processed,
|
||||
"crawls_succeeded": crawls_succeeded,
|
||||
"documents_indexed": documents_indexed,
|
||||
"documents_updated": documents_updated,
|
||||
"documents_skipped": documents_skipped,
|
||||
|
|
|
|||
|
|
@ -469,14 +469,6 @@ def validate_connector_config(
|
|||
if not isinstance(value, list) or not value:
|
||||
raise ValueError(f"{field_name} must be a non-empty list of strings")
|
||||
|
||||
def validate_firecrawl_api_key_format() -> None:
|
||||
"""Validate Firecrawl API key format if provided."""
|
||||
api_key = config.get("FIRECRAWL_API_KEY", "")
|
||||
if api_key and api_key.strip() and not api_key.strip().startswith("fc-"):
|
||||
raise ValueError(
|
||||
"Firecrawl API key should start with 'fc-'. Please verify your API key."
|
||||
)
|
||||
|
||||
def validate_initial_urls() -> None:
|
||||
initial_urls = config.get("INITIAL_URLS", "")
|
||||
if initial_urls and initial_urls.strip():
|
||||
|
|
@ -571,10 +563,9 @@ def validate_connector_config(
|
|||
# },
|
||||
"LUMA_CONNECTOR": {"required": ["LUMA_API_KEY"], "validators": {}},
|
||||
"WEBCRAWLER_CONNECTOR": {
|
||||
"required": [], # No required fields - API key is optional
|
||||
"optional": ["FIRECRAWL_API_KEY", "INITIAL_URLS"],
|
||||
"required": [], # No required fields - URLs are optional
|
||||
"optional": ["INITIAL_URLS"],
|
||||
"validators": {
|
||||
"FIRECRAWL_API_KEY": lambda: validate_firecrawl_api_key_format(),
|
||||
"INITIAL_URLS": lambda: validate_initial_urls(),
|
||||
},
|
||||
},
|
||||
|
|
|
|||
|
|
@ -42,7 +42,6 @@ dependencies = [
|
|||
"faster-whisper>=1.1.0",
|
||||
"celery[redis]>=5.5.3",
|
||||
"redis>=5.2.1",
|
||||
"firecrawl-py>=4.9.0",
|
||||
"boto3>=1.35.0",
|
||||
"azure-storage-blob>=12.23.0",
|
||||
"fake-useragent>=2.2.0",
|
||||
|
|
|
|||
|
|
@ -39,9 +39,9 @@ def patched_side_effects(monkeypatch: pytest.MonkeyPatch):
|
|||
"""Stub the connector setup + checkpointer so only policy/LLM logic runs."""
|
||||
|
||||
async def _fake_setup(_session, *, workspace_id):
|
||||
return (SimpleNamespace(name="connector"), "fc-key")
|
||||
return SimpleNamespace(name="connector")
|
||||
|
||||
monkeypatch.setattr(deps_mod, "setup_connector_and_firecrawl", _fake_setup)
|
||||
monkeypatch.setattr(deps_mod, "setup_connector_service", _fake_setup)
|
||||
return None
|
||||
|
||||
|
||||
|
|
@ -78,7 +78,7 @@ async def test_build_dependencies_resolves_captured_chat_model_id(
|
|||
|
||||
assert captured == {"config_id": -7, "workspace_id": 42}
|
||||
assert result.llm.name == "llm"
|
||||
assert result.firecrawl_api_key == "fc-key"
|
||||
assert result.connector_service.name == "connector"
|
||||
|
||||
|
||||
async def test_build_dependencies_validates_captured_ids(
|
||||
|
|
|
|||
0
surfsense_backend/tests/unit/proprietary/__init__.py
Normal file
0
surfsense_backend/tests/unit/proprietary/__init__.py
Normal file
|
|
@ -0,0 +1,118 @@
|
|||
"""Unit tests for ``WebCrawlerConnector.crawl_url`` outcome semantics (Phase 3a).
|
||||
|
||||
These exercise the Scrapling tier ladder and the explicit ``CrawlOutcome``
|
||||
contract that Phase 3c bills on, by stubbing the per-tier fetch helpers so the
|
||||
ladder logic is tested deterministically without launching browsers/HTTP.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from app.proprietary.web_crawler import (
|
||||
CrawlOutcomeStatus,
|
||||
WebCrawlerConnector,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.unit
|
||||
|
||||
|
||||
def _result(tier: str) -> dict:
|
||||
return {
|
||||
"content": "hello world",
|
||||
"metadata": {"source": "https://example.com", "title": "Example"},
|
||||
"crawler_type": tier,
|
||||
}
|
||||
|
||||
|
||||
async def test_invalid_url_is_failed() -> None:
|
||||
"""A URL that fails validation never reaches a tier and is FAILED."""
|
||||
outcome = await WebCrawlerConnector().crawl_url("not a url")
|
||||
|
||||
assert outcome.status is CrawlOutcomeStatus.FAILED
|
||||
assert outcome.result is None
|
||||
assert "Invalid URL" in (outcome.error or "")
|
||||
|
||||
|
||||
async def test_static_tier_success_short_circuits(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Static success returns SUCCESS and never touches the browser tiers."""
|
||||
crawler = WebCrawlerConnector()
|
||||
later_calls: list[str] = []
|
||||
|
||||
async def _static(_url: str) -> dict:
|
||||
return _result("scrapling-static")
|
||||
|
||||
async def _record_dynamic(_url: str) -> None:
|
||||
later_calls.append("dynamic")
|
||||
return None
|
||||
|
||||
async def _record_stealthy(_url: str) -> None:
|
||||
later_calls.append("stealthy")
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(crawler, "_crawl_with_async_fetcher", _static)
|
||||
monkeypatch.setattr(crawler, "_crawl_with_dynamic", _record_dynamic)
|
||||
monkeypatch.setattr(crawler, "_crawl_with_stealthy", _record_stealthy)
|
||||
|
||||
outcome = await crawler.crawl_url("https://example.com")
|
||||
|
||||
assert outcome.status is CrawlOutcomeStatus.SUCCESS
|
||||
assert outcome.tier == "scrapling-static"
|
||||
assert outcome.result is not None
|
||||
assert outcome.result["crawler_type"] == "scrapling-static"
|
||||
assert later_calls == []
|
||||
|
||||
|
||||
async def test_escalates_to_dynamic_on_static_miss(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Static empty extraction escalates to the dynamic tier."""
|
||||
crawler = WebCrawlerConnector()
|
||||
|
||||
async def _empty(_url: str) -> None:
|
||||
return None
|
||||
|
||||
async def _dynamic(_url: str) -> dict:
|
||||
return _result("scrapling-dynamic")
|
||||
|
||||
monkeypatch.setattr(crawler, "_crawl_with_async_fetcher", _empty)
|
||||
monkeypatch.setattr(crawler, "_crawl_with_dynamic", _dynamic)
|
||||
|
||||
outcome = await crawler.crawl_url("https://example.com")
|
||||
|
||||
assert outcome.status is CrawlOutcomeStatus.SUCCESS
|
||||
assert outcome.tier == "scrapling-dynamic"
|
||||
|
||||
|
||||
async def test_all_tiers_empty_is_empty(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Every tier fetched but extracted nothing -> EMPTY (not billable)."""
|
||||
crawler = WebCrawlerConnector()
|
||||
|
||||
async def _empty(_url: str) -> None:
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(crawler, "_crawl_with_async_fetcher", _empty)
|
||||
monkeypatch.setattr(crawler, "_crawl_with_dynamic", _empty)
|
||||
monkeypatch.setattr(crawler, "_crawl_with_stealthy", _empty)
|
||||
|
||||
outcome = await crawler.crawl_url("https://example.com")
|
||||
|
||||
assert outcome.status is CrawlOutcomeStatus.EMPTY
|
||||
assert outcome.result is None
|
||||
|
||||
|
||||
async def test_all_tiers_raise_is_failed(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Every tier raising (none reachable) -> FAILED with aggregated errors."""
|
||||
crawler = WebCrawlerConnector()
|
||||
|
||||
async def _boom(_url: str) -> None:
|
||||
raise RuntimeError("fetch exploded")
|
||||
|
||||
monkeypatch.setattr(crawler, "_crawl_with_async_fetcher", _boom)
|
||||
monkeypatch.setattr(crawler, "_crawl_with_dynamic", _boom)
|
||||
monkeypatch.setattr(crawler, "_crawl_with_stealthy", _boom)
|
||||
|
||||
outcome = await crawler.crawl_url("https://example.com")
|
||||
|
||||
assert outcome.status is CrawlOutcomeStatus.FAILED
|
||||
assert "fetch exploded" in (outcome.error or "")
|
||||
|
|
@ -332,9 +332,8 @@ def test_validate_connector_config_invalid():
|
|||
with pytest.raises(ValueError):
|
||||
validate_connector_config("SEARXNG_API", {"SEARXNG_HOST": "not-a-url"})
|
||||
|
||||
# Invalid email format (if JIRA was enabled, etc. We test with WEBCRAWLER's custom validation)
|
||||
# Firecrawl key format error:
|
||||
# WEBCRAWLER_CONNECTOR custom validation: malformed INITIAL_URLS rejected.
|
||||
with pytest.raises(ValueError):
|
||||
validate_connector_config(
|
||||
"WEBCRAWLER_CONNECTOR", {"FIRECRAWL_API_KEY": "invalid-prefix-key"}
|
||||
"WEBCRAWLER_CONNECTOR", {"INITIAL_URLS": "not-a-url"}
|
||||
)
|
||||
|
|
|
|||
152
surfsense_backend/uv.lock
generated
152
surfsense_backend/uv.lock
generated
|
|
@ -24,6 +24,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -46,6 +49,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version < '3.13' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -68,6 +74,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -90,6 +99,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -112,6 +124,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -134,6 +149,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -156,6 +174,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -178,6 +199,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -200,6 +224,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -222,6 +249,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -251,6 +281,10 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -273,6 +307,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -295,6 +332,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version < '3.13' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -317,6 +357,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -339,6 +382,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -361,6 +407,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -383,6 +432,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -405,6 +457,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -427,6 +482,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -449,6 +507,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -471,6 +532,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -500,6 +564,10 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -522,6 +590,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -544,6 +615,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version < '3.13' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -566,6 +640,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -588,6 +665,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -610,6 +690,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -632,6 +715,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -654,6 +740,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -676,6 +765,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -698,6 +790,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -720,6 +815,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version < '3.13' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -749,6 +847,10 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -771,6 +873,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -814,6 +919,12 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -836,6 +947,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -858,6 +972,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -901,6 +1018,12 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -944,6 +1067,12 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version < '3.13' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
|
|
@ -966,6 +1095,9 @@ resolution-markers = [
|
|||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_version < '0'",
|
||||
"python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'",
|
||||
]
|
||||
conflicts = [[
|
||||
|
|
@ -3421,24 +3553,6 @@ wheels = [
|
|||
{ url = "https://files.pythonhosted.org/packages/18/79/1b8fa1bb3568781e84c9200f951c735f3f157429f44be0495da55894d620/filetype-1.2.0-py2.py3-none-any.whl", hash = "sha256:7ce71b6880181241cf7ac8697a2f1eb6a8bd9b429f7ad6d27b8db9ba5f1c2d25", size = 19970 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "firecrawl-py"
|
||||
version = "4.21.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "aiohttp" },
|
||||
{ name = "httpx" },
|
||||
{ name = "nest-asyncio" },
|
||||
{ name = "pydantic" },
|
||||
{ name = "python-dotenv" },
|
||||
{ name = "requests" },
|
||||
{ name = "websockets" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/6d/6b/8201b737c0667bf70748b86a6fb117aefc648154b4e05c5ee649432cbc3d/firecrawl_py-4.21.0.tar.gz", hash = "sha256:14a7e0967d816c711c3c53325c9371e2f780a787d1e94333a34d8aea7a43a237", size = 174256 }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/18/f1/1c0f1e5b33a318d7b9705b9e23c4397253d730e516e3d8a2f6aaea4b71a2/firecrawl_py-4.21.0-py3-none-any.whl", hash = "sha256:4e431f36117b4f2aaae633e747859a91626b0f2c6aaa6b7f86dfb7669a3595eb", size = 217607 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "flashrank"
|
||||
version = "0.2.10"
|
||||
|
|
@ -9868,7 +9982,6 @@ dependencies = [
|
|||
{ name = "fastapi" },
|
||||
{ name = "fastapi-users", extra = ["oauth", "sqlalchemy"] },
|
||||
{ name = "faster-whisper" },
|
||||
{ name = "firecrawl-py" },
|
||||
{ name = "fractional-indexing" },
|
||||
{ name = "github3-py" },
|
||||
{ name = "gitingest" },
|
||||
|
|
@ -9986,7 +10099,6 @@ requires-dist = [
|
|||
{ name = "fastapi", specifier = ">=0.115.8" },
|
||||
{ name = "fastapi-users", extras = ["oauth", "sqlalchemy"], specifier = ">=15.0.3" },
|
||||
{ name = "faster-whisper", specifier = ">=1.1.0" },
|
||||
{ name = "firecrawl-py", specifier = ">=4.9.0" },
|
||||
{ name = "fractional-indexing", specifier = ">=0.1.3" },
|
||||
{ name = "github3-py", specifier = "==4.0.1" },
|
||||
{ name = "gitingest", specifier = ">=0.3.1" },
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue