From 5c36cd3071d974496f58ad1b2e0a03492872a124 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Mon, 29 Jun 2026 20:37:56 -0700 Subject: [PATCH] feat(03a): implement Scrapling-only web crawler and remove Firecrawl integration - Standardized the web crawler to use Scrapling exclusively, removing Firecrawl entirely. - Updated the crawler's location to `app/proprietary/web_crawler/connector.py` under a non-Apache-2 license boundary. - Refactored the `WebCrawlerConnector` to eliminate the Firecrawl API key dependency, simplifying the interface for crawling URLs. - Adjusted related components to accommodate the new structure and ensure successful crawl outcomes are properly handled. - Updated documentation to reflect these changes and the new implementation status. --- plans/backend/00-umbrella-plan.md | 5 +- plans/backend/03a-crawler-core.md | 4 + plans/backend/03b-proxy-expansion.md | 4 +- plans/backend/03e-stealth-hardening.md | 4 +- plans/backend/03f-undetectability-testing.md | 2 +- surfsense_backend/Dockerfile | 2 +- .../main_agent/runtime/factory.py | 2 - .../main_agent/tools/registry.py | 4 +- .../main_agent/tools/scrape_webpage.py | 31 +-- .../builtins/research/tools/index.py | 2 +- .../builtins/research/tools/scrape_webpage.py | 31 +-- .../builtin/agent_task/dependencies.py | 6 +- .../actions/builtin/agent_task/invoke.py | 1 - surfsense_backend/app/proprietary/LICENSE | 11 + surfsense_backend/app/proprietary/README.md | 32 +++ surfsense_backend/app/proprietary/__init__.py | 10 + .../app/proprietary/platforms/__init__.py | 7 + .../app/proprietary/web_crawler/__init__.py | 13 ++ .../web_crawler/connector.py} | 221 ++++++++---------- .../app/tasks/chat/streaming/agent/builder.py | 2 - .../streaming/flows/new_chat/orchestrator.py | 8 +- .../flows/resume_chat/orchestrator.py | 8 +- .../flows/shared/pre_stream_setup.py | 23 +- .../connector_indexers/webcrawler_indexer.py | 35 ++- surfsense_backend/app/utils/validators.py | 13 +- surfsense_backend/pyproject.toml | 1 - .../builtin/agent_task/test_dependencies.py | 6 +- .../tests/unit/proprietary/__init__.py | 0 .../unit/proprietary/web_crawler/__init__.py | 0 .../proprietary/web_crawler/test_connector.py | 118 ++++++++++ .../tests/unit/utils/test_validators.py | 5 +- surfsense_backend/uv.lock | 152 ++++++++++-- 32 files changed, 506 insertions(+), 257 deletions(-) create mode 100644 surfsense_backend/app/proprietary/LICENSE create mode 100644 surfsense_backend/app/proprietary/README.md create mode 100644 surfsense_backend/app/proprietary/__init__.py create mode 100644 surfsense_backend/app/proprietary/platforms/__init__.py create mode 100644 surfsense_backend/app/proprietary/web_crawler/__init__.py rename surfsense_backend/app/{connectors/webcrawler_connector.py => proprietary/web_crawler/connector.py} (74%) create mode 100644 surfsense_backend/tests/unit/proprietary/__init__.py create mode 100644 surfsense_backend/tests/unit/proprietary/web_crawler/__init__.py create mode 100644 surfsense_backend/tests/unit/proprietary/web_crawler/test_connector.py diff --git a/plans/backend/00-umbrella-plan.md b/plans/backend/00-umbrella-plan.md index c29bbd5f1..e9f6a3c46 100644 --- a/plans/backend/00-umbrella-plan.md +++ b/plans/backend/00-umbrella-plan.md @@ -79,7 +79,7 @@ The Universal WebURL Crawler is the flagship Type-1 data source (**the moat**). > **Sequencing within Phase 3 (critical path vs hardening).** Only **`03a` + `03b` + `03c`** are on the **MVP critical path** — Phases 4–7 (connector taxonomy, pipelines, execution) consume the `03a` `CrawlOutcome`/`FetchStrategy` contract, the `03b` proxy provider, and the `03c` `WebCrawlCreditService` billing seam. **`03d` (captcha), `03e` (stealth hardening), and `03f` (test harness) are hardening/measurement that nothing downstream imports** — they tune the *same* seam behind the *same* contract, so they can land **in parallel with, or after, Phases 4–7** without blocking the pivot. Recommended build order: `03a → 03b → 03c` (then proceed to Phase 4), with `03e → 03d → 03f` slotted in whenever crawler robustness is prioritized. The dependency notes inside `03d`/`03e`/`03f` (each requires `03a`/`03b`) still hold; this only frees them from gating Phase 4+. -- **`03a-crawler-core.md`** — Standardize the fetch layer on Scrapling. **Remove Firecrawl entirely** (no other frameworks). Define crisp per-URL success/empty/failure semantics, keep Trafilatura extraction, and expose a single billable "successful crawl" signal (one unit per URL that yields usable content, regardless of how many internal fallback tiers ran). +- **`03a-crawler-core.md`** *(✅ IMPLEMENTED — local on `ci_mvp`, uncommitted; crawler moved to `app/proprietary/web_crawler/`, `impersonate="chrome"` + `solve_cloudflare=True` shipped)* — Standardize the fetch layer on Scrapling. **Remove Firecrawl entirely** (no other frameworks). Define crisp per-URL success/empty/failure semantics, keep Trafilatura extraction, and expose a single billable "successful crawl" signal (one unit per URL that yields usable content, regardless of how many internal fallback tiers ran). - **`03b-proxy-expansion.md`** — Add a BYO `CustomProxyProvider` (the only new provider — **no branded vendors**) alongside `anonymous_proxies`, selectable via a **single, app-wide** `Config.PROXY_PROVIDER`. Add bounded client-side rotation+retry via Scrapling's `ProxyRotator`/`is_proxy_error` **only** when the active provider is pool-backed (`CUSTOM_PROXY_URLS`); single-endpoint providers (incl. `anonymous_proxies`) stay the default and no-op the retry. **No per-connector/per-crawl selection** (one provider app-wide); a per-pipeline override is left as a no-op seam for Phase 5/6. - **`03c-crawl-billing.md`** — Charge crawl credits at **$1 / 1000 successful requests = 1000 micro-USD per successful crawl**, drawn from the existing credit wallet (`credit_micros_balance`), gated by a new `WEB_CRAWL_CREDIT_BILLING_ENABLED` flag (off for self-hosted). Two surfaces: **connector/pipeline crawls** billed to the **workspace owner** via a dedicated `WebCrawlCreditService` (mirrors `EtlCreditService`'s gate → `check_credits` → `charge_credits`, **not** `billable_call`); **chat scrapes** fold their crawl cost into the chat turn's existing bill (turn accumulator). No DB migration (uses the existing free-form `web_crawl` usage_type). - **`03e-stealth-hardening.md`** — The in-house "undetectable" layer on top of Scrapling's default patchright-Chromium stealth. **Geoip coherence** (match browser `locale`/`timezone_id` to the proxy's exit geo), **fingerprint flags** (`hide_canvas`/`block_webrtc`), **persistent per-domain profiles** (`user_data_dir`), **headed execution under Xvfb**, **real fonts** in the worker image, and **DIY behavioral humanization** via `page_action` (the Chromium engine has no built-in `humanize`). Adds a **block classifier** (label Cloudflare/DataDome/Kasada/captcha/empty from the response) + **per-domain strategy memory** (Redis, no migration) so the ladder learns the known-good tier per domain. Defines — but does **not** build — the **deferred paid-unblocker `FetchStrategy`** (ZenRows/ScrapFly/Bright Data) as the config-flagged escape hatch, with its own (later) billing. Honest ceiling: defeats Cloudflare + the moderate long tail, **not** top-tier behavioral fingerprinting (DataDome/Kasada/reCAPTCHA-Enterprise) — that's the deferred tier. @@ -155,6 +155,7 @@ These are recorded for continuity but are NOT planned in this umbrella. They sta - Phase 4 search APIs: all 5 enum values dropped (`SERPER_API`/`TAVILY_API`/`SEARXNG_API`/`LINKUP_API`/`BAIDU_SEARCH_API`) in 04b. Survivors (SearXNG/Linkup/Baidu) become PLATFORM providers keyed by env (Linkup/Baidu keys move from per-connector `config` to env — app-wide, not per-workspace). 04b carries a destructive migration deleting the 5 connector types' rows. - Phase 4 structure: split into 04a (taxonomy/gating/MCP-fix, no migration) and 04b (search repurposing + source-discovery endpoint, with migration); intended order 04a -> 04b (both orders safe). - Rename transition policy: HARD CUTOVER of the external API (paths + JSON field names) in Phase 2 — no backward-compat aliases. Rationale: the frontend is (re)built against the corrected backend later, so there is no old client to keep alive; backend correctness is verified via the test suite + OpenAPI rather than the existing UI. +- Crawler code location & licensing boundary (decided during 03a impl): the WebURL crawler engine — and future Phase-8 platform actors — live under `surfsense_backend/app/proprietary/`, a **non-Apache-2 license boundary** (its own `LICENSE`, currently an all-rights-reserved placeholder; the repo root stays Apache-2). 03a's `WebCrawlerConnector` / `CrawlOutcome` / `CrawlOutcomeStatus` moved to `app/proprietary/web_crawler/` (public API re-exported from its `__init__`); the 3 Apache-2 callers (webcrawler indexer + both chat `scrape_webpage` tools) import `from app.proprietary.web_crawler import ...`. Rule: everything under `app/proprietary/**` is non-Apache-2; Apache-2 code may import *from* it but not move *into* it. Rationale: keep the moat under a clearly-bounded, swappable license. - WebURL Crawler framework: STANDARDIZE on Scrapling; **remove Firecrawl entirely** (no other scraping frameworks now or planned). Scrapling's `StealthyFetcher` (patchright-Chromium as of 0.4.9 — **not** Camoufox) handles Cloudflare; `03e` stealth-hardening minimizes challenges; captcha-tools (`03d`) covers the rest. All fetch tiers sit behind a `FetchStrategy` seam returning `CrawlOutcome` (callers never depend on the tier). - Crawl billing: reuse the existing credit wallet (`credit_micros_balance`) with a new `web_crawl` usage_type. Price: **$1 / 1000 successful requests** (1000 micro-USD per success). Connector/pipeline crawls bill the **workspace owner**; chat scrapes fold their crawl cost into the already-billed chat turn. Gated by `WEB_CRAWL_CREDIT_BILLING_ENABLED` (off for self-hosted); no DB migration required. - Billable unit: one unit per URL that returns usable extracted content, regardless of how many internal fallback tiers were attempted (not per HTTP fetch, not per URL-processed). @@ -174,7 +175,7 @@ These are recorded for continuity but are NOT planned in this umbrella. They sta |-------|--------------|--------| | 1 | `01-rename-db.md` | **SHIPPED** (2026-06-27, PR [#1546](https://github.com/MODSetter/SurfSense/pull/1546)/[#1562](https://github.com/MODSetter/SurfSense/pull/1562); migration `170`) | | 2 | `02-rename-backend.md` | **SHIPPED** (2026-06-27, PR [#1546](https://github.com/MODSetter/SurfSense/pull/1546)/[#1562](https://github.com/MODSetter/SurfSense/pull/1562)) | -| 3 | `03a-crawler-core.md` | drafted | +| 3 | `03a-crawler-core.md` | **IMPLEMENTED** (local on `ci_mvp`, uncommitted) — Firecrawl removed, Scrapling-only 3-tier `CrawlOutcome`, crawler relocated to `app/proprietary/web_crawler/` | | 3 | `03b-proxy-expansion.md` | drafted | | 3 | `03c-crawl-billing.md` | drafted | | 3 | `03e-stealth-hardening.md` | drafted | diff --git a/plans/backend/03a-crawler-core.md b/plans/backend/03a-crawler-core.md index a4b11665a..4e93f0de9 100644 --- a/plans/backend/03a-crawler-core.md +++ b/plans/backend/03a-crawler-core.md @@ -5,6 +5,8 @@ > **Implementation note (applies to all Phase-3 plans).** Phases 1–2 are **SHIPPED** (2026-06-27) — `SearchSpace`→`Workspace` and `search_space_id`→`workspace_id` are renamed everywhere, so the **live code already says `workspace_*`**. Where citations below use the **old** `search_space_*`/`SearchSpace` names (written pre-rename), substitute the `workspace_*` equivalent and **grep the new name** (grepping the old name now returns nothing). Apply every edit by **symbol/grep** (e.g. `firecrawl_api_key`, `crawl_url`, `FIRECRAWL_API_KEY`), **not** by the absolute line numbers cited here — the rename (and `03a`'s own Firecrawl removal) shifted them. +> **Status: ✅ IMPLEMENTED (local on `ci_mvp`, uncommitted).** All 5 work items done: Firecrawl removed repo-wide, `crawl_url` is the 3-tier Scrapling ladder returning `CrawlOutcome` (`impersonate="chrome"` on the static tier + `solve_cloudflare=True` on the stealthy tier), `crawls_succeeded` added to the indexer (positional return unchanged — surfaced in task metadata), both `scrape_webpage` tools updated, unit tests added/passing. Crawler relocated to `app/proprietary/web_crawler/connector.py` under the non-Apache-2 boundary (see umbrella decisions log + `app/proprietary/README.md`). + ## Objective Make the Universal WebURL Crawler a **single-framework (Scrapling) component** with deterministic per-URL outcome semantics and a clean, explicit **"successful crawl" signal** that Phase 3c bills on. @@ -20,6 +22,8 @@ This subplan does NOT touch proxy rotation (→ `03b`), credit metering (→ `03 ### The crawler +> **Post-implementation note (location moved).** 03a is implemented. The crawler now lives at `app/proprietary/web_crawler/connector.py` (public API re-exported from `app/proprietary/web_crawler/__init__.py`), under the non-Apache-2 license boundary `app/proprietary/` (see `app/proprietary/README.md` + `LICENSE`). Importers use `from app.proprietary.web_crawler import WebCrawlerConnector, CrawlOutcomeStatus`. The file:line references below describe the *original* `app/connectors/webcrawler_connector.py` surface that 03a refactored away (kept for historical context). + `app/connectors/webcrawler_connector.py` — `WebCrawlerConnector.crawl_url()` is a 4-tier fallback ladder: 1. **Firecrawl** (premium, if `firecrawl_api_key` set) — `_crawl_with_firecrawl()` (lines 89–100, 223–272), imports `from firecrawl import AsyncFirecrawlApp` (line 22). diff --git a/plans/backend/03b-proxy-expansion.md b/plans/backend/03b-proxy-expansion.md index b59476c59..4546c4027 100644 --- a/plans/backend/03b-proxy-expansion.md +++ b/plans/backend/03b-proxy-expansion.md @@ -3,7 +3,7 @@ > Part of **Phase 3 — WebURL Crawler & Crawl Billing**. See `00-umbrella-plan.md`. > Depends on `03a-crawler-core.md` (Scrapling-only crawler). Siblings: `03c-crawl-billing.md`, `03e-stealth-hardening.md`, `03d-captcha-solving.md`. -> **Implementation note.** Same convention as `03a`: Phases 1–2 are **SHIPPED**, so the live code already says `workspace_id`/`Workspace` — substitute for the old `search_space_*`/`SearchSpace` names in citations below and grep the new name. Crucially, the `webcrawler_connector.py` line numbers below (e.g. the three tiers at 287/329/359) and the `scrape_webpage.py` lines **predate `03a`'s Firecrawl removal + `crawl_url` refactor**, so they will have moved by the time `03b` is implemented. Locate code by **symbol/grep**, not absolute lines. +> **Implementation note.** Same convention as `03a`: Phases 1–2 are **SHIPPED**, so the live code already says `workspace_id`/`Workspace` — substitute for the old `search_space_*`/`SearchSpace` names in citations below and grep the new name. Crucially, `03a` is **now IMPLEMENTED**: the crawler moved to **`app/proprietary/web_crawler/connector.py`** (non-Apache-2 boundary) with Firecrawl removed and `crawl_url` refactored — the three Scrapling tiers now pass `proxy=get_proxy_url()` at **267/309/342** (not the pre-refactor 287/329/359). Locate code by **symbol/grep** against the new path, not absolute lines. ## Objective @@ -31,7 +31,7 @@ The zero-arg getters are used in **more than just the crawler**, so any signatur | Consumer | File:line | Uses | |----------|-----------|------| -| WebURL crawler (all 3 Scrapling tiers) | `connectors/webcrawler_connector.py` (287, 329, 359) | `get_proxy_url()` | +| WebURL crawler (all 3 Scrapling tiers) | `proprietary/web_crawler/connector.py` (267, 309, 342) | `get_proxy_url()` | | YouTube transcript route | `routes/youtube_routes.py` (78, 119) | `get_proxy_url()` | | YouTube processor (indexing) | `tasks/document_processors/youtube_processor.py` (223, 229) | `get_requests_proxies()`, `get_proxy_url()` | | Chat scrape tools (YouTube branch) | `main_agent/tools/scrape_webpage.py` (80, 93), `research/tools/scrape_webpage.py` (74, 87) | `get_requests_proxies()`, `get_proxy_url()` | diff --git a/plans/backend/03e-stealth-hardening.md b/plans/backend/03e-stealth-hardening.md index 0b8da4814..09227fe48 100644 --- a/plans/backend/03e-stealth-hardening.md +++ b/plans/backend/03e-stealth-hardening.md @@ -34,7 +34,7 @@ A residential IP in Berlin behind an `en-US`/`America/New_York` browser is an in ### 2b. HTTP-tier TLS fingerprint (the AsyncFetcher tier — `impersonate`) -The cheap static tier (`03a` tier 1) is the **first** thing every crawl hits, yet it currently sets `stealthy_headers=True` but **no** `impersonate` (`webcrawler_connector.py:284–289`), so its TLS ClientHello is curl_cffi's default JA3 — a non-browser signature that fingerprinting WAFs flag before the body even loads. Pass an `impersonate="chrome"` profile (Scrapling's static engine selects a matching curl_cffi browser profile — `references/Scrapling/scrapling/engines/static.py:36–47`) so the HTTP tier's **JA3/JA4/HTTP-2** matches a real Chrome and coheres with the browser tiers' UA. Cheap, safe, default-on. `03f §S3` is the test that validates parity against `tls.peet.ws`. +The cheap static tier (`03a` tier 1) is the **first** thing every crawl hits. `03a` **already ships `impersonate="chrome"`** on it (`app/proprietary/web_crawler/connector.py`, the `AsyncFetcher.get` call) — Scrapling's static engine selects a matching curl_cffi browser profile (`references/Scrapling/scrapling/engines/static.py:36–47`), so the HTTP tier's **JA3/JA4/HTTP-2** already matches a real Chrome and coheres with the browser tiers' UA. `03e`'s remaining work here is **not** to add it, but to (a) fold the `impersonate` profile into the centralized per-tier kwargs builder (below) so it stays the single source of truth, and (b) keep the chosen profile coherent with the proxy exit / UA. `03f §S3` validates the shipped parity against `tls.peet.ws`. ### 3. Persistent per-domain profiles (look like a returning human) @@ -92,7 +92,7 @@ Add (all default OFF / conservative; next to the `03b`/`03c` knobs in `config/__ ## Work items 1. **Geoip coherence** — resolve proxy exit geo → `locale`/`timezone_id`; thread the crawl's chosen endpoint into the strategy context (shared seam with `03d`). -2. **Fingerprint flags** — wire `hide_canvas`/`block_webrtc`/`google_search`/`extra_headers`/`additional_args` from config into the StealthyFetcher tier. Also add `impersonate="chrome"` to the AsyncFetcher (HTTP) tier (§2b) so its TLS coheres. **Centralize this into a single per-tier kwargs builder** (one function that returns the StealthyFetcher / AsyncFetcher kwargs from config) — the crawler *and* `03f`'s harness both import it, so the scorecard grades the exact browser we ship (no test-vs-prod drift). +2. **Fingerprint flags** — wire `hide_canvas`/`block_webrtc`/`google_search`/`extra_headers`/`additional_args` from config into the StealthyFetcher tier. The AsyncFetcher (HTTP) tier **already carries `impersonate="chrome"`** (shipped in `03a`, §2b) — **fold it into** the builder below, don't re-add it. **Centralize this into a single per-tier kwargs builder** (one function that returns the StealthyFetcher / AsyncFetcher kwargs from config) — the crawler *and* `03f`'s harness both import it, so the scorecard grades the exact browser we ship (no test-vs-prod drift). 3. **Persistent profiles** — per-domain `user_data_dir` under `CRAWL_PERSISTENT_PROFILES_DIR` (shared volume). 4. **Headed + Xvfb** — `headless=False` path gated by flag; Xvfb + fonts in the worker image. 5. **Humanization** — a `page_action` humanizer (mouse/scroll/dwell) composing with `03d`'s injector; optional `init_script` shims. diff --git a/plans/backend/03f-undetectability-testing.md b/plans/backend/03f-undetectability-testing.md index 93f02d1d6..02d7c28c4 100644 --- a/plans/backend/03f-undetectability-testing.md +++ b/plans/backend/03f-undetectability-testing.md @@ -54,7 +54,7 @@ We are **not** a single browser; we're the `03a` tier ladder. The harness theref ### S3. HTTP/TLS tier (AsyncFetcher / `Fetcher` — curl_cffi impersonation) -Our HTTP tier is `curl_cffi`-based and *can* impersonate a real Chrome TLS stack (`references/Scrapling/scrapling/fetchers/requests.py:29`; `engines/static.py:6–9,36–47`) — **but only if `impersonate=` is set.** It currently is **not** (`webcrawler_connector.py:284–289` passes `stealthy_headers=True` only), so today this tier's JA3 is curl_cffi's default and **will fail parity**. This row is therefore the *driver* for the `03e §2b` `impersonate="chrome"` lever: run it before (red) and after (green) that fix. +Our HTTP tier is `curl_cffi`-based and impersonates a real Chrome TLS stack (`references/Scrapling/scrapling/fetchers/requests.py:29`; `engines/static.py:6–9,36–47`) **only when `impersonate=` is set** — which `03a` **now ships** (`app/proprietary/web_crawler/connector.py`, the `AsyncFetcher.get` call passes `impersonate="chrome"`). This row therefore **validates the shipped parity** rather than driving a fix: confirm the static tier's JA3/JA4 matches a real Chrome (the `03e §2b` lever). If you ever need a before/after, temporarily drop `impersonate=` to reproduce the curl_cffi-default (red) baseline. - `tls.peet.ws/api/all` (+ `/api/clean`) — JSON **JA3/JA4/Akamai-HTTP2/PeetPrint**; diff against a real-Chrome baseline. - `httpbin.co/headers` (or httpbingo) — header set/order/UA sanity. diff --git a/surfsense_backend/Dockerfile b/surfsense_backend/Dockerfile index 292cb2671..25e186259 100644 --- a/surfsense_backend/Dockerfile +++ b/surfsense_backend/Dockerfile @@ -109,7 +109,7 @@ RUN --mount=type=secret,id=HF_TOKEN \ HF_TOKEN="$(cat /run/secrets/HF_TOKEN 2>/dev/null || true)" \ python -c "from chonkie import AutoEmbeddings; AutoEmbeddings.get_embeddings('${EMBEDDING_MODEL}')" -# Install Scrapling's browser engines (patchright Chromium + Camoufox). +# Install Scrapling's browser engines (patchright Chromium). # Scrapling pulls playwright/patchright via the `fetchers` extra; `scrapling install` # downloads the matching browser binaries used by DynamicFetcher/StealthyFetcher. RUN scrapling install diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/runtime/factory.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/runtime/factory.py index 5cc27b74b..91834ff25 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/runtime/factory.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/runtime/factory.py @@ -68,7 +68,6 @@ async def create_multi_agent_chat_deep_agent( enabled_tools: list[str] | None = None, disabled_tools: list[str] | None = None, additional_tools: Sequence[BaseTool] | None = None, - firecrawl_api_key: str | None = None, thread_visibility: ChatVisibility | None = None, mentioned_document_ids: list[int] | None = None, anon_session_id: str | None = None, @@ -139,7 +138,6 @@ async def create_multi_agent_chat_deep_agent( "workspace_id": workspace_id, "db_session": db_session, "connector_service": connector_service, - "firecrawl_api_key": firecrawl_api_key, "user_id": user_id, "auth_context": auth_context, "thread_id": thread_id, diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/registry.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/registry.py index 23ea8b8d6..a6f0917c4 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/registry.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/registry.py @@ -31,8 +31,8 @@ from .update_memory import ( ) -def _build_scrape_webpage_tool(deps: dict[str, Any]) -> BaseTool: - return create_scrape_webpage_tool(firecrawl_api_key=deps.get("firecrawl_api_key")) +def _build_scrape_webpage_tool(_deps: dict[str, Any]) -> BaseTool: + return create_scrape_webpage_tool() def _build_web_search_tool(deps: dict[str, Any]) -> BaseTool: diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/scrape_webpage.py b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/scrape_webpage.py index c66506ca7..e2eb85a3f 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/scrape_webpage.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/tools/scrape_webpage.py @@ -18,7 +18,10 @@ from requests import Session from scrapling.fetchers import AsyncFetcher from youtube_transcript_api import YouTubeTranscriptApi -from app.connectors.webcrawler_connector import WebCrawlerConnector +from app.proprietary.web_crawler import ( + CrawlOutcomeStatus, + WebCrawlerConnector, +) from app.tasks.document_processors.youtube_processor import get_youtube_video_id from app.utils.proxy import get_proxy_url, get_requests_proxies @@ -167,14 +170,10 @@ async def _scrape_youtube_video( } -def create_scrape_webpage_tool(firecrawl_api_key: str | None = None): +def create_scrape_webpage_tool(): """ Factory function to create the scrape_webpage tool. - Args: - firecrawl_api_key: Optional Firecrawl API key for premium web scraping. - Falls back to Chromium/Trafilatura if not provided. - Returns: A configured tool function for scraping webpages. """ @@ -229,10 +228,10 @@ def create_scrape_webpage_tool(firecrawl_api_key: str | None = None): if video_id: return await _scrape_youtube_video(url, video_id, max_length) - connector = WebCrawlerConnector(firecrawl_api_key=firecrawl_api_key) - result, error = await connector.crawl_url(url, formats=["markdown"]) + connector = WebCrawlerConnector() + outcome = await connector.crawl_url(url) - if error: + if outcome.status is not CrawlOutcomeStatus.SUCCESS or not outcome.result: return { "id": scrape_id, "assetId": url, @@ -240,20 +239,10 @@ def create_scrape_webpage_tool(firecrawl_api_key: str | None = None): "href": url, "title": domain or "Webpage", "domain": domain, - "error": error, - } - - if not result: - return { - "id": scrape_id, - "assetId": url, - "kind": "article", - "href": url, - "title": domain or "Webpage", - "domain": domain, - "error": "No content returned from crawler", + "error": outcome.error or "No content returned from crawler", } + result = outcome.result content = result.get("content", "") metadata = result.get("metadata", {}) diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/tools/index.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/tools/index.py index e1f199cdc..b3ea5a6b0 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/tools/index.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/tools/index.py @@ -25,5 +25,5 @@ def load_tools( workspace_id=d.get("workspace_id"), available_connectors=d.get("available_connectors"), ), - create_scrape_webpage_tool(firecrawl_api_key=d.get("firecrawl_api_key")), + create_scrape_webpage_tool(), ] diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/tools/scrape_webpage.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/tools/scrape_webpage.py index f367d7b57..a3812bb2c 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/tools/scrape_webpage.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/research/tools/scrape_webpage.py @@ -12,7 +12,10 @@ from requests import Session from scrapling.fetchers import AsyncFetcher from youtube_transcript_api import YouTubeTranscriptApi -from app.connectors.webcrawler_connector import WebCrawlerConnector +from app.proprietary.web_crawler import ( + CrawlOutcomeStatus, + WebCrawlerConnector, +) from app.tasks.document_processors.youtube_processor import get_youtube_video_id from app.utils.proxy import get_proxy_url, get_requests_proxies @@ -161,14 +164,10 @@ async def _scrape_youtube_video( } -def create_scrape_webpage_tool(firecrawl_api_key: str | None = None): +def create_scrape_webpage_tool(): """ Factory function to create the scrape_webpage tool. - Args: - firecrawl_api_key: Optional Firecrawl API key for premium web scraping. - Falls back to Chromium/Trafilatura if not provided. - Returns: A configured tool function for scraping webpages. """ @@ -223,10 +222,10 @@ def create_scrape_webpage_tool(firecrawl_api_key: str | None = None): if video_id: return await _scrape_youtube_video(url, video_id, max_length) - connector = WebCrawlerConnector(firecrawl_api_key=firecrawl_api_key) - result, error = await connector.crawl_url(url, formats=["markdown"]) + connector = WebCrawlerConnector() + outcome = await connector.crawl_url(url) - if error: + if outcome.status is not CrawlOutcomeStatus.SUCCESS or not outcome.result: return { "id": scrape_id, "assetId": url, @@ -234,20 +233,10 @@ def create_scrape_webpage_tool(firecrawl_api_key: str | None = None): "href": url, "title": domain or "Webpage", "domain": domain, - "error": error, - } - - if not result: - return { - "id": scrape_id, - "assetId": url, - "kind": "article", - "href": url, - "title": domain or "Webpage", - "domain": domain, - "error": "No content returned from crawler", + "error": outcome.error or "No content returned from crawler", } + result = outcome.result content = result.get("content", "") metadata = result.get("metadata", {}) diff --git a/surfsense_backend/app/automations/actions/builtin/agent_task/dependencies.py b/surfsense_backend/app/automations/actions/builtin/agent_task/dependencies.py index fecf180c2..7d8ee6fa1 100644 --- a/surfsense_backend/app/automations/actions/builtin/agent_task/dependencies.py +++ b/surfsense_backend/app/automations/actions/builtin/agent_task/dependencies.py @@ -16,7 +16,7 @@ from app.automations.services.model_policy import ( from app.db import Workspace from app.tasks.chat.streaming.flows.shared.llm_bundle import load_llm_bundle from app.tasks.chat.streaming.flows.shared.pre_stream_setup import ( - setup_connector_and_firecrawl, + setup_connector_service, ) @@ -31,7 +31,6 @@ class AgentDependencies: llm: Any agent_config: Any connector_service: Any - firecrawl_api_key: str | None checkpointer: Any @@ -82,7 +81,7 @@ async def build_dependencies( if err is not None or llm is None: raise DependencyError(err or "failed to load chat model config") - connector_service, firecrawl_api_key = await setup_connector_and_firecrawl( + connector_service = await setup_connector_service( session, workspace_id=workspace_id ) # Per-task InMemorySaver: the shared Postgres checkpointer's connection @@ -100,6 +99,5 @@ async def build_dependencies( llm=llm, agent_config=agent_config, connector_service=connector_service, - firecrawl_api_key=firecrawl_api_key, checkpointer=checkpointer, ) diff --git a/surfsense_backend/app/automations/actions/builtin/agent_task/invoke.py b/surfsense_backend/app/automations/actions/builtin/agent_task/invoke.py index 91ebd962e..c091aa9f5 100644 --- a/surfsense_backend/app/automations/actions/builtin/agent_task/invoke.py +++ b/surfsense_backend/app/automations/actions/builtin/agent_task/invoke.py @@ -171,7 +171,6 @@ async def run_agent_task( user_id=user_id, thread_id=None, agent_config=deps.agent_config, - firecrawl_api_key=deps.firecrawl_api_key, thread_visibility=ChatVisibility.PRIVATE, mentioned_document_ids=mentioned_document_ids, image_gen_model_id=ctx.image_gen_model_id, diff --git a/surfsense_backend/app/proprietary/LICENSE b/surfsense_backend/app/proprietary/LICENSE new file mode 100644 index 000000000..db6a892f6 --- /dev/null +++ b/surfsense_backend/app/proprietary/LICENSE @@ -0,0 +1,11 @@ +SurfSense Proprietary / Source-Available License (PLACEHOLDER) + +All files in this directory tree (app/proprietary/**) are NOT covered by the +Apache License 2.0 that governs the rest of this repository. + +The final license text is TBD. Until it is finalized, ALL RIGHTS ARE RESERVED: +the code in this directory may not be copied, modified, redistributed, sublicensed, +or relicensed (including under Apache-2.0) without the explicit prior written +permission of the copyright holder. + +Copyright (c) 2026 SurfSense. All rights reserved. diff --git a/surfsense_backend/app/proprietary/README.md b/surfsense_backend/app/proprietary/README.md new file mode 100644 index 000000000..b1c29aa35 --- /dev/null +++ b/surfsense_backend/app/proprietary/README.md @@ -0,0 +1,32 @@ +# `app.proprietary` — non-Apache-2 license boundary + +Everything in this directory tree is licensed **separately** from the rest of +SurfSense (which is Apache-2.0). See [`LICENSE`](./LICENSE). + +## Why this exists + +This package holds the product moat: + +- the in-house **undetectable web crawler** (Scrapling tiers + stealth/captcha + hardening), and +- (future) **platform-specific actors** that scrape/extract structured data from + individual platforms. + +Keeping it in one clearly-named directory makes the license boundary +unambiguous: a single rule — *everything under `app/proprietary/**` is not +Apache-2.0* — instead of per-file headers scattered across the tree. + +## Layout + +- `web_crawler/` — the Scrapling-based crawler engine. Public API: + `WebCrawlerConnector`, `CrawlOutcome`, `CrawlOutcomeStatus` + (`from app.proprietary.web_crawler import ...`). +- `platforms/` — (future, Phase 8) platform-specific actors; scaffolded/empty. + +## Rules + +- **Do not** add Apache-2.0-intended code here. +- Apache-2.0 code elsewhere **may import from** this package (the indexer and the + chat `scrape_webpage` tools do); that does not move them under this license. +- Depend only on the public API exported from each subpackage's `__init__`, not + on internal modules, so the boundary stays clean and swappable. diff --git a/surfsense_backend/app/proprietary/__init__.py b/surfsense_backend/app/proprietary/__init__.py new file mode 100644 index 000000000..2c1031e99 --- /dev/null +++ b/surfsense_backend/app/proprietary/__init__.py @@ -0,0 +1,10 @@ +"""SurfSense proprietary packages (non-Apache-2 license boundary). + +Everything under ``app.proprietary`` is licensed **separately** from the +Apache-2.0 project root — see ``app/proprietary/LICENSE``. This package holds +the in-house undetectable crawler engine and (future) platform-specific actors +that form the product's moat. + +Apache-2.0 code elsewhere in the app may *import from* this package, but code +placed *inside* it is not covered by the repository's Apache-2.0 license. +""" diff --git a/surfsense_backend/app/proprietary/platforms/__init__.py b/surfsense_backend/app/proprietary/platforms/__init__.py new file mode 100644 index 000000000..2c505c8ad --- /dev/null +++ b/surfsense_backend/app/proprietary/platforms/__init__.py @@ -0,0 +1,7 @@ +"""Platform-specific crawl/extraction actors (non-Apache-2; see ../LICENSE). + +Scaffolded for future work (Phase 8 — Platform Actors). Each platform (e.g. +maps, professional networks) will get its own subpackage that reuses the shared +fetch strategies in ``app.proprietary.web_crawler`` under a platform-specific +structured extractor. Empty for now by design. +""" diff --git a/surfsense_backend/app/proprietary/web_crawler/__init__.py b/surfsense_backend/app/proprietary/web_crawler/__init__.py new file mode 100644 index 000000000..d2a7d118a --- /dev/null +++ b/surfsense_backend/app/proprietary/web_crawler/__init__.py @@ -0,0 +1,13 @@ +"""Proprietary web crawler engine (non-Apache-2; see ``app/proprietary/LICENSE``). + +Public API for the single-framework (Scrapling) undetectable crawler. Callers +depend only on these symbols, never on internal tier/strategy details. +""" + +from app.proprietary.web_crawler.connector import ( + CrawlOutcome, + CrawlOutcomeStatus, + WebCrawlerConnector, +) + +__all__ = ["CrawlOutcome", "CrawlOutcomeStatus", "WebCrawlerConnector"] diff --git a/surfsense_backend/app/connectors/webcrawler_connector.py b/surfsense_backend/app/proprietary/web_crawler/connector.py similarity index 74% rename from surfsense_backend/app/connectors/webcrawler_connector.py rename to surfsense_backend/app/proprietary/web_crawler/connector.py index 1d4ff1f58..64f0e4afd 100644 --- a/surfsense_backend/app/connectors/webcrawler_connector.py +++ b/surfsense_backend/app/proprietary/web_crawler/connector.py @@ -1,25 +1,34 @@ +# SurfSense proprietary crawler engine. +# +# This module is part of the ``app.proprietary`` package and is licensed +# SEPARATELY from the Apache-2.0 project root. See ``app/proprietary/LICENSE``. +# Do not relicense or redistribute this file under Apache-2.0. """ WebCrawler Connector Module -A module for crawling web pages and extracting content using Firecrawl or -Scrapling's tiered fetchers, with Trafilatura for HTML -> markdown extraction. -Provides a unified interface for web scraping. +A single-framework (Scrapling) web crawler with Trafilatura for HTML -> markdown +extraction. Provides a unified interface for web scraping. -Fallback order: - 1. Firecrawl (if API key is configured) - 2. Scrapling AsyncFetcher (fast static HTTP, no browser subprocess) - 3. Scrapling DynamicFetcher (full browser, run in a thread) - 4. Scrapling StealthyFetcher (anti-bot stealth browser, run in a thread) +Fallback ladder (the ``FetchStrategy`` seam — see ``plans/backend/03a-crawler-core.md``): + 1. Scrapling AsyncFetcher (fast static HTTP, TLS-impersonated, no subprocess) + 2. Scrapling DynamicFetcher (full browser, run in a thread) + 3. Scrapling StealthyFetcher (patchright-Chromium anti-bot + Cloudflare solving, + run in a thread) + +Every tier returns extracted content via the same ``CrawlOutcome`` contract, so +callers (indexer, chat tool, crawl billing) depend only on the outcome, never on +which tier produced it. """ import asyncio import logging import time +from dataclasses import dataclass +from enum import Enum from typing import Any import trafilatura import validators -from firecrawl import AsyncFirecrawlApp from scrapling.fetchers import AsyncFetcher, DynamicFetcher, StealthyFetcher from app.utils.proxy import get_proxy_url @@ -30,76 +39,68 @@ logger = logging.getLogger(__name__) _PERF = "[webcrawler][perf]" +class CrawlOutcomeStatus(str, Enum): + """Deterministic per-URL crawl result, single-sourcing the billable signal.""" + + SUCCESS = "success" # a tier returned usable extracted content + EMPTY = "empty" # fetched, but no usable content after ALL tiers + FAILED = "failed" # invalid URL or every tier errored / was unavailable + + +@dataclass +class CrawlOutcome: + """Explicit ``crawl_url`` result shared by every caller. + + The **billable success predicate is single-sourced**: + ``status == CrawlOutcomeStatus.SUCCESS`` (Phase 3c meters on it). Picking a + dataclass over a tuple lets later subplans append fields without breaking + callers (03d adds ``captcha_attempts``/``captcha_solved`` for per-attempt + billing; 03e's block classifier can attach a ``block_type``). + """ + + status: CrawlOutcomeStatus + result: dict[str, Any] | None = None + error: str | None = None + tier: str | None = None + + class WebCrawlerConnector: """Class for crawling web pages and extracting content.""" - def __init__(self, firecrawl_api_key: str | None = None): - """ - Initialize the WebCrawlerConnector class. - - Args: - firecrawl_api_key: Firecrawl API key (optional). If provided, Firecrawl will be tried first - and Scrapling will be used as fallback if Firecrawl fails. If not provided, - Scrapling fetchers are used directly. - """ - self.firecrawl_api_key = firecrawl_api_key - self.use_firecrawl = bool(firecrawl_api_key) - - def set_api_key(self, api_key: str) -> None: - """ - Set the Firecrawl API key and enable Firecrawl usage. - - Args: - api_key: Firecrawl API key - """ - self.firecrawl_api_key = api_key - self.use_firecrawl = True - - async def crawl_url( - self, url: str, formats: list[str] | None = None - ) -> tuple[dict[str, Any] | None, str | None]: + async def crawl_url(self, url: str) -> CrawlOutcome: """ Crawl a single URL and extract its content. - Fallback order: - 1. Firecrawl (if API key configured) - 2. Scrapling AsyncFetcher (fast static HTTP, no subprocess) - 3. Scrapling DynamicFetcher (full browser, run in a thread) - 4. Scrapling StealthyFetcher (anti-bot stealth browser, run in a thread) + Fallback ladder: + 1. Scrapling AsyncFetcher (fast static HTTP, TLS-impersonated) + 2. Scrapling DynamicFetcher (full browser, run in a thread) + 3. Scrapling StealthyFetcher (anti-bot stealth browser + Cloudflare + solving, run in a thread) Args: url: URL to crawl - formats: List of formats to extract (e.g., ["markdown", "html"]) - only for Firecrawl Returns: - Tuple containing (crawl result dict, error message or None) - Result dict contains: - - content: Extracted content (markdown or HTML) + A ``CrawlOutcome``. On ``SUCCESS``, ``result`` is a dict containing: + - content: Extracted content (markdown) - metadata: Page metadata (title, description, etc.) - - source: Original URL - - crawler_type: Type of crawler used + - crawler_type: Identifier of the tier that produced the content """ total_start = time.perf_counter() try: if not validators.url(url): - return None, f"Invalid URL: {url}" + return CrawlOutcome( + status=CrawlOutcomeStatus.FAILED, + error=f"Invalid URL: {url}", + ) errors: list[str] = [] + # True once any tier fetched the page but extraction yielded nothing + # (distinguishes EMPTY from FAILED, where every tier raised/was + # unavailable). + reached_without_content = False - # --- 1. Firecrawl (premium, if configured) --- - if self.use_firecrawl: - tier_start = time.perf_counter() - try: - logger.info(f"[webcrawler] Using Firecrawl for: {url}") - result = await self._crawl_with_firecrawl(url, formats) - self._log_tier_outcome("firecrawl", url, tier_start, "success") - self._log_total(url, "firecrawl", total_start) - return result, None - except Exception as exc: - errors.append(f"Firecrawl: {exc!s}") - self._log_tier_outcome("firecrawl", url, tier_start, "error", exc) - - # --- 2. Scrapling AsyncFetcher (fast static HTTP) --- + # --- 1. Scrapling AsyncFetcher (fast static HTTP) --- tier_start = time.perf_counter() try: logger.info(f"[webcrawler] Using Scrapling AsyncFetcher for: {url}") @@ -109,7 +110,12 @@ class WebCrawlerConnector: "scrapling-static", url, tier_start, "success" ) self._log_total(url, "scrapling-static", total_start) - return result, None + return CrawlOutcome( + status=CrawlOutcomeStatus.SUCCESS, + result=result, + tier="scrapling-static", + ) + reached_without_content = True errors.append("Scrapling static: empty extraction") self._log_tier_outcome("scrapling-static", url, tier_start, "empty") except Exception as exc: @@ -118,7 +124,7 @@ class WebCrawlerConnector: "scrapling-static", url, tier_start, "error", exc ) - # --- 3. Scrapling DynamicFetcher (full browser) --- + # --- 2. Scrapling DynamicFetcher (full browser) --- tier_start = time.perf_counter() try: logger.info(f"[webcrawler] Using Scrapling DynamicFetcher for: {url}") @@ -128,7 +134,12 @@ class WebCrawlerConnector: "scrapling-dynamic", url, tier_start, "success" ) self._log_total(url, "scrapling-dynamic", total_start) - return result, None + return CrawlOutcome( + status=CrawlOutcomeStatus.SUCCESS, + result=result, + tier="scrapling-dynamic", + ) + reached_without_content = True errors.append("Scrapling dynamic: empty extraction") self._log_tier_outcome("scrapling-dynamic", url, tier_start, "empty") except NotImplementedError: @@ -145,7 +156,7 @@ class WebCrawlerConnector: "scrapling-dynamic", url, tier_start, "error", exc ) - # --- 4. Scrapling StealthyFetcher (anti-bot, last resort) --- + # --- 3. Scrapling StealthyFetcher (anti-bot, last resort) --- tier_start = time.perf_counter() try: logger.info(f"[webcrawler] Using Scrapling StealthyFetcher for: {url}") @@ -155,7 +166,12 @@ class WebCrawlerConnector: "scrapling-stealthy", url, tier_start, "success" ) self._log_total(url, "scrapling-stealthy", total_start) - return result, None + return CrawlOutcome( + status=CrawlOutcomeStatus.SUCCESS, + result=result, + tier="scrapling-stealthy", + ) + reached_without_content = True errors.append("Scrapling stealthy: empty extraction") self._log_tier_outcome("scrapling-stealthy", url, tier_start, "empty") except NotImplementedError: @@ -173,11 +189,22 @@ class WebCrawlerConnector: ) self._log_total(url, "none", total_start) - return None, f"All crawl methods failed for {url}. {'; '.join(errors)}" + if reached_without_content: + return CrawlOutcome( + status=CrawlOutcomeStatus.EMPTY, + error=f"No content extracted for {url}. {'; '.join(errors)}", + ) + return CrawlOutcome( + status=CrawlOutcomeStatus.FAILED, + error=f"All crawl methods failed for {url}. {'; '.join(errors)}", + ) except Exception as e: self._log_total(url, "error", total_start) - return None, f"Error crawling URL {url}: {e!s}" + return CrawlOutcome( + status=CrawlOutcomeStatus.FAILED, + error=f"Error crawling URL {url}: {e!s}", + ) @staticmethod def _log_tier_outcome( @@ -220,57 +247,6 @@ class WebCrawlerConnector: total_ms, ) - async def _crawl_with_firecrawl( - self, url: str, formats: list[str] | None = None - ) -> dict[str, Any]: - """ - Crawl URL using Firecrawl. - - Args: - url: URL to crawl - formats: List of formats to extract - - Returns: - Dict containing crawled content and metadata - - Raises: - ValueError: If Firecrawl scraping fails - """ - if not self.firecrawl_api_key: - raise ValueError("Firecrawl API key not set. Call set_api_key() first.") - - firecrawl_app = AsyncFirecrawlApp(api_key=self.firecrawl_api_key) - - # Default to markdown format - if formats is None: - formats = ["markdown"] - - # v2 API returns Document directly and raises an exception on failure - scrape_result = await firecrawl_app.scrape(url, formats=formats) - - if not scrape_result: - raise ValueError("Firecrawl returned no result") - - # Extract content based on format - content = scrape_result.markdown or scrape_result.html or "" - - # Extract metadata - v2 returns DocumentMetadata object - metadata_obj = scrape_result.metadata - metadata = metadata_obj.model_dump() if metadata_obj else {} - - return { - "content": content, - "metadata": { - "source": url, - "title": metadata.get("title", url), - "description": metadata.get("description", ""), - "language": metadata.get("language", ""), - "sourceURL": metadata.get("source_url", url), - **metadata, - }, - "crawler_type": "firecrawl", - } - async def _crawl_with_async_fetcher(self, url: str) -> dict[str, Any] | None: """ Crawl URL using Scrapling's AsyncFetcher (static HTTP) + Trafilatura. @@ -281,9 +257,13 @@ class WebCrawlerConnector: rendered SPAs) so the caller can fall through to the browser tiers. """ fetch_start = time.perf_counter() + # ``impersonate="chrome"`` makes curl_cffi present a real Chrome TLS + # ClientHello (JA3/JA4) instead of its default fingerprint, keeping the + # static tier coherent with the browser tiers' UA (see 03e §2b). page = await AsyncFetcher.get( url, stealthy_headers=True, + impersonate="chrome", proxy=get_proxy_url(), timeout=20, ) @@ -340,7 +320,7 @@ class WebCrawlerConnector: async def _crawl_with_stealthy(self, url: str) -> dict[str, Any] | None: """ - Crawl URL using Scrapling's StealthyFetcher (Camoufox) + Trafilatura. + Crawl URL using Scrapling's StealthyFetcher (patchright-Chromium) + Trafilatura. Last-resort tier with anti-bot features. Runs the sync fetch in a worker thread for the same event-loop-safety reasons as DynamicFetcher. Falls @@ -351,11 +331,14 @@ class WebCrawlerConnector: def _crawl_with_stealthy_sync(self, url: str) -> dict[str, Any] | None: """Synchronous StealthyFetcher crawl executed in a worker thread.""" fetch_start = time.perf_counter() + # ``solve_cloudflare=True`` runs the full Turnstile/Interstitial challenge + # loop; scoped to this last-resort tier only (it spins up the browser). page = StealthyFetcher.fetch( url, headless=True, network_idle=True, block_ads=True, + solve_cloudflare=True, proxy=get_proxy_url(), ) fetch_ms = (time.perf_counter() - fetch_start) * 1000 diff --git a/surfsense_backend/app/tasks/chat/streaming/agent/builder.py b/surfsense_backend/app/tasks/chat/streaming/agent/builder.py index 49c442695..a199d860a 100644 --- a/surfsense_backend/app/tasks/chat/streaming/agent/builder.py +++ b/surfsense_backend/app/tasks/chat/streaming/agent/builder.py @@ -29,7 +29,6 @@ async def build_main_agent_for_thread( user_id: str | None, thread_id: int | None, agent_config: AgentConfig | None, - firecrawl_api_key: str | None, thread_visibility: ChatVisibility | None, filesystem_selection: FilesystemSelection | None, disabled_tools: list[str] | None = None, @@ -45,7 +44,6 @@ async def build_main_agent_for_thread( user_id=user_id, thread_id=thread_id, agent_config=agent_config, - firecrawl_api_key=firecrawl_api_key, thread_visibility=thread_visibility, filesystem_selection=filesystem_selection, disabled_tools=disabled_tools, diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/orchestrator.py b/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/orchestrator.py index 093b5bdbe..4451b4b37 100644 --- a/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/orchestrator.py +++ b/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/orchestrator.py @@ -83,7 +83,7 @@ from app.tasks.chat.streaming.flows.shared.first_frames import ( from app.tasks.chat.streaming.flows.shared.llm_bundle import load_llm_bundle from app.tasks.chat.streaming.flows.shared.pre_stream_setup import ( get_chat_checkpointer, - setup_connector_and_firecrawl, + setup_connector_service, ) from app.tasks.chat.streaming.flows.shared.premium_quota import ( CreditReservation, @@ -376,11 +376,11 @@ async def stream_new_chat( ) _t0 = time.perf_counter() - connector_service, firecrawl_api_key = await setup_connector_and_firecrawl( + connector_service = await setup_connector_service( session, workspace_id=workspace_id ) _perf_log.info( - "[stream_new_chat] Connector service + firecrawl key in %.3fs", + "[stream_new_chat] Connector service in %.3fs", time.perf_counter() - _t0, ) @@ -410,7 +410,6 @@ async def stream_new_chat( user_id=user_id, thread_id=chat_id, agent_config=agent_config, - firecrawl_api_key=firecrawl_api_key, thread_visibility=visibility, filesystem_selection=filesystem_selection, disabled_tools=disabled_tools, @@ -665,7 +664,6 @@ async def stream_new_chat( user_id=user_id, thread_id=chat_id, agent_config=agent_config, - firecrawl_api_key=firecrawl_api_key, thread_visibility=visibility, filesystem_selection=filesystem_selection, disabled_tools=disabled_tools, diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/resume_chat/orchestrator.py b/surfsense_backend/app/tasks/chat/streaming/flows/resume_chat/orchestrator.py index 2bd0572ef..8b7956f4c 100644 --- a/surfsense_backend/app/tasks/chat/streaming/flows/resume_chat/orchestrator.py +++ b/surfsense_backend/app/tasks/chat/streaming/flows/resume_chat/orchestrator.py @@ -62,7 +62,7 @@ from app.tasks.chat.streaming.flows.shared.first_frames import ( from app.tasks.chat.streaming.flows.shared.llm_bundle import load_llm_bundle from app.tasks.chat.streaming.flows.shared.pre_stream_setup import ( get_chat_checkpointer, - setup_connector_and_firecrawl, + setup_connector_service, ) from app.tasks.chat.streaming.flows.shared.premium_quota import ( CreditReservation, @@ -314,11 +314,11 @@ async def stream_resume_chat( # --- Pre-stream setup --- _t0 = time.perf_counter() - connector_service, firecrawl_api_key = await setup_connector_and_firecrawl( + connector_service = await setup_connector_service( session, workspace_id=workspace_id ) _perf_log.info( - "[stream_resume] Connector service + firecrawl key in %.3fs", + "[stream_resume] Connector service in %.3fs", time.perf_counter() - _t0, ) @@ -344,7 +344,6 @@ async def stream_resume_chat( user_id=user_id, thread_id=chat_id, agent_config=agent_config, - firecrawl_api_key=firecrawl_api_key, thread_visibility=visibility, filesystem_selection=filesystem_selection, disabled_tools=disabled_tools, @@ -480,7 +479,6 @@ async def stream_resume_chat( user_id=user_id, thread_id=chat_id, agent_config=agent_config, - firecrawl_api_key=firecrawl_api_key, thread_visibility=visibility, filesystem_selection=filesystem_selection, disabled_tools=disabled_tools, diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/shared/pre_stream_setup.py b/surfsense_backend/app/tasks/chat/streaming/flows/shared/pre_stream_setup.py index 8c27aa935..4cfb4fd60 100644 --- a/surfsense_backend/app/tasks/chat/streaming/flows/shared/pre_stream_setup.py +++ b/surfsense_backend/app/tasks/chat/streaming/flows/shared/pre_stream_setup.py @@ -1,33 +1,20 @@ -"""Pre-stream setup: connector service, firecrawl key, checkpointer.""" +"""Pre-stream setup: connector service, checkpointer.""" from __future__ import annotations from sqlalchemy.ext.asyncio import AsyncSession from app.agents.chat.runtime.checkpointer import get_checkpointer -from app.db import SearchSourceConnectorType from app.services.connector_service import ConnectorService -async def setup_connector_and_firecrawl( +async def setup_connector_service( session: AsyncSession, *, workspace_id: int, -) -> tuple[ConnectorService, str | None]: - """Build the per-turn connector service and pull the firecrawl API key. - - Returns ``(connector_service, firecrawl_api_key)``. ``firecrawl_api_key`` is - ``None`` when no web-crawler connector is configured (the agent simply - skips firecrawl-backed tools in that case). - """ - connector_service = ConnectorService(session, workspace_id=workspace_id) - firecrawl_api_key: str | None = None - webcrawler_connector = await connector_service.get_connector_by_type( - SearchSourceConnectorType.WEBCRAWLER_CONNECTOR, workspace_id - ) - if webcrawler_connector and webcrawler_connector.config: - firecrawl_api_key = webcrawler_connector.config.get("FIRECRAWL_API_KEY") - return connector_service, firecrawl_api_key +) -> ConnectorService: + """Build the per-turn connector service for the workspace.""" + return ConnectorService(session, workspace_id=workspace_id) async def get_chat_checkpointer(): diff --git a/surfsense_backend/app/tasks/connector_indexers/webcrawler_indexer.py b/surfsense_backend/app/tasks/connector_indexers/webcrawler_indexer.py index 2390804a4..b26e77dfa 100644 --- a/surfsense_backend/app/tasks/connector_indexers/webcrawler_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/webcrawler_indexer.py @@ -13,7 +13,10 @@ from datetime import datetime from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.ext.asyncio import AsyncSession -from app.connectors.webcrawler_connector import WebCrawlerConnector +from app.proprietary.web_crawler import ( + CrawlOutcomeStatus, + WebCrawlerConnector, +) from app.db import Document, DocumentStatus, DocumentType, SearchSourceConnectorType from app.services.task_logging_service import TaskLoggingService from app.utils.document_converters import ( @@ -111,9 +114,6 @@ async def index_crawled_urls( f"Connector with ID {connector_id} not found or is not a webcrawler connector", ) - # Get the Firecrawl API key from the connector config (optional) - api_key = connector.config.get("FIRECRAWL_API_KEY") - # Get URLs from connector config raw_initial_urls = connector.config.get("INITIAL_URLS") urls = parse_webcrawler_urls(raw_initial_urls) @@ -132,11 +132,10 @@ async def index_crawled_urls( f"Initializing webcrawler client for connector {connector_id}", { "stage": "client_initialization", - "use_firecrawl": bool(api_key), }, ) - crawler = WebCrawlerConnector(firecrawl_api_key=api_key) + crawler = WebCrawlerConnector() # Validate URLs if not urls: @@ -169,6 +168,10 @@ async def index_crawled_urls( documents_skipped = 0 documents_failed = 0 duplicate_content_count = 0 + # Explicit "crawl succeeded" count: one per URL that yielded usable + # extracted content, independent of downstream KB dedupe/unchanged + # bucketing. This is the billable unit Phase 3c meters on. + crawls_succeeded = 0 # Heartbeat tracking - update notification periodically to prevent appearing stuck last_heartbeat_time = time.time() @@ -294,16 +297,27 @@ async def index_crawled_urls( ) # Crawl the URL - crawl_result, error = await crawler.crawl_url(url) + outcome = await crawler.crawl_url(url) - if error or not crawl_result: - logger.warning(f"Failed to crawl URL {url}: {error}") - document.status = DocumentStatus.failed(error or "Crawl failed") + if ( + outcome.status is not CrawlOutcomeStatus.SUCCESS + or not outcome.result + ): + logger.warning(f"Failed to crawl URL {url}: {outcome.error}") + document.status = DocumentStatus.failed( + outcome.error or "Crawl failed" + ) document.updated_at = get_current_timestamp() await session.commit() documents_failed += 1 continue + # A tier extracted usable content: count the successful crawl now, + # before the dedupe/unchanged branches (those still represent a + # successful crawl and must bill — see 03c). + crawls_succeeded += 1 + crawl_result = outcome.result + # Extract content and metadata content = crawl_result.get("content", "") metadata = crawl_result.get("metadata", {}) @@ -457,6 +471,7 @@ async def index_crawled_urls( f"Successfully completed crawled web page indexing for connector {connector_id}", { "urls_processed": total_processed, + "crawls_succeeded": crawls_succeeded, "documents_indexed": documents_indexed, "documents_updated": documents_updated, "documents_skipped": documents_skipped, diff --git a/surfsense_backend/app/utils/validators.py b/surfsense_backend/app/utils/validators.py index 215e78ab7..4511fb6ac 100644 --- a/surfsense_backend/app/utils/validators.py +++ b/surfsense_backend/app/utils/validators.py @@ -469,14 +469,6 @@ def validate_connector_config( if not isinstance(value, list) or not value: raise ValueError(f"{field_name} must be a non-empty list of strings") - def validate_firecrawl_api_key_format() -> None: - """Validate Firecrawl API key format if provided.""" - api_key = config.get("FIRECRAWL_API_KEY", "") - if api_key and api_key.strip() and not api_key.strip().startswith("fc-"): - raise ValueError( - "Firecrawl API key should start with 'fc-'. Please verify your API key." - ) - def validate_initial_urls() -> None: initial_urls = config.get("INITIAL_URLS", "") if initial_urls and initial_urls.strip(): @@ -571,10 +563,9 @@ def validate_connector_config( # }, "LUMA_CONNECTOR": {"required": ["LUMA_API_KEY"], "validators": {}}, "WEBCRAWLER_CONNECTOR": { - "required": [], # No required fields - API key is optional - "optional": ["FIRECRAWL_API_KEY", "INITIAL_URLS"], + "required": [], # No required fields - URLs are optional + "optional": ["INITIAL_URLS"], "validators": { - "FIRECRAWL_API_KEY": lambda: validate_firecrawl_api_key_format(), "INITIAL_URLS": lambda: validate_initial_urls(), }, }, diff --git a/surfsense_backend/pyproject.toml b/surfsense_backend/pyproject.toml index 8c9e96852..044bbf2ab 100644 --- a/surfsense_backend/pyproject.toml +++ b/surfsense_backend/pyproject.toml @@ -42,7 +42,6 @@ dependencies = [ "faster-whisper>=1.1.0", "celery[redis]>=5.5.3", "redis>=5.2.1", - "firecrawl-py>=4.9.0", "boto3>=1.35.0", "azure-storage-blob>=12.23.0", "fake-useragent>=2.2.0", diff --git a/surfsense_backend/tests/unit/automations/actions/builtin/agent_task/test_dependencies.py b/surfsense_backend/tests/unit/automations/actions/builtin/agent_task/test_dependencies.py index bc59ee99e..15ef61418 100644 --- a/surfsense_backend/tests/unit/automations/actions/builtin/agent_task/test_dependencies.py +++ b/surfsense_backend/tests/unit/automations/actions/builtin/agent_task/test_dependencies.py @@ -39,9 +39,9 @@ def patched_side_effects(monkeypatch: pytest.MonkeyPatch): """Stub the connector setup + checkpointer so only policy/LLM logic runs.""" async def _fake_setup(_session, *, workspace_id): - return (SimpleNamespace(name="connector"), "fc-key") + return SimpleNamespace(name="connector") - monkeypatch.setattr(deps_mod, "setup_connector_and_firecrawl", _fake_setup) + monkeypatch.setattr(deps_mod, "setup_connector_service", _fake_setup) return None @@ -78,7 +78,7 @@ async def test_build_dependencies_resolves_captured_chat_model_id( assert captured == {"config_id": -7, "workspace_id": 42} assert result.llm.name == "llm" - assert result.firecrawl_api_key == "fc-key" + assert result.connector_service.name == "connector" async def test_build_dependencies_validates_captured_ids( diff --git a/surfsense_backend/tests/unit/proprietary/__init__.py b/surfsense_backend/tests/unit/proprietary/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/proprietary/web_crawler/__init__.py b/surfsense_backend/tests/unit/proprietary/web_crawler/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/proprietary/web_crawler/test_connector.py b/surfsense_backend/tests/unit/proprietary/web_crawler/test_connector.py new file mode 100644 index 000000000..81759787f --- /dev/null +++ b/surfsense_backend/tests/unit/proprietary/web_crawler/test_connector.py @@ -0,0 +1,118 @@ +"""Unit tests for ``WebCrawlerConnector.crawl_url`` outcome semantics (Phase 3a). + +These exercise the Scrapling tier ladder and the explicit ``CrawlOutcome`` +contract that Phase 3c bills on, by stubbing the per-tier fetch helpers so the +ladder logic is tested deterministically without launching browsers/HTTP. +""" + +import pytest + +from app.proprietary.web_crawler import ( + CrawlOutcomeStatus, + WebCrawlerConnector, +) + +pytestmark = pytest.mark.unit + + +def _result(tier: str) -> dict: + return { + "content": "hello world", + "metadata": {"source": "https://example.com", "title": "Example"}, + "crawler_type": tier, + } + + +async def test_invalid_url_is_failed() -> None: + """A URL that fails validation never reaches a tier and is FAILED.""" + outcome = await WebCrawlerConnector().crawl_url("not a url") + + assert outcome.status is CrawlOutcomeStatus.FAILED + assert outcome.result is None + assert "Invalid URL" in (outcome.error or "") + + +async def test_static_tier_success_short_circuits( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Static success returns SUCCESS and never touches the browser tiers.""" + crawler = WebCrawlerConnector() + later_calls: list[str] = [] + + async def _static(_url: str) -> dict: + return _result("scrapling-static") + + async def _record_dynamic(_url: str) -> None: + later_calls.append("dynamic") + return None + + async def _record_stealthy(_url: str) -> None: + later_calls.append("stealthy") + return None + + monkeypatch.setattr(crawler, "_crawl_with_async_fetcher", _static) + monkeypatch.setattr(crawler, "_crawl_with_dynamic", _record_dynamic) + monkeypatch.setattr(crawler, "_crawl_with_stealthy", _record_stealthy) + + outcome = await crawler.crawl_url("https://example.com") + + assert outcome.status is CrawlOutcomeStatus.SUCCESS + assert outcome.tier == "scrapling-static" + assert outcome.result is not None + assert outcome.result["crawler_type"] == "scrapling-static" + assert later_calls == [] + + +async def test_escalates_to_dynamic_on_static_miss( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Static empty extraction escalates to the dynamic tier.""" + crawler = WebCrawlerConnector() + + async def _empty(_url: str) -> None: + return None + + async def _dynamic(_url: str) -> dict: + return _result("scrapling-dynamic") + + monkeypatch.setattr(crawler, "_crawl_with_async_fetcher", _empty) + monkeypatch.setattr(crawler, "_crawl_with_dynamic", _dynamic) + + outcome = await crawler.crawl_url("https://example.com") + + assert outcome.status is CrawlOutcomeStatus.SUCCESS + assert outcome.tier == "scrapling-dynamic" + + +async def test_all_tiers_empty_is_empty(monkeypatch: pytest.MonkeyPatch) -> None: + """Every tier fetched but extracted nothing -> EMPTY (not billable).""" + crawler = WebCrawlerConnector() + + async def _empty(_url: str) -> None: + return None + + monkeypatch.setattr(crawler, "_crawl_with_async_fetcher", _empty) + monkeypatch.setattr(crawler, "_crawl_with_dynamic", _empty) + monkeypatch.setattr(crawler, "_crawl_with_stealthy", _empty) + + outcome = await crawler.crawl_url("https://example.com") + + assert outcome.status is CrawlOutcomeStatus.EMPTY + assert outcome.result is None + + +async def test_all_tiers_raise_is_failed(monkeypatch: pytest.MonkeyPatch) -> None: + """Every tier raising (none reachable) -> FAILED with aggregated errors.""" + crawler = WebCrawlerConnector() + + async def _boom(_url: str) -> None: + raise RuntimeError("fetch exploded") + + monkeypatch.setattr(crawler, "_crawl_with_async_fetcher", _boom) + monkeypatch.setattr(crawler, "_crawl_with_dynamic", _boom) + monkeypatch.setattr(crawler, "_crawl_with_stealthy", _boom) + + outcome = await crawler.crawl_url("https://example.com") + + assert outcome.status is CrawlOutcomeStatus.FAILED + assert "fetch exploded" in (outcome.error or "") diff --git a/surfsense_backend/tests/unit/utils/test_validators.py b/surfsense_backend/tests/unit/utils/test_validators.py index 7757bb157..82846bdb9 100644 --- a/surfsense_backend/tests/unit/utils/test_validators.py +++ b/surfsense_backend/tests/unit/utils/test_validators.py @@ -332,9 +332,8 @@ def test_validate_connector_config_invalid(): with pytest.raises(ValueError): validate_connector_config("SEARXNG_API", {"SEARXNG_HOST": "not-a-url"}) - # Invalid email format (if JIRA was enabled, etc. We test with WEBCRAWLER's custom validation) - # Firecrawl key format error: + # WEBCRAWLER_CONNECTOR custom validation: malformed INITIAL_URLS rejected. with pytest.raises(ValueError): validate_connector_config( - "WEBCRAWLER_CONNECTOR", {"FIRECRAWL_API_KEY": "invalid-prefix-key"} + "WEBCRAWLER_CONNECTOR", {"INITIAL_URLS": "not-a-url"} ) diff --git a/surfsense_backend/uv.lock b/surfsense_backend/uv.lock index bdce64f30..ef7045eb0 100644 --- a/surfsense_backend/uv.lock +++ b/surfsense_backend/uv.lock @@ -24,6 +24,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -46,6 +49,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -68,6 +74,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -90,6 +99,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -112,6 +124,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -134,6 +149,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -156,6 +174,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -178,6 +199,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -200,6 +224,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -222,6 +249,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra == 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -251,6 +281,10 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -273,6 +307,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -295,6 +332,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'linux' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -317,6 +357,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -339,6 +382,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -361,6 +407,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -383,6 +432,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -405,6 +457,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -427,6 +482,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -449,6 +507,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -471,6 +532,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra == 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -500,6 +564,10 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -522,6 +590,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -544,6 +615,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'linux' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -566,6 +640,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -588,6 +665,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -610,6 +690,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -632,6 +715,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -654,6 +740,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -676,6 +765,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -698,6 +790,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform != 'linux' and sys_platform != 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -720,6 +815,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'win32' and extra == 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -749,6 +847,10 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -771,6 +873,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -814,6 +919,12 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -836,6 +947,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -858,6 +972,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform == 'emscripten' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -901,6 +1018,12 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version == '3.13.*' and sys_platform != 'emscripten' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -944,6 +1067,12 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform != 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", "python_version < '0'", "python_version < '0'", @@ -966,6 +1095,9 @@ resolution-markers = [ "python_version < '0'", "python_version < '0'", "python_version < '0'", + "python_version < '0'", + "python_version < '0'", + "python_version < '0'", "python_full_version < '3.13' and sys_platform == 'win32' and extra != 'extra-16-surf-new-backend-cpu' and extra != 'extra-16-surf-new-backend-cu126' and extra != 'extra-16-surf-new-backend-cu128'", ] conflicts = [[ @@ -3421,24 +3553,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/18/79/1b8fa1bb3568781e84c9200f951c735f3f157429f44be0495da55894d620/filetype-1.2.0-py2.py3-none-any.whl", hash = "sha256:7ce71b6880181241cf7ac8697a2f1eb6a8bd9b429f7ad6d27b8db9ba5f1c2d25", size = 19970 }, ] -[[package]] -name = "firecrawl-py" -version = "4.21.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "aiohttp" }, - { name = "httpx" }, - { name = "nest-asyncio" }, - { name = "pydantic" }, - { name = "python-dotenv" }, - { name = "requests" }, - { name = "websockets" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/6d/6b/8201b737c0667bf70748b86a6fb117aefc648154b4e05c5ee649432cbc3d/firecrawl_py-4.21.0.tar.gz", hash = "sha256:14a7e0967d816c711c3c53325c9371e2f780a787d1e94333a34d8aea7a43a237", size = 174256 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/18/f1/1c0f1e5b33a318d7b9705b9e23c4397253d730e516e3d8a2f6aaea4b71a2/firecrawl_py-4.21.0-py3-none-any.whl", hash = "sha256:4e431f36117b4f2aaae633e747859a91626b0f2c6aaa6b7f86dfb7669a3595eb", size = 217607 }, -] - [[package]] name = "flashrank" version = "0.2.10" @@ -9868,7 +9982,6 @@ dependencies = [ { name = "fastapi" }, { name = "fastapi-users", extra = ["oauth", "sqlalchemy"] }, { name = "faster-whisper" }, - { name = "firecrawl-py" }, { name = "fractional-indexing" }, { name = "github3-py" }, { name = "gitingest" }, @@ -9986,7 +10099,6 @@ requires-dist = [ { name = "fastapi", specifier = ">=0.115.8" }, { name = "fastapi-users", extras = ["oauth", "sqlalchemy"], specifier = ">=15.0.3" }, { name = "faster-whisper", specifier = ">=1.1.0" }, - { name = "firecrawl-py", specifier = ">=4.9.0" }, { name = "fractional-indexing", specifier = ">=0.1.3" }, { name = "github3-py", specifier = "==4.0.1" }, { name = "gitingest", specifier = ">=0.3.1" },