diff --git a/docker/proxy/Caddyfile b/docker/proxy/Caddyfile index 534a8c2c2..abc5ff199 100644 --- a/docker/proxy/Caddyfile +++ b/docker/proxy/Caddyfile @@ -31,6 +31,10 @@ flush_interval -1 } + # Zero sync auth context is a backend (FastAPI) endpoint. More specific than + # /zero/*, so Caddy's matcher-specificity sort routes it here, not to zero-cache. + reverse_proxy /zero/context backend:8000 + # Zero accepts a single path-component base URL (Zero >= 0.6). # Preserve /zero so browser cacheURL can be ${SURFSENSE_PUBLIC_URL}/zero. reverse_proxy /zero/* zero-cache:4848 diff --git a/docker/scripts/install.ps1 b/docker/scripts/install.ps1 index 23b14c3c4..761fef80e 100644 --- a/docker/scripts/install.ps1 +++ b/docker/scripts/install.ps1 @@ -330,11 +330,13 @@ Write-Info "Installation directory: $InstallDir" New-Item -ItemType Directory -Path "$InstallDir\scripts" -Force | Out-Null New-Item -ItemType Directory -Path "$InstallDir\searxng" -Force | Out-Null +New-Item -ItemType Directory -Path "$InstallDir\proxy" -Force | Out-Null $Files = @( @{ Src = "docker/docker-compose.yml"; Dest = "docker-compose.yml" } @{ Src = "docker/docker-compose.gpu.yml"; Dest = "docker-compose.gpu.yml" } @{ Src = "docker/.env.example"; Dest = ".env.example" } + @{ Src = "docker/proxy/Caddyfile"; Dest = "proxy/Caddyfile" } @{ Src = "docker/postgresql.conf"; Dest = "postgresql.conf" } @{ Src = "docker/scripts/migrate-database.ps1"; Dest = "scripts/migrate-database.ps1" } @{ Src = "docker/searxng/settings.yml"; Dest = "searxng/settings.yml" } diff --git a/plans/backend/00-umbrella-plan.md b/plans/backend/00-umbrella-plan.md index fce43f137..060865f41 100644 --- a/plans/backend/00-umbrella-plan.md +++ b/plans/backend/00-umbrella-plan.md @@ -40,7 +40,7 @@ flowchart TD - MCP-availability audit complete: BookStack (community MCP servers), Elasticsearch (official Elastic Agent Builder MCP), and Luma (community MCP servers) all have MCP available, so none are `DISABLED` — they're tagged `MIGRATING` (turned off for MVP like the other branded natives, pending the post-MVP MCP re-point). - `Pipeline` and `PipelineRun` are new first-class tables. A Pipeline references a connector + config + schedule + KB destination. File upload creates/uses a pipeline and registers a run; uploads always save to KB. - The chat agent gets read-only access to pipeline run history (pipelines + their recent runs/status) as context, so it can reason about what was fetched, when, and whether runs succeeded — even for data not saved to the KB. -- Deferred (post-MVP): platform scraper implementations, public pay-as-you-go API for Type-1 connectors, public MCP server exposing the KB. +- Deferred (post-MVP): platform scraper implementations (**Phase 8 — Platform actors**, public-data-only first; built in-house on the Phase-3 fetch core), public pay-as-you-go API for Type-1 connectors, public MCP server exposing the KB. Logged-in/account-based scraping deferred beyond that. ## Platform connector research list (deferred build, MVP = "coming soon") @@ -69,14 +69,18 @@ flowchart TD - Consolidate API to `/workspaces` and fix the `/searchspaces` vs `/search-spaces` inconsistency. - High-touch files: `routes/search_spaces_routes.py`, `routes/rbac_routes.py`, `utils/rbac.py` (`check_search_space_access`), `schemas/search_space.py`, plus `search_space_id` threading through agents/Redis keys/storage paths (`documents/{id}/...`). -### Phase 3 — WebURL Crawler & Crawl Billing (backend) [`subplans: 03a–03d`] +### Phase 3 — WebURL Crawler & Crawl Billing (backend) [`subplans: 03a–03f`] -The Universal WebURL Crawler is the flagship Type-1 data source (the moat). This phase hardens it on a single framework (Scrapling), generalizes proxy support, introduces pay-as-you-go crawl credits, and (deferred) adds opt-in captcha solving. It is broken into focused subplans: +The Universal WebURL Crawler is the flagship Type-1 data source (**the moat**). This phase hardens it into an **in-house, best-effort "undetectable, captcha-bypassing" crawler** on a single framework (Scrapling): it standardizes the fetch layer, generalizes proxy support, introduces pay-as-you-go crawl credits, adds **stealth hardening** (geoip coherence, persistent profiles, headed/Xvfb, fonts, humanization, a block classifier + per-domain strategy memory), and adds **opt-in captcha solving**. All tiers plug in behind a single `FetchStrategy` seam returning `CrawlOutcome`, so callers never depend on *how* a page was fetched — that seam is what lets the moat grow (and lets a **deferred paid-unblocker tier** drop in later by config). **Strategy decision (recorded in the log):** CloakBrowser is rejected on licensing and external unblocker APIs are deferred; we hold an in-house bypass moat for ~4–6 months, then move hostile targets to a paid tier if demand/maintenance justifies it. **Logged-in/account-based bypass is out of scope** (public data only this MVP). It is broken into focused subplans: + +> **Sequencing within Phase 3 (critical path vs hardening).** Only **`03a` + `03b` + `03c`** are on the **MVP critical path** — Phases 4–7 (connector taxonomy, pipelines, execution) consume the `03a` `CrawlOutcome`/`FetchStrategy` contract, the `03b` proxy provider, and the `03c` `WebCrawlCreditService` billing seam. **`03d` (captcha), `03e` (stealth hardening), and `03f` (test harness) are hardening/measurement that nothing downstream imports** — they tune the *same* seam behind the *same* contract, so they can land **in parallel with, or after, Phases 4–7** without blocking the pivot. Recommended build order: `03a → 03b → 03c` (then proceed to Phase 4), with `03e → 03d → 03f` slotted in whenever crawler robustness is prioritized. The dependency notes inside `03d`/`03e`/`03f` (each requires `03a`/`03b`) still hold; this only frees them from gating Phase 4+. - **`03a-crawler-core.md`** — Standardize the fetch layer on Scrapling. **Remove Firecrawl entirely** (no other frameworks). Define crisp per-URL success/empty/failure semantics, keep Trafilatura extraction, and expose a single billable "successful crawl" signal (one unit per URL that yields usable content, regardless of how many internal fallback tiers ran). - **`03b-proxy-expansion.md`** — Add a BYO `CustomProxyProvider` (the only new provider — **no branded vendors**) alongside `anonymous_proxies`, selectable via a **single, app-wide** `Config.PROXY_PROVIDER`. Add bounded client-side rotation+retry via Scrapling's `ProxyRotator`/`is_proxy_error` **only** when the active provider is pool-backed (`CUSTOM_PROXY_URLS`); single-endpoint providers (incl. `anonymous_proxies`) stay the default and no-op the retry. **No per-connector/per-crawl selection** (one provider app-wide); a per-pipeline override is left as a no-op seam for Phase 5/6. - **`03c-crawl-billing.md`** — Charge crawl credits at **$1 / 1000 successful requests = 1000 micro-USD per successful crawl**, drawn from the existing credit wallet (`credit_micros_balance`), gated by a new `WEB_CRAWL_CREDIT_BILLING_ENABLED` flag (off for self-hosted). Two surfaces: **connector/pipeline crawls** billed to the **workspace owner** via a dedicated `WebCrawlCreditService` (mirrors `EtlCreditService`'s gate → `check_credits` → `charge_credits`, **not** `billable_call`); **chat scrapes** fold their crawl cost into the chat turn's existing bill (turn accumulator). No DB migration (uses the existing free-form `web_crawl` usage_type). -- **`03d-captcha-solving.md`** *(DEFERRED — sequenced last, non-MVP-blocking)* — Covers the captcha types Scrapling does **not** (reCAPTCHA v2/v3, hCaptcha, image) via `captchatools`. `captchatools` is **itself** the provider registry (`new_harvester(solving_site=…)` across capmonster/2captcha/anticaptcha/capsolver/captchaai), so we do **not** rebuild a provider hierarchy — our layer is thin: config resolution + a StealthyFetcher `page_action` that detects the sitekey, harvests a token, and injects it. Scrapling already handles Cloudflare Turnstile (`03a`). Flags the **billing asymmetry** (solvers charge per *attempt*, `03c` bills per *success*) for resolution at build time. Requires a paid solver account. +- **`03e-stealth-hardening.md`** — The in-house "undetectable" layer on top of Scrapling's default patchright-Chromium stealth. **Geoip coherence** (match browser `locale`/`timezone_id` to the proxy's exit geo), **fingerprint flags** (`hide_canvas`/`block_webrtc`), **persistent per-domain profiles** (`user_data_dir`), **headed execution under Xvfb**, **real fonts** in the worker image, and **DIY behavioral humanization** via `page_action` (the Chromium engine has no built-in `humanize`). Adds a **block classifier** (label Cloudflare/DataDome/Kasada/captcha/empty from the response) + **per-domain strategy memory** (Redis, no migration) so the ladder learns the known-good tier per domain. Defines — but does **not** build — the **deferred paid-unblocker `FetchStrategy`** (ZenRows/ScrapFly/Bright Data) as the config-flagged escape hatch, with its own (later) billing. Honest ceiling: defeats Cloudflare + the moderate long tail, **not** top-tier behavioral fingerprinting (DataDome/Kasada/reCAPTCHA-Enterprise) — that's the deferred tier. +- **`03d-captcha-solving.md`** *(ACTIVE — sequenced last in Phase 3, after `03e`)* — Covers the captcha types Scrapling does **not** (reCAPTCHA v2/v3, hCaptcha, image) via `captchatools`, **opt-in + off by default**. `captchatools` is **itself** the provider registry (`new_harvester(solving_site=…)` across capmonster/2captcha/anticaptcha/capsolver/captchaai), so we do **not** rebuild a provider hierarchy — our layer is thin: config resolution + a StealthyFetcher `page_action` that detects the sitekey, harvests a token (egressing from the **same** proxy IP as the crawl), and injects it. Scrapling already handles Cloudflare Turnstile (`03a`), and `page_action` runs **after** `solve_cloudflare`, so the tiers compose. The **billing asymmetry** is now **resolved**: a **separate per-attempt** `web_crawl_captcha` unit (solvers charge per attempt regardless of crawl success), attached via a `WebCrawlCreditService` seam in `03c`. `ErrNoBalance` stops solving (no retry loop → avoids IP bans). Requires a paid solver account. +- **`03f-undetectability-testing.md`** *(MANUAL-only — no CI gating; sequenced last)* — A **manual scorecard harness** that drives the real Scrapling tiers against the industry-standard detection + sandbox sites (modeled on CloakBrowser's `bin/cloaktest` suite) to **quantify the free-stack ceiling** over time. Two labeled axes: **Suite S (stealth/anti-bot)** — browser-tier (`bot.sannysoft`, `bot.incolumitas`, CreepJS, `deviceandbrowserinfo`, FingerprintJS demo, reCAPTCHA-v3 score, `fingerprint-scan`/Castle.js, `browserscan`), HTTP/TLS-tier JA3/JA4 parity (`tls.peet.ws`, **informational** not a gate), and proxy/leak checks (`httpbin/ip`, WebRTC/DNS); **Suite E (extraction correctness)** — toscrape/scrapethissite sandboxes for the HTTP vs JS (DynamicFetcher) tiers. Reuses `03d`'s `page_action`+closure-cell for JS-object verdicts. Adopts CloakBrowser's bars as **aspirational** (sannysoft 0 fails, CreepJS ≤30%, reCAPTCHA ≥0.7) while recording **our actual numbers as the committed baseline**; the scorecard is the documented **trigger** for flipping `03e`'s deferred paid-unblocker tier. ### Phase 4 — Connector two-type restructure (backend) [`subplans: 04a–04b`] @@ -114,6 +118,13 @@ Split into two independently testable subplans: - A small migration adds a partial unique index `ON pipelines(workspace_id) WHERE connector_id IS NULL` (enforces the singleton + race-safe get-or-create). - KB-save-secondary: the opt-in `save_to_kb` + destination folder for connector pipelines already shipped in Phases 5–6; Phase 7 only records the inverse invariant (uploads always KB) and guards the connector default stays `False`. +### Phase 8 — Platform actors (FUTURE — post-MVP, public data only) [`subplan: TBD`] + +NOT planned in this umbrella; recorded so the Phase-3 architecture stays aimed at it. Once the hardened fetch core (Phases 3a–3e) is solid, **platform actors** layer **on top** of it: per-platform structured extractors (Google Maps/Local, LinkedIn public profiles/companies, Amazon products, etc. — see "Platform connector research list" above), built in-house "Apify-style" rather than via third-party paid actors. They reuse the existing machinery end-to-end: each actor is a **Type-1 data-source connector** (Phase 4 taxonomy), runs via **Pipelines** (Phases 5–6, manual/cron), saves to the **KB** (opt-in folder), and is **billed per run** (extending `03c`/`06`). They consume the `03a` `FetchStrategy` core (proxies + `03e` hardening + `03d` captcha) under their own extractors. + +- **Public data only** at first — discovery/extraction of publicly visible pages. **Logged-in/account-based bypass is explicitly deferred** beyond Phase 8's first cut; it needs sticky/static proxies + credential management (`03b` static-proxy hand-off) and is the higher-risk, later workstream. +- The **deferred paid-unblocker tier** (`03e §8`) is the fallback for any platform whose anti-bot exceeds the in-house ceiling. + ## Deferred — Frontend & client phases (separate umbrella, planned LATER) These are recorded for continuity but are NOT planned in this umbrella. They start once the backend phases above are working. @@ -140,10 +151,14 @@ These are recorded for continuity but are NOT planned in this umbrella. They sta - Phase 4 search APIs: all 5 enum values dropped (`SERPER_API`/`TAVILY_API`/`SEARXNG_API`/`LINKUP_API`/`BAIDU_SEARCH_API`) in 04b. Survivors (SearXNG/Linkup/Baidu) become PLATFORM providers keyed by env (Linkup/Baidu keys move from per-connector `config` to env — app-wide, not per-workspace). 04b carries a destructive migration deleting the 5 connector types' rows. - Phase 4 structure: split into 04a (taxonomy/gating/MCP-fix, no migration) and 04b (search repurposing + source-discovery endpoint, with migration); intended order 04a -> 04b (both orders safe). - Rename transition policy: HARD CUTOVER of the external API (paths + JSON field names) in Phase 2 — no backward-compat aliases. Rationale: the frontend is (re)built against the corrected backend later, so there is no old client to keep alive; backend correctness is verified via the test suite + OpenAPI rather than the existing UI. -- WebURL Crawler framework: STANDARDIZE on Scrapling; **remove Firecrawl entirely** (no other scraping frameworks now or planned). Scrapling's `StealthyFetcher` handles Cloudflare; captcha-tools (deferred) covers the rest. +- WebURL Crawler framework: STANDARDIZE on Scrapling; **remove Firecrawl entirely** (no other scraping frameworks now or planned). Scrapling's `StealthyFetcher` (patchright-Chromium as of 0.4.9 — **not** Camoufox) handles Cloudflare; `03e` stealth-hardening minimizes challenges; captcha-tools (`03d`) covers the rest. All fetch tiers sit behind a `FetchStrategy` seam returning `CrawlOutcome` (callers never depend on the tier). - Crawl billing: reuse the existing credit wallet (`credit_micros_balance`) with a new `web_crawl` usage_type. Price: **$1 / 1000 successful requests** (1000 micro-USD per success). Connector/pipeline crawls bill the **workspace owner**; chat scrapes fold their crawl cost into the already-billed chat turn. Gated by `WEB_CRAWL_CREDIT_BILLING_ENABLED` (off for self-hosted); no DB migration required. - Billable unit: one unit per URL that returns usable extracted content, regardless of how many internal fallback tiers were attempted (not per HTTP fetch, not per URL-processed). -- Captcha solving (captcha-tools): DEFERRED to the last Phase-3 subplan (`03d`); non-MVP-blocking. +- Captcha solving (captcha-tools): **ACTIVE** (no longer deferred) — sequenced last in Phase 3 (`03d`), **after** `03e` hardening, **opt-in + off by default**. Cloudflare stays in-framework (`03a`); reCAPTCHA/hCaptcha/image use `captchatools`. **Billing asymmetry RESOLVED → option (a): a separate per-attempt `web_crawl_captcha` unit** (`WEB_CRAWL_CAPTCHA_*` knobs on `WebCrawlCreditService`), since solvers charge per attempt regardless of crawl success. `ErrNoBalance` halts solving (no retry-loop IP bans). +- Crawler stealth strategy (the moat): **CloakBrowser REJECTED** (source-patched Chromium binary requires an OEM/SaaS license incompatible with our model). **External unblocker APIs DEFERRED** (ZenRows/ScrapFly/Bright Data) — pre-wired as a config-flagged `FetchStrategy` (`03e §8`) but not built. Plan: maintain an **in-house bypass moat for ~4–6 months** (Scrapling stealth + residential proxies + `03e` hardening + `03d` captcha), then move hostile/top-tier-fingerprinted targets (DataDome/Kasada/reCAPTCHA-Enterprise) to a paid tier if demand/maintenance justifies it. Realistic ceiling acknowledged: in-house beats Cloudflare + the moderate long tail, not top-tier behavioral fingerprinting. +- Authenticated/logged-in scraping: **OUT OF SCOPE this MVP** (public data only). Sticky/static proxies + credential management are deferred and paired with the future platform actors (`03b` static-proxy hand-off + Phase 8). +- Phase 3 stealth-hardening subplan `03e` ADDED: geoip locale/tz coherence, `hide_canvas`/`block_webrtc`, persistent per-domain profiles, headed+Xvfb, fonts, DIY humanization (`page_action`; Chromium engine has no built-in `humanize`), a block classifier, and per-domain strategy memory (Redis, no migration). +- Phase 3 test-harness subplan `03f` ADDED: **manual-only** (no CI/automated gating now) undetectability + extraction scorecard, modeled on CloakBrowser's `bin/cloaktest`. Two labeled axes (Suite S stealth + Suite E extraction) so they scale independently. Drives the **real** Scrapling tiers (browser + curl_cffi HTTP/TLS), reuses `03d`'s `page_action`+closure-cell for JS-object verdicts. **TLS JA3/JA4 parity = informational axis, not a hard gate.** Adopt CloakBrowser bars as aspirational; record our actual free-stack numbers as the committed baseline. The scorecard is the documented evidence/trigger for flipping `03e`'s deferred paid-unblocker tier. - Roadmap: WebURL Crawler & Crawl Billing inserted as the new Phase 3; connector two-type → Phase 4; pipelines → Phases 5/6/7. - Phase 5 pipelines data model: two new tables `pipelines` (mutable) + `pipeline_runs` (append-only), modeled on `automations`/`automation_runs`; ORM lives in `db.py` next to connectors/folders. `connector_id` nullable (NULL = Phase-7 Uploads), eligibility enforced at create via 04a's `is_pipeline_eligible`. Schedule = `schedule_cron` + `schedule_timezone` (default UTC) + `next_scheduled_at` (cron, matching automations). `pipeline_runs` pre-includes `charged_micros`/`crawls_*`/`result_blob_key` so Phase 6 needs no extra migration. Both tables published to Zero **full-row** (like folders/connectors). Routes reuse `CONNECTORS_*` permissions. Phase 5 ships the data model + API surface only; the `/run` endpoint enqueues a Phase-6 task stub. - Phase 7 uploads-as-pipeline: a **singleton "Uploads" pipeline** per workspace (`connector_id NULL`, `save_to_kb=true`), lazily get-or-created (race-safe via a partial unique index `ON pipelines(workspace_id) WHERE connector_id IS NULL`). Each `fileupload`/`folder-upload` request writes a **terminal audit `PipelineRun(trigger=upload, status=succeeded, documents_indexed=)`** — uploads are **route-recorded, not engine-executed** (Phase 6 fails NULL-connector runs by design; existing upload code stays the executor). Best-effort via an **inner** try/except (never 5xx the upload — the route's outer handler would otherwise 500 an already-committed upload). No crawl billing (uploads aren't crawls; `charged_micros` NULL). Per-file ETL truth stays on `Document.status`; accurate roll-up needs the deferred `documents.pipeline_run_id` provenance. Connector `save_to_kb` default stays `False` (opt-in for connectors, mandatory for uploads). Phase 7 also **guards Phase-5's generic CRUD** against the system Uploads pipeline: `POST /pipelines` rejects `connector_id=None` (supersedes Phase 5's permissive create), `/run` and schedule-`PUT` reject NULL-connector pipelines, and Phase 6's scheduler `_claim_due` filters `connector_id IS NOT NULL` as a backstop (so the Uploads pipeline can never be manually-run or scheduled into perpetually-failing runs). See `07-upload-pipeline-kb.md`. @@ -158,7 +173,9 @@ These are recorded for continuity but are NOT planned in this umbrella. They sta | 3 | `03a-crawler-core.md` | drafted | | 3 | `03b-proxy-expansion.md` | drafted | | 3 | `03c-crawl-billing.md` | drafted | -| 3 | `03d-captcha-solving.md` | drafted (deferred — last) | +| 3 | `03e-stealth-hardening.md` | drafted | +| 3 | `03d-captcha-solving.md` | drafted (active — sequenced after 03e) | +| 3 | `03f-undetectability-testing.md` | drafted (manual scorecard — sequenced last) | | 4 | `04a-connector-category.md` | drafted | | 4 | `04b-source-discovery.md` | drafted | | 5 | `05-pipelines-model.md` | drafted | diff --git a/plans/backend/03a-crawler-core.md b/plans/backend/03a-crawler-core.md index afa96e368..3c43c44e8 100644 --- a/plans/backend/03a-crawler-core.md +++ b/plans/backend/03a-crawler-core.md @@ -1,7 +1,7 @@ # Phase 3a — WebURL Crawler core (Scrapling-only) + success semantics > Part of **Phase 3 — WebURL Crawler & Crawl Billing**. See `00-umbrella-plan.md`. -> Sibling subplans: `03b-proxy-expansion.md`, `03c-crawl-billing.md`, `03d-captcha-solving.md` (deferred). +> Sibling subplans: `03b-proxy-expansion.md`, `03c-crawl-billing.md`, `03e-stealth-hardening.md`, `03d-captcha-solving.md`, `03f-undetectability-testing.md` (manual scorecard). > **Implementation note (applies to all Phase-3 plans).** Phase 3 lands **after** Phases 1–2, which rename `SearchSpace`→`Workspace` and `search_space_id`→`workspace_id` everywhere. Citations below use **today's** names so they stay greppable against current code; when implementing, the live code will already say `workspace_*` — map accordingly. Apply every edit by **symbol/grep** (e.g. `firecrawl_api_key`, `crawl_url`, `FIRECRAWL_API_KEY`), **not** by the absolute line numbers cited here — Phase 2's rename (and `03a`'s own Firecrawl removal) shift line numbers. @@ -11,7 +11,7 @@ Make the Universal WebURL Crawler a **single-framework (Scrapling) component** w Two hard requirements from the decisions log: -1. **Remove Firecrawl entirely.** No other scraping framework now or planned. Scrapling's `StealthyFetcher` can bypass Cloudflare Turnstile when invoked with `solve_cloudflare=True` (see tier design below); captcha-tools (`03d`, deferred) covers the rest. +1. **Remove Firecrawl entirely.** No other scraping framework now or planned. Scrapling's `StealthyFetcher` can bypass Cloudflare Turnstile when invoked with `solve_cloudflare=True` (see tier design below); captcha-tools (`03d`) covers reCAPTCHA/hCaptcha, and `03e` adds the broader stealth-hardening that makes the crawler "undetectable" as far as free tooling reaches. 2. **One billable unit = one URL that yields usable extracted content**, regardless of how many internal fallback tiers ran. `03a` must expose that signal; `03c` meters it. This subplan does NOT touch proxy rotation (→ `03b`), credit metering (→ `03c`), or captcha (→ `03d`). @@ -25,7 +25,11 @@ This subplan does NOT touch proxy rotation (→ `03b`), credit metering (→ `03 1. **Firecrawl** (premium, if `firecrawl_api_key` set) — `_crawl_with_firecrawl()` (lines 89–100, 223–272), imports `from firecrawl import AsyncFirecrawlApp` (line 22). 2. Scrapling `AsyncFetcher` (static HTTP, curl_cffi) — `_crawl_with_async_fetcher()` (lines 274–310). 3. Scrapling `DynamicFetcher` (browser, run in a thread) — `_crawl_with_dynamic()` (lines 312–339). -4. Scrapling `StealthyFetcher` (Camoufox anti-bot, run in a thread) — `_crawl_with_stealthy()` (lines 341–369). +4. Scrapling `StealthyFetcher` (patchright-Chromium anti-bot, run in a thread) — `_crawl_with_stealthy()` (lines 341–369). + +> **Engine note (do not say "Camoufox").** As of the pinned `scrapling[fetchers]>=0.4.9` (`pyproject.toml:91`), `StealthyFetcher` is **"completely stealthy built on top of Chromium"** (`references/Scrapling/scrapling/fetchers/stealth_chrome.py:8`) driven by **patchright** (`references/Scrapling/scrapling/engines/_browsers/_stealth.py:8–9`), **not** Camoufox — Scrapling removed Camoufox (zero matches in the 0.4.9 tree; `uv.lock` carries `patchright`, no `camoufox`). The default Playwright `channel` is `"chromium"` (patchright), or `"chrome"` when `real_chrome=True` (`_browsers/_base.py:469`). Stealth = the compiled-in flag set `DEFAULT_ARGS + STEALTH_ARGS` (`engines/constants.py:24–99`, incl. `--disable-blink-features=AutomationControlled` `:94`) + a persistent context by default (`_stealth.py:90–93`) — this is **runtime/config-level** stealth, which sets the realistic ceiling (see `03e`). +> +> **Scrub the in-code "Camoufox" mentions too.** The connector still carries stale Camoufox wording in its own docstrings — the module header tier list (`webcrawler_connector.py:10–12`) and `_crawl_with_stealthy`'s docstring (`:343` "StealthyFetcher (Camoufox)"). Fix these in the 03a refactor so the code matches reality (patchright-Chromium). Extraction is Trafilatura HTML→markdown in `_build_result()` (lines 371–469). Every Scrapling tier passes `proxy=get_proxy_url()` (lines 287, 329, 359). `crawl_url()` returns a tuple `(result_dict | None, error | None)`; `result_dict` has `content` / `metadata` / `crawler_type`. @@ -57,7 +61,7 @@ Firecrawl's API key is plumbed end-to-end and must be removed everywhere: ### Runtime/deps already in place -- `Dockerfile:112–115` runs `RUN scrapling install` to fetch patchright Chromium + Camoufox; the `scrapling[fetchers]` extra pulls playwright/patchright. **No new install step needed** once Firecrawl is gone. +- `Dockerfile:112–115` runs `RUN scrapling install`; the `scrapling[fetchers]` extra pulls playwright/patchright. **No new install step needed** once Firecrawl is gone. *(Accuracy fix: the Dockerfile comment says "patchright Chromium + Camoufox", but `scrapling install` in 0.4.9 only fetches Chromium — `references/Scrapling/scrapling/cli.py:122,131` runs `playwright install chromium` + `install-deps chromium`, no Camoufox. Drop the stale "+ Camoufox" wording from the comment when touching this file. `03e` may add `install-deps` extras for fonts/Xvfb.)* - Proxy is read via `app/utils/proxy/get_proxy_url()` (`__init__.py:13`), backed by the `PROXY_PROVIDER` registry (`config/__init__.py:983`). `03a` leaves this single-URL model untouched (rotation is `03b`). ## Target design @@ -66,9 +70,9 @@ Firecrawl's API key is plumbed end-to-end and must be removed everywhere: `crawl_url()` becomes a 3-tier ladder, preserving the existing thread-offload + `NotImplementedError` handling for the browser tiers (Windows `SelectorEventLoop` cannot spawn subprocesses — lines 134–141, 161–168): -1. `AsyncFetcher.get(...)` — fast static HTTP. +1. `AsyncFetcher.get(...)` — fast static HTTP. **TLS gap to close:** the current call passes `stealthy_headers=True` but **not** `impersonate` (`webcrawler_connector.py:284–289`), so its TLS ClientHello is curl_cffi's *default* JA3 — trivially bot-flagged and incoherent with the browser tiers' UA. Add an `impersonate="chrome"` profile here (Scrapling's static engine accepts it — `references/Scrapling/scrapling/engines/static.py:36–47`). Tracked as a lever in `03e §2b` and validated by `03f §S3`. 2. `DynamicFetcher.fetch(...)` — full browser (via `asyncio.to_thread`). -3. `StealthyFetcher.fetch(...)` — Camoufox anti-bot, last resort. Enable Cloudflare solving here by passing **`solve_cloudflare=True`** — a documented `StealthyFetcher.fetch` kwarg ("Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response", `references/Scrapling/scrapling/fetchers/stealth_chrome.py:38`; it's a `StealthSession` TypedDict key passed via `**kwargs`). The current stealthy call (connector lines 354–360) passes `headless`/`network_idle`/`block_ads`/`proxy` but **not** `solve_cloudflare`, so this is a real behavior add. (Note: `solve_cloudflare` runs the full browser challenge loop, so it's correctly scoped to the last-resort tier only.) +3. `StealthyFetcher.fetch(...)` — patchright-Chromium anti-bot, last resort. Enable Cloudflare solving here by passing **`solve_cloudflare=True`** — a documented `StealthyFetcher.fetch` kwarg ("Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response", `references/Scrapling/scrapling/fetchers/stealth_chrome.py:38`; it's a `StealthSession` TypedDict key passed via `**kwargs`). The current stealthy call (connector lines 354–360) passes `headless`/`network_idle`/`block_ads`/`proxy` but **not** `solve_cloudflare`, so this is a real behavior add. (Note: `solve_cloudflare` runs the full browser challenge loop, so it's correctly scoped to the last-resort tier only.) Trafilatura extraction (`_build_result`) and `format_to_structured_document()` are unchanged. @@ -83,7 +87,7 @@ class CrawlOutcomeStatus(str, Enum): FAILED = "failed" # invalid URL or every tier errored ``` -`crawl_url()` returns a small dataclass `CrawlOutcome(status, result, error, tier)` — **commit to the dataclass** (not a tuple): `03c` keys billing off `status == SUCCESS`, and Phase 6's fetch-only path (`06-pipelines-exec.md`) consumes `outcome.status` / `outcome.result` / `outcome.error` as attributes, so a tuple form would break that consumer. The **billable success predicate is single-sourced**: `status == CrawlOutcomeStatus.SUCCESS`. +`crawl_url()` returns a small dataclass `CrawlOutcome(status, result, error, tier)` — **commit to the dataclass** (not a tuple): `03c` keys billing off `status == SUCCESS`, and Phase 6's fetch-only path (`06-pipelines-exec.md`) consumes `outcome.status` / `outcome.result` / `outcome.error` as attributes, so a tuple form would break that consumer. The **billable success predicate is single-sourced**: `status == CrawlOutcomeStatus.SUCCESS`. Picking a dataclass (over a tuple) also leaves room for later subplans to **append fields without breaking callers** — `03d` adds `captcha_attempts` / `captcha_solved` for per-attempt billing, and `03e`'s block classifier can attach a `block_type`. (This is distinct from the indexer's positional return, which must stay 2-tuple — see the wrapper note below.) | Outcome | When | Billable (`03c`)? | Document status (indexer) | |---------|------|-------------------|---------------------------| @@ -93,6 +97,17 @@ class CrawlOutcomeStatus(str, Enum): **Billing-policy note for `03c`:** success is the *crawl* succeeding (we fetched + extracted), independent of downstream KB dedupe. The indexer currently marks unchanged content as `skipped` (`webcrawler_indexer.py:341–347`) and cross-connector duplicates as `failed` (`:350–369`) — those still represent a **successful crawl** and should bill. `03c` must count `CrawlOutcomeStatus.SUCCESS`, not `documents_indexed`. Flagging here; final call lives in `03c`. +### Extensibility seam (the tier ladder is a strategy chain) + +The 3-tier ladder is the first instance of a deliberate **`FetchStrategy` seam**: an *ordered list of strategies*, each `(url, ctx) -> CrawlOutcome`, tried in order until one returns `SUCCESS`. `crawl_url()` owns the chain; **every caller depends only on `CrawlOutcome`, never on which strategy produced it** (the indexer, the chat tool, and `03c` metering already do — keep that invariant sacred). This is what lets the moat grow without rework: + +- `03d` (captcha) attaches by escalating to the StealthyFetcher strategy with a `page_action` token-injector — a *parameterization* of the last tier, not a new caller contract. +- `03e` (stealth-hardening) tunes/adds strategies (humanize, headed/Xvfb, persistent profiles, fingerprint flags) **behind the same return type**. +- A future **paid-unblocker tier** (deferred — see `03e`) is just one more strategy appended last, flippable by config; no caller changes. +- Future **platform actors** (Phase 8) reuse the same fetch strategies under their own structured extractors. + +MVP scope here is only the 3 Scrapling tiers + the `CrawlOutcome` contract; the seam is a design constraint (keep tiers pluggable + callers outcome-only), **not** a call for a heavyweight Strategy framework now. + ### Success counter for the indexer Add an explicit `crawls_succeeded` counter in `index_crawled_urls()` incremented whenever `crawl_url` returns `SUCCESS` (right after line 297's call, before the dedupe/unchanged branches), and surface it in the task-success metadata (lines 455–466). `03c` meters against this **in-function** counter (it charges inside the indexer — the count does not need to escape via the return). @@ -113,7 +128,7 @@ Drop the Firecrawl-only `formats=["markdown"]` arg (markdown is already the Traf ## Risks / trade-offs -- **Loss of a managed fallback.** Firecrawl was a hosted last resort for hostile anti-bot sites. Mitigation: `StealthyFetcher` + Cloudflare solving now, `03b` proxy rotation, `03d` captcha solving. Acceptable per the decisions log (single-framework intent). +- **Loss of a managed fallback.** Firecrawl was a hosted last resort for hostile anti-bot sites. Mitigation: the in-house stack — `StealthyFetcher` + Cloudflare solving (this plan), `03b` proxy rotation, `03e` stealth-hardening (humanize/headed/profiles/fonts + block-classifier), `03d` captcha solving — plus a **deferred paid-unblocker tier** behind the seam for the hostile residual. Realistic ceiling: this defeats Cloudflare + the long tail of moderate anti-bot, but runtime-level (patchright) stealth does **not** reliably beat top-tier fingerprinting (DataDome/Kasada/reCAPTCHA-Enterprise) — that's the deferred paid tier's job (`03e`). Acceptable per the decisions log (single-framework intent + in-house moat). - **Browser tiers on dev/Windows.** `DynamicFetcher`/`StealthyFetcher` need subprocess support; the existing `to_thread` + `NotImplementedError` guards (lines 134–141, 161–168) are preserved so static-only crawling still works in `uvicorn --reload`. - **`uv.lock` churn.** Removing `firecrawl-py` requires a lockfile regen + image rebuild; no new runtime deps are added. @@ -121,5 +136,6 @@ Drop the Firecrawl-only `formats=["markdown"]` arg (markdown is already the Traf - Proxy provider expansion + rotation → `03b`. - Crawl credit metering on `CrawlOutcomeStatus.SUCCESS` → `03c`. -- reCAPTCHA/hCaptcha solving via captcha-tools → `03d` (deferred). Cloudflare Turnstile stays in-framework (Scrapling). +- reCAPTCHA/hCaptcha solving via captcha-tools → `03d` (**now active**, sequenced after `03e`). Cloudflare Turnstile stays in-framework (Scrapling). +- Stealth-hardening (humanize, headed/Xvfb, persistent profiles, fonts, geoip locale/tz, block-classifier + per-domain strategy memory) and the deferred paid-unblocker tier → `03e`. - Whether ad-hoc **chat** scrapes are billed (vs only pipeline crawls) → decided in `03c`. diff --git a/plans/backend/03b-proxy-expansion.md b/plans/backend/03b-proxy-expansion.md index abf20b547..35379cebb 100644 --- a/plans/backend/03b-proxy-expansion.md +++ b/plans/backend/03b-proxy-expansion.md @@ -1,7 +1,7 @@ # Phase 3b — Proxy provider expansion + rotation > Part of **Phase 3 — WebURL Crawler & Crawl Billing**. See `00-umbrella-plan.md`. -> Depends on `03a-crawler-core.md` (Scrapling-only crawler). Siblings: `03c-crawl-billing.md`, `03d-captcha-solving.md` (deferred). +> Depends on `03a-crawler-core.md` (Scrapling-only crawler). Siblings: `03c-crawl-billing.md`, `03e-stealth-hardening.md`, `03d-captcha-solving.md`. > **Implementation note.** Same convention as `03a`: citations use **today's** names (`search_space_id`/`SearchSpace`) — map to `workspace_id`/`Workspace` post Phases 1–2. Crucially, the `webcrawler_connector.py` line numbers below (e.g. the three tiers at 287/329/359) and the `scrape_webpage.py` lines **predate `03a`'s Firecrawl removal + `crawl_url` refactor**, so they will have moved by the time `03b` is implemented. Locate code by **symbol/grep**, not absolute lines. @@ -123,4 +123,6 @@ Because there is one global provider and rotation lives **inside** it, the crawl - Branded-vendor provider subclasses → not planned (use `CustomProxyProvider`). - **Static / sticky-session proxies (future).** A later capability will add **static proxy** support — sticky IPs held for the duration of a session — most likely paired with **authenticated/account-based scraping** to bypass logged-in platforms (the deferred platform connectors: LinkedIn, Instagram, etc.). This is a *different axis* from the rotating pool here: rotation maximizes IP diversity, whereas account bypass needs IP **stability** so a session/cookie stays bound to one IP. It is additive to this design — a new `ProxyProvider` (or a "sticky" mode/flag on `CustomProxyProvider`) registered under a new `PROXY_PROVIDER` key, with no change to the zero-arg getter contract — and stays consistent with the single-provider model (the active provider would be the static one when that workflow is selected). Build it alongside the platform connectors, not in Phase 3. - Crawl credit metering (proxy cost is absorbed into the flat `$1 / 1000 successful` price, **not** metered separately) → `03c`. -- Captcha solving → `03d` (deferred). +- Captcha solving → `03d`. +- **Geoip fingerprint coherence** (matching browser `locale`/`timezone_id` to the proxy's exit geo) → `03e`. 03b only owns *selecting* the proxy; `03e` consumes the chosen endpoint's geo. The provider's `RESIDENTIAL_PROXY_LOCATION` (`config/__init__.py`) is one input; resolving the actual exit IP's geo is `03e`'s job. +- **Surfacing the crawl's chosen endpoint into the strategy.** Both `03d` (IP-bound captcha solves) and `03e` (geoip + sticky reuse) need the **exact** endpoint a crawl used, *not* a fresh `get_proxy_url()` call (which rotates on a pool-backed `CustomProxyProvider`). The capture-once seam lives in `03a`'s `FetchStrategy` context; 03b only guarantees the rotating getter and notes the consumers. diff --git a/plans/backend/03c-crawl-billing.md b/plans/backend/03c-crawl-billing.md index 0ea6a01cd..9e30d8233 100644 --- a/plans/backend/03c-crawl-billing.md +++ b/plans/backend/03c-crawl-billing.md @@ -1,7 +1,7 @@ # Phase 3c — Crawl credit billing ($1 / 1000 successful requests) > Part of **Phase 3 — WebURL Crawler & Crawl Billing**. See `00-umbrella-plan.md`. -> Depends on `03a-crawler-core.md` (the `CrawlOutcomeStatus.SUCCESS` signal + `crawls_succeeded` counter). Sibling: `03b-proxy-expansion.md`. `03d` is deferred. +> Depends on `03a-crawler-core.md` (the `CrawlOutcomeStatus.SUCCESS` signal + `crawls_succeeded` counter). Siblings: `03b-proxy-expansion.md`, `03e-stealth-hardening.md`, `03d-captcha-solving.md` (now active). `03d` extends this service with a **separate per-attempt captcha unit** (see §"Captcha billing seam" + `03d §4`). ## Objective @@ -49,6 +49,8 @@ New `app/services/web_crawl_credit_service.py` — a dedicated service (independ A separate `CreditMeterService` generalization is deliberately deferred until a third per-unit biller appears — exposing the price/flag as statics already lets the chat surface share the math without coupling the two flows. +**Captcha billing seam (for `03d`).** `03d` (now active) adds a **separate per-attempt** captcha unit to this same service — `captcha_solves_to_micros(n) = n * config.WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE`, gated by `WEB_CRAWL_CAPTCHA_BILLING_ENABLED` (both new knobs, default FALSE), with `usage_type="web_crawl_captcha"`. It is **independent** of crawl SUCCESS because solvers charge per *attempt* regardless of crawl outcome (the asymmetry in the §"Risks" proxy/captcha note). Build the `WebCrawlCreditService` statics so `03d` can add one more `*_to_micros` static + one charge path without restructuring — no need to implement the captcha methods in `03c`, just leave the shape open. + ### 2. Wiring in `webcrawler_indexer.py` 1. **Resolve the owner.** Bill the **workspace owner** (the product concept), not the triggering `user_id`: `owner_user_id = (select SearchSpace.user_id where id == search_space_id)`. (Mirrors how `billable_calls._resolve_agent_billing_for_search_space` (func `:441`) reads `search_space.user_id` at `billable_calls.py:499` — but we only need the owner id, so a direct `select` is enough; no need to call that LLM-model-resolving helper.) @@ -117,7 +119,8 @@ One unit per `CrawlOutcomeStatus.SUCCESS` — a URL that yielded usable extracte - **Session coupling.** `charge_credits` commits the indexer session; placing it after the final doc commit avoids entangling a debit with mid-run document state. - **Gating asymmetry (intended).** The **indexer** path bills whenever `WEB_CRAWL_CREDIT_BILLING_ENABLED` is on and the owner has credit — no "premium tier" requirement (matches ETL, which gates only on its own flag). The **chat** path additionally requires the turn to be premium, because it piggybacks the turn's reserve/finalize machinery. Same price, two different gate sets — call this out in the credit-status UI copy (frontend phase). - **No cross-surface double charge.** A chat scrape calls `crawl_url` directly and never enters `webcrawler_indexer`, so a given crawl is billed by exactly one surface. -- **Proxy/captcha cost.** Absorbed into the flat $1/1000 (margin), **not** metered separately (see `03b`, and `03d` for captcha which has its own per-solve upstream cost — revisit there). +- **Proxy cost.** Absorbed into the flat $1/1000 (margin), **not** metered separately (see `03b`). +- **Captcha cost.** **Metered separately** as a per-attempt unit (`03d §4`, decided: option (a)) because the solver charges per attempt regardless of crawl success — it cannot ride the per-success flat rate. The `captcha_solves_to_micros` seam above is where it attaches. ## Resolved decisions (this pass) @@ -128,5 +131,5 @@ One unit per `CrawlOutcomeStatus.SUCCESS` — a URL that yielded usable extracte ## Out of scope (hand-offs) - Per-**pipeline** run accounting / `charged_micros` persistence for idempotency → Phases 5–7. Specifically, **Phase 6 moves pipeline-run billing out of the indexer to the run level**: it adds a `bill: bool = True` param to this indexer wiring and calls `index_crawled_urls(bill=False)` from the pipeline engine, which then does its own `check_credits` + `charge_credits(crawls_succeeded)` and stamps `PipelineRun.charged_micros` (idempotency). So this phase's in-indexer billing keeps serving the **connector `/index` + periodic** paths; the "pipeline crawls" half of the objective is refined by `06`. Structure the §2 wiring so it's easy to gate behind that flag. -- Captcha-solver upstream cost pass-through → `03d` (deferred). +- Captcha-solver upstream cost pass-through → `03d` (now active; separate per-attempt `web_crawl_captcha` unit via the seam above). - Surfacing crawl spend in the credit-status UI → frontend umbrella. diff --git a/plans/backend/03d-captcha-solving.md b/plans/backend/03d-captcha-solving.md index e2dc92773..7a95789fd 100644 --- a/plans/backend/03d-captcha-solving.md +++ b/plans/backend/03d-captcha-solving.md @@ -1,11 +1,16 @@ -# Phase 3d — Captcha solving (DEFERRED — sequenced last, non-MVP-blocking) +# Phase 3d — Captcha solving (reCAPTCHA / hCaptcha / image) > Part of **Phase 3 — WebURL Crawler & Crawl Billing**. See `00-umbrella-plan.md`. -> **Status: deferred.** Build only after `03a`/`03b`/`03c` ship and there's a real need. Depends on `03a` (the StealthyFetcher tier) and the app-wide proxy provider from `03b` (env-selected `get_active_provider()`, accessed via the zero-arg `get_proxy_url()`). Touches the crawl billing model from `03c`. +> **Status: ACTIVE (sequenced last in Phase 3).** No longer deferred — captcha bypass is now an explicit goal of the "undetectable, captcha-bypassing universal WebURL crawler." Build **after** `03a` (StealthyFetcher tier + `FetchStrategy` seam + `CrawlOutcome`), `03b` (app-wide proxy provider, accessed via the zero-arg `get_proxy_url()`), `03c` (crawl wallet), and `03e` (stealth-hardening — solving is the *last* resort once humanization + fingerprint coherence have minimized challenges). +> **Decision recap (this pass):** Cloudflare stays in-framework (free, `03a`); reCAPTCHA/hCaptcha/image use a **paid** solver via `captchatools`, **opt-in + off by default**, and are billed as a **separate per-attempt unit** (option (a) below — now decided, not deferred). Logged-in/account-gated captcha flows are **out of scope** (no authenticated scraping in this MVP — see umbrella Phase 8 / `03b` static-proxy hand-off). -## Why deferred +## Why this is the last tier, not the first -Scrapling already solves **Cloudflare Turnstile/Interstitial** via `solve_cloudflare=True` on the StealthyFetcher tier (`03a`; `references/Scrapling/scrapling/fetchers/stealth_chrome.py:38,90`). That covers the most common interstitial. The remaining captcha types (reCAPTCHA v2/v3, hCaptcha, image) need a **paid third-party solver**, add latency (10–60s/solve), have <100% success, and carry target-ToS/legal considerations. None of that is MVP-blocking, so it's sequenced last. +A solve is the **most expensive and least reliable** move available (paid per attempt, 10–60 s latency, <100% success, ToS-sensitive). So the crawler must exhaust the cheaper, in-house levers first; captcha solving only fires when everything else has already failed: + +1. `03e` humanization + fingerprint coherence (geoip locale/tz, persistent profile, headed/Xvfb) → **avoid** triggering a challenge at all. +2. `03a` `solve_cloudflare=True` on StealthyFetcher → free Cloudflare Turnstile/Interstitial. +3. **This subplan** → only the residual reCAPTCHA v2/v3 + hCaptcha + image challenges, and only when `CAPTCHA_SOLVING_ENABLED`. ## Scope boundary @@ -13,46 +18,89 @@ Scrapling already solves **Cloudflare Turnstile/Interstitial** via `solve_cloudf | --- | --- | --- | | Cloudflare Turnstile / Interstitial | Scrapling `solve_cloudflare=True` | `03a`, StealthyFetcher tier | | reCAPTCHA v2/v3, hCaptcha, image | **`captchatools`** (this subplan) | StealthyFetcher `page_action` | +| DataDome / Kasada / reCAPTCHA-**Enterprise** | **none in the free stack** | deferred paid-unblocker tier (`03e`) | ## Grounding (libraries verified) -- **`captchatools` is itself the provider registry.** `new_harvester(api_key, solving_site, sitekey, captcha_url, captcha_type="v2"|"v3"|"hcaptcha"|"image", ...)` → `.get_token(proxy=…, proxy_type=…, user_agent=…, b64_img=…)` returns a **token string** (`references/Captcha-Tools/README.md:28–47`). `solving_site` ∈ {`capmonster`,`2captcha`,`anticaptcha`,`capsolver`,`captchaai`} (`:113–127`). Errors: `ErrNoBalance`, `ErrWrongAPIKey`, `ErrWrongSitekey`, … via `captchatools.exceptions` (`:136–155`). **Implication:** we do **not** rebuild a multi-provider class hierarchy (unlike `03b`'s proxy registry) — `captchatools` already dispatches across the 5 services. Our layer is thin: config resolution + page detection/injection glue. +- **`captchatools` is itself the provider registry.** `new_harvester(api_key, solving_site, sitekey, captcha_url, captcha_type="v2"|"v3"|"hcaptcha"|"image", invisible_captcha=…, min_score=…, action=…)` → `.get_token(proxy=…, proxy_type=…, user_agent=…, b64_img=…)` returns a **token string** (`references/Captcha-Tools/README.md:28–47`). `solving_site` ∈ {`capmonster`,`2captcha`,`anticaptcha`,`capsolver`,`captchaai`} (`:113–127`). Errors: `ErrNoBalance`, `ErrWrongAPIKey`, `ErrWrongSitekey`, `ErrIncorrectCapType`, `ErrNoHarvester` via `captchatools.exceptions` (`:136–155`). **Implication:** we do **not** rebuild a multi-provider class hierarchy (unlike `03b`'s proxy registry) — `captchatools` already dispatches across the 5 services. Our layer is thin: config resolution + page detection/injection glue + billing. - **`captchatools` only harvests a token; it does not inject it.** The caller must drop the token into the page (`g-recaptcha-response` / `h-captcha-response` textarea, or invoke the JS callback) and submit. -- **Injection requires a browser page**, so this only works on the **StealthyFetcher** tier. Scrapling exposes `page_action`: "a function that takes the `page` object, runs after navigation, and does the automation you need" (`stealth_chrome.py:30,82`). AsyncFetcher (HTTP) and DynamicFetcher cannot solve interactive captchas — a captcha hit there must escalate to StealthyFetcher. +- **Injection requires a live browser page**, so this only works on the **StealthyFetcher** tier. Scrapling exposes `page_action`: "a function that takes the `page` object, runs after navigation, and does the automation you need" (`references/Scrapling/scrapling/fetchers/stealth_chrome.py:30,82`; engine docstrings `_browsers/_stealth.py:50,191,326,466`). The HTTP (`AsyncFetcher`) and `DynamicFetcher` tiers cannot solve interactive captchas — a captcha hit there must **escalate to StealthyFetcher** (`FetchStrategy` seam from `03a`; the ladder already ends there). +- **Ordering is already correct in Scrapling.** `solve_cloudflare` runs **before** `page_action` (sync `_stealth.py:253–254` then `:258–260`; async `:529–530` then `:534–536`). So Cloudflare is cleared first, *then* our injector runs — exactly the tier order we want. `page_action` exceptions are caught + logged by Scrapling (`:262` / `:538`), so our factory must also return a clear signal (it can't rely on raising to abort the fetch). -## Sketch (when built) +## Target design -1. **Config layer** (`app/utils/captcha/`, mirroring `app/utils/proxy/`'s config resolution from `03b`): - - `CaptchaConfig` = `(enabled, solving_site, api_key)`, resolved from **env only** — one app-wide config, mirroring `03b`'s env-only single-provider model (`get_active_provider()`); **no per-connector config** (that path was dropped in `03b`). - - Env: `CAPTCHA_SOLVING_ENABLED` (default FALSE), `CAPTCHA_SOLVER_PROVIDER`, `CAPTCHA_SOLVER_API_KEY`. Off by default → zero captcha attempts (and zero solver cost). -2. **Detection + injection `page_action` factory** — builds a callback passed to `StealthyFetcher.async_fetch(..., page_action=…)`: - - Detect sitekey in DOM (`.g-recaptcha[data-sitekey]`, `.h-captcha[data-sitekey]`, reCAPTCHA-v3 via `grecaptcha.execute`). - - Harvest: `new_harvester(solving_site, api_key, sitekey, captcha_url=page.url, captcha_type=…).get_token(proxy=, user_agent=)` — see the IP-binding caveat below. - - Inject token + dispatch events / invoke callback; submit; wait for navigation. -3. **Crawler escalation**: only the StealthyFetcher tier attempts solving; a captcha detected on a lower tier escalates to StealthyFetcher (the ladder already ends there per `03a`). -4. **Dependency**: add `captchatools` to `pyproject.toml` (build-time only). +### 1. Config layer (`app/utils/captcha/`, mirroring `app/utils/proxy/` from `03b`) -## The billing asymmetry (the hard part — decide at build time) +- `CaptchaConfig = (enabled, solving_site, api_key, captcha_type_default, min_score, max_attempts, timeout_s)`, resolved from **env only** — one app-wide config, mirroring `03b`'s env-only single-provider model (`get_active_provider()`); **no per-connector config** (that path was dropped in `03b`). +- Env knobs (next to the proxy + crawl-billing knobs in `config/__init__.py`): + - `CAPTCHA_SOLVING_ENABLED` (default `FALSE`) — off ⇒ zero solve attempts and zero solver cost. + - `CAPTCHA_SOLVER_PROVIDER` (e.g. `capsolver`), `CAPTCHA_SOLVER_API_KEY`. + - `CAPTCHA_MAX_ATTEMPTS_PER_URL` (default `1`), `CAPTCHA_SOLVE_TIMEOUT_S` (default `120`). +- A `captcha_enabled()` static (mirrors `WebCrawlCreditService.billing_enabled()`) so callers gate cheaply before constructing anything. -`03c` bills the workspace owner **per successful crawl** (`CrawlOutcomeStatus.SUCCESS`), and absorbs proxy cost into the flat $1/1000. **Captcha is different**: the solver charges **per attempt** (~$1–3 / 1000, type-dependent) **regardless of whether the crawl ultimately succeeds**. So a failed solve = real upstream cost with **no billable success** — proxy's absorb-it model doesn't transfer cleanly. +### 2. Detection + injection `page_action` factory -Options (resolve when this is actually built): -- **(a) Separate per-solve charge** — meter each solve attempt as its own unit (e.g. `web_crawl_captcha` usage_type, its own `*_MICROS_PER_*` knob), independent of crawl SUCCESS. Most cost-honest; bills even on failed solves (matching the upstream charge). -- **(b) Higher crawl price when captcha enabled** — absorb into a fatter flat rate; simplest UX, but cross-subsidizes failed solves and easy-vs-hard pages unevenly. -- **(c) Cost-plus pass-through** — meter the solver's reported cost × margin. +Builds the **sync** callable passed to `StealthyFetcher.fetch(..., page_action=…)` (03a runs StealthyFetcher via `asyncio.to_thread`, so the sync engine path `_stealth.py:258–262` applies — the factory returns a plain `def(page): ...`, not a coroutine): -Recommendation leaning **(a)** (separate per-attempt unit) because the upstream cost is per-attempt and significant, but defer the final call. +1. **Detect** the challenge + sitekey in the DOM: + - reCAPTCHA v2: `.g-recaptcha[data-sitekey]` (or iframe `src*="recaptcha"`). + - hCaptcha: `.h-captcha[data-sitekey]` (or iframe `src*="hcaptcha"`). + - reCAPTCHA v3: presence of `grecaptcha.execute` + a known `action` (config/site-mapped). + - If no sitekey found → **no-op return** (nothing to solve; let `wait_selector`/extraction proceed). +2. **Bind the proxy/UA** (the IP-coherence caveat below): harvest with the **exact endpoint used for this crawl tier** — + `new_harvester(solving_site, api_key, sitekey, captcha_url=page.url, captcha_type=…, min_score=…, action=…).get_token(proxy=, proxy_type="HTTP", user_agent=)` (`README.md:44–46,107–110`). +3. **Inject + submit**: set the response token into `g-recaptcha-response` / `h-captcha-response` (make textarea visible if needed), dispatch `input`/`change`, invoke the JS callback when present (`grecaptcha.getResponse` flows / `___grecaptcha_cfg` callbacks), then submit the form and `page.wait_for_load_state`. For v3 (no widget) the token is consumed by the site's own `execute()` flow. +4. **Signal outcome** via a mutable closure cell (since Scrapling swallows `page_action` exceptions at `:262` **and discards its return value** at `:260`/`:536`): record `solved: bool` + `attempts: int` into a captured `dict` the crawler reads after `fetch()` returns, so billing (§4) and the retry cap can act on it. **This `page_action`+closure-cell helper is shared with `03f`'s test harness** (which uses the same mechanism to read JS-object verdicts like CreepJS `window.Fingerprint`) — factor it once. + +### 3. Crawler escalation (uses the `03a` `FetchStrategy` seam) + +- Only the **StealthyFetcher** strategy attempts solving. A captcha detected on a lower tier (HTTP/DynamicFetcher) returns non-`SUCCESS`, and the ladder already escalates to StealthyFetcher (`03a`), which this time is constructed with the captcha `page_action` when `captcha_enabled()`. +- **Attempt cap:** at most `CAPTCHA_MAX_ATTEMPTS_PER_URL` solves per URL so one hostile page can't burn unbounded solver credit. The closure cell's `attempts` enforces it. +- No new caller *contract*: the strategy still returns `CrawlOutcome` (`03a` invariant). Captcha is a *parameterization* of the last tier, not a new return shape. +- **But billing needs the attempt count to escape the closure cell.** §4 charges per attempt on **both** the indexer and chat paths, so the count can't stay buried inside `page_action`'s closure — the crawler must copy it onto the outcome. Add **`captcha_attempts: int = 0`** (and `captcha_solved: bool = False`) as **dataclass fields on `CrawlOutcome`** (`03a`). This is safe: `03a`'s "don't widen the return" rule is about the *indexer's* positional `(total_processed, error)` tuple unpacked by length, **not** about `crawl_url`'s dataclass — adding fields to a dataclass breaks no consumer. The indexer sums `outcome.captcha_attempts`; the chat tool reads it off the same outcome. + +### 4. Billing — separate per-attempt unit (option (a), DECIDED) + +`03c` bills per **successful crawl** (`CrawlOutcomeStatus.SUCCESS`) and absorbs proxy cost into the flat $1/1000. **Captcha can't ride that model**: the solver charges **per attempt regardless of whether the crawl ultimately succeeds**, so a failed solve = real upstream cost with no billable crawl success. We therefore meter solves **independently**: + +- New static on `WebCrawlCreditService` (defined in `03c`, knobs added here): `captcha_solves_to_micros(n) = n * config.WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE`. +- Each **attempt** (not each success) is charged when `CAPTCHA_SOLVING_ENABLED` **and** `WEB_CRAWL_CAPTCHA_BILLING_ENABLED`: + - **Indexer/pipeline path:** after the crawl loop, debit `captcha_solves_to_micros(total_attempts)` — where `total_attempts = Σ outcome.captcha_attempts` over the batch — from the **workspace owner** (same owner resolution as `03c §2`) and `record_token_usage(usage_type="web_crawl_captcha", …, cost_micros=…, call_details={"attempts": n, "solved": k})`. Added before the charge commit, same one-transaction pattern as `03c`. + - **Chat-scrape path:** fold `captcha_solves_to_micros(outcome.captcha_attempts)` into the turn accumulator with `call_kind="web_crawl_captcha"` (same mechanism as `03c §3`), so it settles with the premium turn; non-premium/anonymous turns record-but-don't-debit. +- **No pre-block on solves** (unlike the crawl batch): attempts are bounded by `CAPTCHA_MAX_ATTEMPTS_PER_URL × len(urls)`, an upper bound the indexer can optionally pre-check against the wallet if we want symmetry with `03c §2` (recommended: pre-check the combined crawl + worst-case captcha estimate so a run can't strand mid-batch on solver insolvency). +- New env (next to `03c`'s knobs): `WEB_CRAWL_CAPTCHA_BILLING_ENABLED` (default `FALSE`), `WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE` (default e.g. `3000` = $3/1000 attempts; type-dependent, set with margin over the solver's per-attempt price). **No DB migration** — `web_crawl_captcha` is just another free-form `TokenUsage.usage_type` value (`db.py` `usage_type String(50)`). + +### 5. Dependency + Docker + +- Add `captchatools` to `pyproject.toml` (build-time only). It's a thin HTTP client for the solver services — no browser binaries. +- No `Dockerfile` change beyond the dependency (browser already installed by `03a`/`03e`). + +## Error handling (must not loop or silently burn credit) + +- `ErrNoBalance` / `ErrWrongAPIKey` (`README.md:134,136–143`) → **stop solving for the rest of the run** (set a process/run flag), surface a clear "captcha solver out of balance / misconfigured" message via `log_task_failure`, and fall through to a non-`SUCCESS` `CrawlOutcome`. The README explicitly warns balance-exhausted loops can get the **IP temporarily banned** (`:132–134`) — never retry on `ErrNoBalance`. +- `ErrWrongSitekey` / `ErrIncorrectCapType` → log + skip this URL's solve (likely detection bug, not transient); count the attempt for billing only if the solver actually charged. +- Timeout (`CAPTCHA_SOLVE_TIMEOUT_S`) → abort the solve, count one attempt, return non-`SUCCESS`. ## Risks / considerations - **Solver-tier only.** No captcha solving on the HTTP/DynamicFetcher tiers; must escalate to StealthyFetcher (slower, browser-backed). -- **Latency & flakiness.** Solves take 10–60s and aren't guaranteed; tune timeouts and a max-attempts cap so a single URL can't burn unbounded solver credit. -- **Solver-account balance.** `ErrNoBalance`/`ErrWrongAPIKey` (`README.md:136–155`) must surface clearly and disable solving rather than loop. -- **Proxy coherence (rotating-pool caveat).** Some captchas IP-bind the token, so the solver must egress from the **same IP** as the crawl. Under `03b`'s single app-wide provider this is automatic for single-endpoint providers (`anonymous_proxies`, single-URL `custom`). But a **pool-backed `CustomProxyProvider`** returns the *next* endpoint on each zero-arg `get_proxy_url()` call — so the `page_action` must **reuse the endpoint actually used for this crawl tier** (capture it once, pass it to `get_token(proxy=…)`), NOT call `get_proxy_url()` again (which would rotate to a different IP). This needs a small seam to surface the crawl's chosen endpoint into the `page_action` — note it when `03d` is built on top of `03b`'s rotation. -- **Policy / ToS.** Automated captcha solving may violate a target site's terms; gate behind the explicit `CAPTCHA_SOLVING_ENABLED` flag and treat as an opt-in, owner-acknowledged capability. +- **Latency & flakiness.** Solves take 10–60 s and aren't guaranteed; the `CAPTCHA_MAX_ATTEMPTS_PER_URL` + `CAPTCHA_SOLVE_TIMEOUT_S` caps keep a single URL from burning unbounded solver credit. +- **Proxy coherence (rotating-pool caveat).** Many captchas IP-bind the token, so the solver must egress from the **same IP** as the crawl. Under `03b`'s single app-wide provider this is automatic for single-endpoint providers (`anonymous_proxies`, single-URL `custom`). But a **pool-backed `CustomProxyProvider`** returns the *next* endpoint on each zero-arg `get_proxy_url()` call — so the `page_action` must **reuse the endpoint actually used for this crawl tier** (the crawler captures it once and passes it into the factory), NOT call `get_proxy_url()` again (which would rotate to a different IP). This is the same "surface the chosen endpoint" seam `03e`/`03d` both rely on — build it once in `03a`'s strategy context. +- **Enterprise captchas are out of reach.** reCAPTCHA **Enterprise**, DataDome, and Kasada use behavioral + device signals the token-injection model doesn't satisfy reliably; those route to the deferred paid-unblocker tier (`03e`), not here. +- **Policy / ToS.** Automated captcha solving may violate a target site's terms; gate behind the explicit `CAPTCHA_SOLVING_ENABLED` flag and treat as an opt-in, owner-acknowledged capability. Public data only (no logged-in bypass). + +## Work items + +1. **`app/utils/captcha/`**: `CaptchaConfig` + env resolution + `captcha_enabled()` static. +2. **`page_action` factory**: detection (v2/v3/hcaptcha/image sitekey), harvest via `captchatools` with bound proxy/UA, inject+submit, outcome closure cell. +3. **Crawler wiring**: construct StealthyFetcher with the factory when `captcha_enabled()`; enforce per-URL attempt cap; surface the crawl's chosen proxy endpoint into the factory. +4. **Billing**: `WebCrawlCreditService.captcha_solves_to_micros` + the two knobs; debit per-attempt on the indexer path (owner) and fold into the turn on the chat path; `record_token_usage(usage_type="web_crawl_captcha")`. +5. **Config + docs**: all env knobs in `Config` + `.env.example` (commented, hosted=TRUE / self-hosted=FALSE), plus the `captchatools` dependency. +6. **Tests**: solving disabled → zero attempts, zero solver cost (self-hosted); v2 detected → harvest+inject+submit path invoked with the **crawl's** proxy endpoint (not a re-rotated one); `ErrNoBalance` → stops solving + no retry loop; attempt cap honored; billing enabled → per-attempt debit even when the crawl ultimately fails; chat path folds captcha micros into a premium turn and record-only on anonymous; `web_crawl_captcha` `TokenUsage` rows written. ## Out of scope -- Anything in `03a`/`03b`/`03c`. +- Anything owned by `03a`/`03b`/`03c`/`03e`. - Cloudflare Turnstile (already handled by Scrapling in `03a`). -- Final captcha billing model — chosen at build time (see options above). +- Logged-in / account-gated captcha flows (no authenticated scraping this MVP → umbrella Phase 8 + `03b` static-proxy hand-off). +- Enterprise/behavioral anti-bot (DataDome/Kasada/reCAPTCHA-Enterprise) → deferred paid-unblocker tier (`03e`). diff --git a/plans/backend/03e-stealth-hardening.md b/plans/backend/03e-stealth-hardening.md new file mode 100644 index 000000000..05fb18343 --- /dev/null +++ b/plans/backend/03e-stealth-hardening.md @@ -0,0 +1,119 @@ +# Phase 3e — Stealth hardening (the in-house "undetectable" moat) + +> Part of **Phase 3 — WebURL Crawler & Crawl Billing**. See `00-umbrella-plan.md`. +> Depends on `03a` (Scrapling StealthyFetcher tier + `FetchStrategy` seam + `CrawlOutcome`) and `03b` (app-wide proxy provider). Precedes `03d` (captcha) in the escalation order — hardening **avoids** challenges; captcha solving is the paid last resort for the ones we can't avoid. Touches `03c` only via the deferred paid-unblocker tier (its own billing, decided if/when built). + +> **Implementation note.** Same convention as `03a`–`03d`: citations use **today's** names (`search_space_id`/`SearchSpace`) and predate `03a`'s `crawl_url` refactor; locate code by **symbol/grep**, not absolute lines. Scrapling references point at the on-disk (gitignored) `references/Scrapling/` checkout pinned to `scrapling[fetchers]>=0.4.9` (`pyproject.toml:91`). + +## Objective + +Push the Universal WebURL Crawler as far toward "undetectable" as the **free / in-house stack** allows, so we hold a scraping moat for the next ~4–6 months **without** a third-party unblocker (ZenRows/ScrapFly/Bright Data) or a source-patched browser (CloakBrowser — rejected on licensing). Everything here is **runtime/config-level** stealth layered on Scrapling's patchright-Chromium (`03a` engine note): geoip fingerprint coherence, persistent profiles, headed execution, real fonts, and behavioral humanization — plus a **block classifier** + **per-domain strategy memory** so the ladder learns and the cheap tiers get skipped once a domain's working strategy is known. The paid-unblocker tier is defined here as a **deferred** `FetchStrategy` (the explicit escape hatch for when in-house maintenance gets too costly), not built now. + +## The realistic ceiling (be honest with downstream devs) + +Scrapling already ships strong **runtime** stealth by default: `DEFAULT_ARGS + STEALTH_ARGS` (`references/Scrapling/scrapling/engines/constants.py:24–99`, incl. `--disable-blink-features=AutomationControlled` `:94`), a persistent context (`_browsers/_stealth.py:91–93,366–368`), and `navigator.webdriver` masking via patchright. With `03b` residential proxies + this plan's coherence/humanization on top, the crawler reliably handles **Cloudflare** (via `03a` `solve_cloudflare`) and the long tail of **moderate** anti-bot. + +What the free stack does **not** reliably beat: **DataDome, Kasada, reCAPTCHA-Enterprise**, and other behavioral/device-fingerprint systems. Those generally require C++ source-level browser patches (CloakBrowser's 58-patch approach — out on licensing) or a commercial unblocker. For **competitive-intelligence targets** these top-tier defenses are the exception, not the rule, so the in-house moat is a sound 4–6 month bet — with the deferred paid tier (§8) as the pre-wired escape hatch. **Do not promise "beats everything."** + +## Levers (all behind the `03a` `FetchStrategy` seam — callers stay outcome-only) + +### 1. Geoip fingerprint coherence (match the browser to the proxy exit IP) + +A residential IP in Berlin behind an `en-US`/`America/New_York` browser is an instant tell. Make the fingerprint cohere with the proxy's exit geo: + +- Resolve the proxy exit IP → country/locale/timezone (lightweight geoip; e.g. a bundled MaxmindLite/`geoip2` DB or the proxy provider's location knob `RESIDENTIAL_PROXY_LOCATION` from `03b`). +- Pass the matched values into StealthyFetcher: `locale=` (drives `navigator.language` + `Accept-Language` — `stealth_chrome.py:34–35`) and `timezone_id=` (`:36`). These flow into the Playwright context (`_browsers/_base.py:440–441`). +- This needs the crawl's **chosen proxy endpoint** surfaced into the strategy (the same seam `03d` needs for IP-bound solves) — capture it once in `03a`'s strategy context, don't re-call `get_proxy_url()` (which rotates on a pool-backed provider, `03b`). + +### 2. Fingerprint flags Scrapling already exposes (turn them on) + +- `hide_canvas=True` — random noise on canvas ops to defeat canvas fingerprinting (`stealth_chrome.py:40`). +- `block_webrtc=True` — forces WebRTC to respect the proxy, preventing the **real local IP leak** that unmasks proxied browsers (`:41`). +- `google_search=True` (default) — sets a Google referer so the first hit looks like organic arrival (`:45`); override per-need via `extra_headers` (`:46`). +- `additional_args=` — last-priority Playwright context overrides for anything not surfaced as a first-class param (`:51`). + +### 2b. HTTP-tier TLS fingerprint (the AsyncFetcher tier — `impersonate`) + +The cheap static tier (`03a` tier 1) is the **first** thing every crawl hits, yet it currently sets `stealthy_headers=True` but **no** `impersonate` (`webcrawler_connector.py:284–289`), so its TLS ClientHello is curl_cffi's default JA3 — a non-browser signature that fingerprinting WAFs flag before the body even loads. Pass an `impersonate="chrome"` profile (Scrapling's static engine selects a matching curl_cffi browser profile — `references/Scrapling/scrapling/engines/static.py:36–47`) so the HTTP tier's **JA3/JA4/HTTP-2** matches a real Chrome and coheres with the browser tiers' UA. Cheap, safe, default-on. `03f §S3` is the test that validates parity against `tls.peet.ws`. + +### 3. Persistent per-domain profiles (look like a returning human) + +Scrapling defaults to a **temporary** user-data dir (fresh = suspicious). Use `user_data_dir=` (`stealth_chrome.py:48`; `launch_persistent_context` `_stealth.py:93,366–368`) to keep a **persistent profile per domain** (or per domain+proxy-geo), so cookies/localStorage/site-trust carry across crawls and the browser presents as a returning visitor rather than a brand-new incognito session. Store profiles under a configured dir (`shared_tmp`-style volume so API + worker share them). + +### 4. Headed execution under Xvfb (defeat headless tells) + +`headless` defaults to hidden (`stealth_chrome.py:19/71`). Many WAFs flag headless Chromium. Run **headful** (`headless=False`) inside a virtual framebuffer (**Xvfb**) in the Docker worker so the browser is "visible" to itself but needs no real display. Gate behind a config flag (off by default for self-hosted, on for hosted workers that have Xvfb installed). + +### 5. Real fonts (canvas/emoji hash realism) + +A minimal container has almost no fonts, making canvas/emoji fingerprint hashes obviously synthetic. Install real font packages (DejaVu, Liberation, Noto incl. CJK + emoji) in the worker image so font-enumeration + canvas hashes resemble a real desktop. `Dockerfile` change only (`apt-get install fonts-*`). + +### 6. Behavioral humanization (DIY — Scrapling has no `humanize` for the Chromium engine) + +Unlike Camoufox, the patchright-Chromium StealthyFetcher exposes **no built-in `humanize`** (verified: zero matches in the StealthyFetcher param set). So humanization is custom, injected via the existing hooks: + +- `page_action=` (runs after navigation, `stealth_chrome.py:30`; engine `_stealth.py:258–262`) — randomized mouse moves/curves, scrolls, hover-before-click, and small think-time delays before extraction. This is the **same hook `03d` uses** for token injection, so the two compose (humanize → optional captcha solve). +- `init_script=` (JS executed on page creation, `:33`) — early shims for any residual JS tells not covered by patchright. +- Tunable `wait`/`network_idle` (`:29,27`) so dwell time isn't robotically constant. + +### 7. Block classifier + per-domain strategy memory (the "learning ladder") + +Two small in-house pieces make the ladder smart instead of brute-force: + +- **Block classifier** — inspect each `Response` (status + body/cookie markers) and label the outcome: `OK` / `CLOUDFLARE` / `CAPTCHA_RECAPTCHA` / `CAPTCHA_HCAPTCHA` / `DATADOME` / `KASADA` / `RATE_LIMITED` / `EMPTY`. (Markers: `"Just a moment"` + `cf-mitigated` → Cloudflare; `datadome` cookie/script → DataDome; `g-recaptcha`/`h-captcha` sitekeys → captcha; etc.) The label drives escalation (which tier/lever next), routes captcha types to `03d`, and feeds the memory below. It also makes `CrawlOutcome.status` decisions principled instead of "empty == fail". +- **Per-domain strategy memory** — cache the **strategy that last succeeded per domain** (e.g. Redis key `crawl:strategy:{domain}` with TTL, reusing the existing Redis the indexer already uses for `indexing_locks`). Next crawl of that domain starts at the known-good tier/lever set, skipping cheaper tiers that always fail there — fewer requests, lower latency, less proxy/captcha spend. No DB migration (Redis, best-effort, self-healing on miss). + +## 8. Deferred: paid-unblocker tier (the escape hatch, NOT built now) + +The moat strategy is explicit: **maintain in-house bypass for ~4–6 months, then move hostile targets to a paid unblocker if demand/maintenance justifies it.** That switch is **evidence-driven, not a guess** — `03f`'s manual scorecard quantifies the free-stack ceiling over time and is the documented trigger for flipping this tier. Pre-wire the seam so the switch is a config flip, not a refactor: + +- Define (but do not implement) a `PaidUnblockerStrategy` that satisfies the `03a` `FetchStrategy` contract `(url, ctx) -> CrawlOutcome`, appended **last** in the ladder and active only when an env flag + API key are set. +- It would call an external unblocker (ZenRows/ScrapFly/Bright Data Web Unlocker) for the residual `DATADOME`/`KASADA`/`reCAPTCHA-Enterprise` domains the block classifier flags as unreachable in-house. +- **Its own billing** (cost-plus pass-through, decided at build time) — separate from `03c`'s flat crawl unit and `03d`'s per-solve unit, because unblocker pricing is per-request and provider-specific. +- Until built, those domains simply return non-`SUCCESS` (free under `03c`). This keeps the umbrella's "WebURL Crawler is the moat" honest while bounding our maintenance risk. + +## Config / env changes + +Add (all default OFF / conservative; next to the `03b`/`03c` knobs in `config/__init__.py` + `.env.example`): + +- `CRAWL_GEOIP_MATCH_ENABLED` (default FALSE) + optional geoip DB path. +- `CRAWL_HIDE_CANVAS` / `CRAWL_BLOCK_WEBRTC` (default TRUE — cheap, safe). +- `CRAWL_PERSISTENT_PROFILES_DIR` (unset → Scrapling's temp default; set → per-domain profiles). +- `CRAWL_HEADED_XVFB_ENABLED` (default FALSE; requires Xvfb in the image). +- `CRAWL_HUMANIZE_ENABLED` (default TRUE) + dwell/jitter bounds. +- `CRAWL_STRATEGY_MEMORY_TTL_S` (default e.g. `86400`; 0 → disabled). +- `CRAWL_PAID_UNBLOCKER_ENABLED` (default FALSE) + provider/key (deferred tier). + +## Docker changes (`surfsense_backend/Dockerfile`) + +- Install **Xvfb** + a font set (`fonts-dejavu`, `fonts-liberation`, `fonts-noto`, `fonts-noto-cjk`, `fonts-noto-color-emoji`) in the worker image. (Note from `03a`: also drop the stale "+ Camoufox" comment near `:112`; `scrapling install` only fetches Chromium.) +- Headed runs need the browser launched under `xvfb-run` (or an Xvfb display in the worker entrypoint), gated by `CRAWL_HEADED_XVFB_ENABLED`. + +## Work items + +1. **Geoip coherence** — resolve proxy exit geo → `locale`/`timezone_id`; thread the crawl's chosen endpoint into the strategy context (shared seam with `03d`). +2. **Fingerprint flags** — wire `hide_canvas`/`block_webrtc`/`google_search`/`extra_headers`/`additional_args` from config into the StealthyFetcher tier. Also add `impersonate="chrome"` to the AsyncFetcher (HTTP) tier (§2b) so its TLS coheres. **Centralize this into a single per-tier kwargs builder** (one function that returns the StealthyFetcher / AsyncFetcher kwargs from config) — the crawler *and* `03f`'s harness both import it, so the scorecard grades the exact browser we ship (no test-vs-prod drift). +3. **Persistent profiles** — per-domain `user_data_dir` under `CRAWL_PERSISTENT_PROFILES_DIR` (shared volume). +4. **Headed + Xvfb** — `headless=False` path gated by flag; Xvfb + fonts in the worker image. +5. **Humanization** — a `page_action` humanizer (mouse/scroll/dwell) composing with `03d`'s injector; optional `init_script` shims. +6. **Block classifier** — `classify(response) -> BlockType`, used by the ladder + `CrawlOutcome.status` + `03d` routing. +7. **Per-domain strategy memory** — Redis read/write around the ladder; start at known-good tier; self-heal on miss. +8. **Paid-unblocker seam** — define the deferred `FetchStrategy` + config flag only (no provider integration). +9. **Instrumentation** — log `(domain, block_type, winning_strategy, attempts, latency)` for tuning the ladder. +10. **Config + Docker + tests**. + +## Risks / trade-offs + +- **Arms race / maintenance.** Fingerprint bypasses rot as WAFs update; this is exactly the cost the deferred paid tier (§8) hedges. Instrumentation (work item 9) plus **`03f`'s scorecard** are what tell us when in-house upkeep stops being worth it. +- **Headed/Xvfb cost.** Headful browsers use more CPU/RAM than headless; gate per-flag and only escalate to headed when the block classifier says cheaper tiers fail for a domain (per-domain memory keeps it from being the default). +- **Profile growth.** Persistent profiles accumulate disk; add a size/TTL cap and periodic prune. +- **Geoip accuracy.** A coarse country→locale map is fine; over-fitting per-city tz isn't worth it. Wrong-but-coherent beats default-mismatched. +- **No silver bullet.** Reiterate the ceiling (§"realistic ceiling") in any user-facing copy: the crawler is "best-effort undetectable," not guaranteed. + +## Out of scope (hand-offs) + +- Cloudflare solving (`03a`), reCAPTCHA/hCaptcha solving + its per-solve billing (`03d`), proxy rotation (`03b`), flat crawl billing (`03c`). +- **Logged-in / account-based bypass** (sticky/static proxies + credential management) — deferred to the platform-actor work (umbrella Phase 8 + `03b` static-proxy hand-off). Public data only this MVP. +- Building the paid-unblocker provider integration — deferred (§8 leaves only the seam + flag). +- **Measuring** undetectability (the scorecard that grades these levers) → `03f` (manual harness). This plan *builds* the levers; `03f` *tests* them. +- Platform-specific structured extractors (Google Maps, LinkedIn public, …) — these sit **on top** of this hardened fetch core as Phase-8 actors; this plan only delivers the core they depend on. diff --git a/plans/backend/03f-undetectability-testing.md b/plans/backend/03f-undetectability-testing.md new file mode 100644 index 000000000..93f02d1d6 --- /dev/null +++ b/plans/backend/03f-undetectability-testing.md @@ -0,0 +1,124 @@ +# Phase 3f — Undetectability & extraction test harness (manual scorecard) + +> Part of **Phase 3 — WebURL Crawler & Crawl Billing**. See `00-umbrella-plan.md`. +> **Status: ACTIVE, sequenced last in Phase 3** (after `03a`–`03e` ship — the harness measures what those built). **Manual-only** — no CI/automated gating for now (resolved decision); it's a dev-run scorecard, not a build gate. Depends on `03a` (the Scrapling tiers + `FetchStrategy`/`CrawlOutcome`), `03b` (proxy provider), `03e` (the stealth levers being tested), and reuses the `page_action` + closure-cell mechanism shared with `03d`. + +> **Convention note.** This is **dev/operator tooling**, not a product code path, so it's untouched by the Phase 1–2 rename (no `search_space_id`/`workspace_id` concern). Citations to the gitignored reference checkouts (`references/CloakBrowser/`, `references/Scrapling/`) are pinned to what's on disk; locate code by **symbol/grep** if lines drift. + +## Objective + +A **manual, repeatable scorecard** that answers one question: *how undetectable (and how correct) is our Universal WebURL Crawler right now?* It drives the real Scrapling tiers against the industry's standard detection + sandbox sites, parses each site's verdict, and prints a pass/fail + numeric scorecard. Its real job is to **quantify the free-stack ceiling** (`03e`) over time so we know — with evidence — when fingerprint maintenance stops being worth it and we should flip the deferred paid-unblocker tier. + +It is explicitly **two axes, two suites** (kept separate so each scales independently): + +- **Suite S — Stealth / anti-bot** (is the crawler detected?): browser-tier fingerprint/bot tests, HTTP/TLS-tier fingerprint, proxy/leak verification. +- **Suite E — Extraction correctness** (does the crawler get the content right?): scraping sandboxes for the HTTP vs JS (DynamicFetcher) tiers + Trafilatura quality. + +## How CloakBrowser tests (the pattern we copy) + +CloakBrowser's suite is a **driven-browser verdict-scraper**, not unit tests (`references/CloakBrowser/examples/stealth_test.py`, run via `bin/cloaktest`). The repeated shape: + +1. Launch a real page with **proxy + geoip**, toggleable **headed/headless** (`stealth_test.py:231` `launch(headless=…, proxy=…, geoip=True)`). +2. `page.goto(site, wait_until="networkidle")` then **sleep** — scores compute async (Castle.js ~20 s `fingerprint_scan_test.py:31`; CreepJS ~30 s `:96`; reCAPTCHA polls up to 30 s `stealth_test.py:156–164`). +3. `page.evaluate(js)` to **parse the verdict** from the rendered DOM (sannysoft table `stealth_test.py:32–45`) or from internal JS objects (`window.Fingerprint.headless` `fingerprint_scan_test.py:117–127`). +4. Apply an explicit **pass threshold** + screenshot (`stealth_test.py:169–217`). + +Their shipped bars (we adopt as **aspirational targets**, see Scorecard): sannysoft **0 fails**; `bot.incolumitas` ≤ `{WEBDRIVER, connectionRTT}` known-FPs; `browserscan` **0 Abnormal**; `deviceandbrowserinfo` `isBot=false`; FingerprintJS demo **not blocked**; reCAPTCHA v3 **≥0.7** (they hit 0.9); CreepJS **headless ≤30% / stealth ≤30%** (`fingerprint_scan_test.py:112–114,166–171`). + +## The bridge to our Scrapling crawler + +We are **not** a single browser; we're the `03a` tier ladder. The harness therefore tests **per tier**, and extracts verdicts two ways: + +- **DOM/JSON-rendered verdicts** → just `StealthyFetcher.fetch(url, …)` (or `Fetcher.get` for JSON endpoints) and parse the **returned post-JS page** with Scrapling's selector (`load_dom` is on by default — `references/Scrapling/scrapling/fetchers/stealth_chrome.py:43`). Covers sannysoft, incolumitas, deviceandbrowserinfo, the reCAPTCHA score text, and every JSON endpoint (`tls.peet.ws/api/all`, `httpbin/headers`). +- **Internal JS-object verdicts** (CreepJS `window.Fingerprint`, Castle.js score node) → a **`page_action`** that runs `page.evaluate()` and stashes the result into a **closure cell**, because Scrapling **discards `page_action`'s return value** (sync `_stealth.py:260`, async `:536`). **This is the exact same `page_action`+closure-cell plumbing as `03d`'s token injector** — building the harness de-risks `03d` (and vice-versa); factor it once. + +## Suite S — Stealth / anti-bot + +### S1. Browser tier (StealthyFetcher — the "undetectable" tier) + +| Site | Signal | Extraction | Aspirational bar | +|---|---|---|---| +| `bot.sannysoft.com` | webdriver/chrome/plugins/UA leaks | DOM table | 0 fails | +| `bot.incolumitas.com` | 30+ checks incl. behavioral | JSON-in-body | ≤ `{WEBDRIVER, connectionRTT}` | +| `browserscan.net/bot-detection` | WebDriver/CDP/Navigator | DOM text | 0 Abnormal | +| `deviceandbrowserinfo.com/are_you_a_bot` | fingerprint + behavioral | JSON `isBot` | `isBot=false` | +| `abrahamjuliot.github.io/creepjs` | fingerprint **consistency/lies**, headless% | `window.Fingerprint` (page_action) | headless ≤30%, stealth ≤30% | +| `fingerprint-scan.com` | Castle.js bot-risk + headless signals | DOM node + evaluate | low risk; headless signals 0 | +| `demo.fingerprint.com/web-scraping` | **behavioral block** (click→blocked vs flights) | page_action click + DOM | not blocked | +| `recaptcha-demo.appspot.com/...v3-request-scores.php` | Google server-verified human score | `wait_for_function` + regex (`recaptcha_score.py:22–28`) | score ≥0.7 | + +### S2. Per-property fingerprint detail (manual/debug — validates `03e` levers directly) + +- `browserleaks.com/canvas`, `/webgl`, `/fonts` — confirms `03e` `hide_canvas` + font packages. +- `browserleaks.com/webrtc` + DNS leak — confirms `03e` `block_webrtc` (no real-IP leak through the proxy). + +### S3. HTTP/TLS tier (AsyncFetcher / `Fetcher` — curl_cffi impersonation) + +Our HTTP tier is `curl_cffi`-based and *can* impersonate a real Chrome TLS stack (`references/Scrapling/scrapling/fetchers/requests.py:29`; `engines/static.py:6–9,36–47`) — **but only if `impersonate=` is set.** It currently is **not** (`webcrawler_connector.py:284–289` passes `stealthy_headers=True` only), so today this tier's JA3 is curl_cffi's default and **will fail parity**. This row is therefore the *driver* for the `03e §2b` `impersonate="chrome"` lever: run it before (red) and after (green) that fix. + +- `tls.peet.ws/api/all` (+ `/api/clean`) — JSON **JA3/JA4/Akamai-HTTP2/PeetPrint**; diff against a real-Chrome baseline. +- `httpbin.co/headers` (or httpbingo) — header set/order/UA sanity. + +**Recommendation (resolved): TLS parity is a first-class *axis* but an *informational threshold*, not a hard gate.** We record JA3/JA4 and flag drift from the Chrome baseline, but don't "fail" on it — curl_cffi impersonation is strong yet JA-hashes shift across versions, and this is a manual scorecard anyway. Treat a *mismatch* as a tuning signal (pick a closer `impersonate` profile), not a regression. + +### S4. Proxy / leak verification + +- `httpbin.org/ip` — exit IP == the proxy endpoint actually used (capture-once seam from `03b`/`03e`, not a re-rotated `get_proxy_url()`). +- WebRTC/DNS (S2) — no real-IP leak. + +## Suite E — Extraction correctness (separate axis) + +Purpose-built, ToS-safe sandboxes — validates the HTTP vs **DynamicFetcher (JS)** tiers and Trafilatura output, independent of stealth: + +- `books.toscrape.com` — static catalog + pagination (HTTP tier; baseline extraction). +- `quotes.toscrape.com` — has **`/js`** (JS-rendered), **`/js-delayed`** (`?delay=`), **scroll** (infinite), **login** (CSRF) variants → exercises the DynamicFetcher JS tier + `wait_selector`/`network_idle`. +- `scrapethissite.com` — mixed structures for extraction robustness. + +Assertion style: known expected values (e.g. first book title, quote count per page) so extraction regressions are caught deterministically. + +## Scorecard & thresholds (resolved) + +- **Adopt CloakBrowser's bars as aspirational targets** (above), but the harness's primary output is **our actual measured numbers recorded as the baseline** — a committed `scorecard.md`/JSON snapshot per run (date, tier, proxy on/off, headed/headless, per-site result). Subsequent runs diff against the last baseline so we see drift (ours improving, or a WAF tightening). +- Each row reports: site, tier, verdict, numeric (where applicable), PASS/FAIL vs aspirational bar, and screenshot path. +- A run is summarized as `passed/total` per suite (like `stealth_test.py:314–317`), **never blocking** anything. + +## Harness design + +- Lives under dev tooling (e.g. `surfsense_backend/scripts/crawler_testbench/` or `tests/manual/` — not collected by the normal pytest run, since it hits the live internet + needs proxies). A thin CLI mirrors `bin/cloaktest`: `python -m ... [--proxy URL] [--headed] [--headless] [--suite S|E|all] [--no-screenshots]`. +- **Reuse split (important — `crawl_url` is *not* a drop-in here):** + - **Suite E** drives the **real `crawl_url`** end-to-end — extraction correctness *is* the production path (auto-ladder + Trafilatura markdown is exactly what we want to assert). + - **Suite S** does **not** use `crawl_url`. Two reasons: (1) `crawl_url` auto-ladders and stops at the first `SUCCESS`, so a detection site might be answered by the cheap HTTP tier when we mean to grade the **StealthyFetcher** tier; (2) `crawl_url` returns Trafilatura **markdown** (`_build_result`), but verdict parsing needs the **raw post-JS DOM** (and `window.*` objects need a live page). So Suite S drives the **individual Scrapling fetchers directly**, per tier. + - **Avoid test-vs-prod drift:** Suite S must construct each fetcher from the **same centralized stealth-config builder** `03e` introduces (the single source of truth for `locale`/`timezone_id`/`hide_canvas`/`block_webrtc`/`impersonate`/profile/headed), **not** a hand-rolled kwargs set — otherwise the scorecard grades a browser we don't ship. (`03e` work item: expose that builder so both the crawler and this harness import it.) +- Outputs: console summary + screenshots + the `scorecard` snapshot. +- Runs with the app-wide proxy provider (`03b`) and `03e` levers on, so the scorecard reflects production fetch behavior — **except captcha solving, which Suite S forces OFF** (`CAPTCHA_SOLVING_ENABLED=false`): we measure the *unaided* stealth/score, and it avoids the `03d` injector firing paid solves against the reCAPTCHA-demo / FingerprintJS rows. Captcha solving is exercised separately (functional `03d` test), not in the stealth scorecard. + +## The ceiling-decision loop (why this matters) + +The scorecard is the evidence base for the moat strategy: re-run it (a) on a cadence and (b) whenever crawls start failing in the wild. When the **hard** rows (FingerprintJS demo, reCAPTCHA-v3 ≥0.7, and any DataDome/Kasada targets) degrade and `03e` levers can't recover them, that's the documented trigger to flip the **deferred paid-unblocker `FetchStrategy`** (`03e §8`). Without this harness, that decision is guesswork. + +## Config / dependencies + +- **No new production dependencies** — reuses Scrapling (already pinned) + the crawler. Optionally a tiny dev helper for the scorecard diff (or just stdlib `json`). +- Needs **residential proxy creds** (the `03b` env) to be meaningful; document a "no-proxy" mode that still runs but is expected to fail the harder rows (datacenter IP). +- A results dir (gitignored screenshots; committed `scorecard` snapshots). + +## Work items + +1. **Shared `page_action`+closure-cell helper** (factored with `03d`) for JS-object verdict extraction. +2. **Suite S runners** (S1 browser, S3 TLS, S4 proxy-leak) with per-site parse + aspirational thresholds; S2 as manual debug links. +3. **Suite E runners** against the toscrape/scrapethissite sandboxes with deterministic assertions. +4. **CLI + scorecard** (snapshot writer + diff vs last baseline) mirroring `bin/cloaktest` ergonomics. +5. **Docs**: a short runbook (how to run, read the scorecard, interpret the ceiling trigger). + +## Risks / trade-offs + +- **Flaky/rate-limited/changing sites.** Detection sites move DOM and tighten over time; the harness must tolerate parse misses (report `ERROR`, not crash — `stealth_test.py:298–300`) and is **manual** precisely so flakiness never blocks development. +- **Proxy required for realism.** Without residential egress the hard rows fail by design (datacenter IP); document this so a red scorecard isn't misread. +- **ToS.** These are public detection/sandbox sites intended for this purpose; keep volume low and don't hammer. +- **Not a guarantee.** Passing sannysoft/CreepJS ≠ beating DataDome/Kasada; the scorecard's value is *trend + ceiling visibility*, not a green checkmark. + +## Out of scope (hand-offs) + +- **CI automation** — deferred (resolved: manual-only now). If revisited, it needs tolerant thresholds + proxy creds + nightly cadence; the scorecard JSON is designed to make that easy later. +- The levers themselves (`03e`), captcha solving (`03d`), proxy provider (`03b`), billing (`03c`) — this plan only **measures** them. +- The deferred paid-unblocker integration (`03e §8`) — the harness defines its *trigger*, not its build. diff --git a/surfsense_backend/alembic/versions/168_harden_refresh_token_schema.py b/surfsense_backend/alembic/versions/168_harden_refresh_token_schema.py index 4168da007..75c2725c3 100644 --- a/surfsense_backend/alembic/versions/168_harden_refresh_token_schema.py +++ b/surfsense_backend/alembic/versions/168_harden_refresh_token_schema.py @@ -16,35 +16,48 @@ depends_on: str | Sequence[str] | None = None def upgrade() -> None: op.execute( - "ALTER TABLE refresh_tokens " - "ADD COLUMN IF NOT EXISTS revoked_at TIMESTAMP WITH TIME ZONE" + "ALTER TABLE refresh_tokens ADD COLUMN IF NOT EXISTS " + "revoked_at TIMESTAMP WITH TIME ZONE" ) op.execute( - "ALTER TABLE refresh_tokens " - "ADD COLUMN IF NOT EXISTS absolute_expiry TIMESTAMP WITH TIME ZONE" + "ALTER TABLE refresh_tokens ADD COLUMN IF NOT EXISTS " + "absolute_expiry TIMESTAMP WITH TIME ZONE" ) + + bind = op.get_bind() + is_revoked_exists = bind.execute( + sa.text( + """ + SELECT EXISTS ( + SELECT FROM information_schema.columns + WHERE table_schema = current_schema() + AND table_name = 'refresh_tokens' + AND column_name = 'is_revoked' + ) + """ + ) + ).scalar() + + if is_revoked_exists: + op.execute( + """ + UPDATE refresh_tokens + SET revoked_at = NOW() + WHERE is_revoked = TRUE + AND revoked_at IS NULL + """ + ) + op.execute( - """ - DO $$ - BEGIN - IF EXISTS ( - SELECT 1 FROM information_schema.columns - WHERE table_name = 'refresh_tokens' - AND column_name = 'is_revoked' - ) THEN - UPDATE refresh_tokens SET revoked_at = NOW() WHERE is_revoked = TRUE; - END IF; - END $$; - """ + "ALTER TABLE refresh_tokens ALTER COLUMN token_hash TYPE VARCHAR(64)" ) - op.execute("ALTER TABLE refresh_tokens ALTER COLUMN token_hash TYPE VARCHAR(64)") op.execute("ALTER TABLE refresh_tokens DROP COLUMN IF EXISTS is_revoked") def downgrade() -> None: op.execute( - "ALTER TABLE refresh_tokens " - "ADD COLUMN IF NOT EXISTS is_revoked BOOLEAN NOT NULL DEFAULT false" + "ALTER TABLE refresh_tokens ADD COLUMN IF NOT EXISTS " + "is_revoked BOOLEAN NOT NULL DEFAULT false" ) op.execute( """ @@ -61,6 +74,8 @@ def downgrade() -> None: """ ) op.execute("ALTER TABLE refresh_tokens ALTER COLUMN is_revoked DROP DEFAULT") - op.execute("ALTER TABLE refresh_tokens ALTER COLUMN token_hash TYPE VARCHAR(256)") + op.execute( + "ALTER TABLE refresh_tokens ALTER COLUMN token_hash TYPE VARCHAR(256)" + ) op.execute("ALTER TABLE refresh_tokens DROP COLUMN IF EXISTS absolute_expiry") op.execute("ALTER TABLE refresh_tokens DROP COLUMN IF EXISTS revoked_at") diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index c4ee33c30..9e69d6d53 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -762,7 +762,7 @@ class Config: TURNSTILE_SECRET_KEY = os.getenv("TURNSTILE_SECRET_KEY", "") # Auth - AUTH_TYPE = os.getenv("AUTH_TYPE") + AUTH_TYPE = os.getenv("AUTH_TYPE", "LOCAL") REGISTRATION_ENABLED = os.getenv("REGISTRATION_ENABLED", "TRUE").upper() == "TRUE" # Google OAuth @@ -916,7 +916,7 @@ class Config: # JWT Token Lifetimes ACCESS_TOKEN_LIFETIME_SECONDS = int( - os.getenv("ACCESS_TOKEN_LIFETIME_SECONDS", str(30 * 60)) # 30 minutes + os.getenv("ACCESS_TOKEN_LIFETIME_SECONDS", str(60 * 60)) # 60 minutes ) MIN_ISSUED_AT = int(os.getenv("MIN_ISSUED_AT", "0")) REFRESH_TOKEN_LIFETIME_SECONDS = int( diff --git a/surfsense_backend/app/tasks/document_processors/_save.py b/surfsense_backend/app/tasks/document_processors/_save.py index e30666a20..2509a70ef 100644 --- a/surfsense_backend/app/tasks/document_processors/_save.py +++ b/surfsense_backend/app/tasks/document_processors/_save.py @@ -10,6 +10,7 @@ from app.utils.document_converters import ( create_document_chunks, embed_text, generate_content_hash, + truncate_for_embedding, ) from ._helpers import ( @@ -73,7 +74,9 @@ async def save_file_document( if should_skip: return doc - document_content = f"File: {file_name}\n\n{markdown_content[:4000]}" + document_content = ( + f"File: {file_name}\n\n{truncate_for_embedding(markdown_content)}" + ) document_embedding = embed_text(document_content) chunks = await create_document_chunks(markdown_content) doc_metadata = {"FILE_NAME": file_name, "ETL_SERVICE": etl_service} diff --git a/surfsense_backend/app/tasks/document_processors/markdown_processor.py b/surfsense_backend/app/tasks/document_processors/markdown_processor.py index 9e31327d9..8632e212b 100644 --- a/surfsense_backend/app/tasks/document_processors/markdown_processor.py +++ b/surfsense_backend/app/tasks/document_processors/markdown_processor.py @@ -13,6 +13,7 @@ from app.utils.document_converters import ( create_document_chunks, embed_text, generate_content_hash, + truncate_for_embedding, ) from ._helpers import ( @@ -182,7 +183,9 @@ async def add_received_markdown_file_document( return doc # Content changed - continue to update - summary_content = f"File: {file_name}\n\n{file_in_markdown[:4000]}" + summary_content = ( + f"File: {file_name}\n\n{truncate_for_embedding(file_in_markdown)}" + ) summary_embedding = embed_text(summary_content) # Process chunks diff --git a/surfsense_backend/app/utils/blocknote_to_markdown.py b/surfsense_backend/app/utils/blocknote_to_markdown.py index e26a9f4ee..38393a370 100644 --- a/surfsense_backend/app/utils/blocknote_to_markdown.py +++ b/surfsense_backend/app/utils/blocknote_to_markdown.py @@ -145,7 +145,7 @@ def _render_block( code_text = _render_inline_content(content) if content else "" lines.append(f"{prefix}```{language}") for code_line in code_text.split("\n"): - lines.append(f"{prefix}{code_line}") + lines.append(code_line) lines.append(f"{prefix}```") elif block_type == "table": diff --git a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx index 9c3a7c617..64971c3bb 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx @@ -71,7 +71,7 @@ import { useMessagesSync } from "@/hooks/use-messages-sync"; import { useThreadDetail, useThreadMessages } from "@/hooks/use-thread-queries"; import { getAgentFilesystemSelection } from "@/lib/agent-filesystem"; import { documentsApiService } from "@/lib/apis/documents-api.service"; -import { getDesktopAccessToken } from "@/lib/auth-fetch"; +import { authenticatedFetch } from "@/lib/auth-fetch"; import { type ChatFlow, classifyChatError } from "@/lib/chat/chat-error-classifier"; import { tagPreAcceptSendFailure, toHttpResponseError } from "@/lib/chat/chat-request-errors"; import { getMentionDocKey } from "@/lib/chat/mention-doc-key"; @@ -932,14 +932,11 @@ export default function NewChatPage() { // Cancel ongoing request const cancelRun = useCallback(async () => { if (threadId) { - const token = await getDesktopAccessToken(); try { - const response = await fetch( + const response = await authenticatedFetch( buildBackendUrl(`/api/v1/threads/${threadId}/cancel-active-turn`), { method: "POST", - headers: token ? { Authorization: `Bearer ${token}` } : undefined, - credentials: "include", } ); if (response.ok) { @@ -986,8 +983,6 @@ export default function NewChatPage() { if (!userQuery.trim() && userImages.length === 0) return; - const token = await getDesktopAccessToken(); - // Lazy thread creation: create thread on first message if it doesn't exist let currentThreadId = threadId; let isNewThread = false; @@ -1159,13 +1154,11 @@ export default function NewChatPage() { const hasThreadIds = mentionPayload.thread_ids.length > 0; const response = await fetchWithTurnCancellingRetry(() => - fetch(buildBackendUrl("/api/v1/new_chat"), { + authenticatedFetch(buildBackendUrl("/api/v1/new_chat"), { method: "POST", headers: { "Content-Type": "application/json", - ...(token ? { Authorization: `Bearer ${token}` } : {}), }, - credentials: "include", body: JSON.stringify({ chat_id: currentThreadId, user_query: userQuery.trim(), @@ -1546,8 +1539,6 @@ export default function NewChatPage() { stagedDecisionsByInterruptIdRef.current.clear(); setIsRunning(true); - const token = await getDesktopAccessToken(); - const controller = new AbortController(); abortControllerRef.current = controller; @@ -1648,13 +1639,11 @@ export default function NewChatPage() { localFilesystemEnabled, }); const response = await fetchWithTurnCancellingRetry(() => - fetch(buildBackendUrl(`/api/v1/threads/${resumeThreadId}/resume`), { + authenticatedFetch(buildBackendUrl(`/api/v1/threads/${resumeThreadId}/resume`), { method: "POST", headers: { "Content-Type": "application/json", - ...(token ? { Authorization: `Bearer ${token}` } : {}), }, - credentials: "include", body: JSON.stringify({ search_space_id: searchSpaceId, decisions, @@ -1986,8 +1975,6 @@ export default function NewChatPage() { abortControllerRef.current = null; } - const token = await getDesktopAccessToken(); - // Extract the original user query BEFORE removing messages (for reload mode) let userQueryToDisplay: string | undefined; let originalUserMessageContent: ThreadMessageLike["content"] | null = null; @@ -2105,13 +2092,11 @@ export default function NewChatPage() { } } const response = await fetchWithTurnCancellingRetry(() => - fetch(getRegenerateUrl(threadId), { + authenticatedFetch(getRegenerateUrl(threadId), { method: "POST", headers: { "Content-Type": "application/json", - ...(token ? { Authorization: `Bearer ${token}` } : {}), }, - credentials: "include", body: JSON.stringify(requestBody), signal: controller.signal, }) diff --git a/surfsense_web/components/assistant-ui/inline-mention-editor.tsx b/surfsense_web/components/assistant-ui/inline-mention-editor.tsx index 5fc942e54..ee2d1c873 100644 --- a/surfsense_web/components/assistant-ui/inline-mention-editor.tsx +++ b/surfsense_web/components/assistant-ui/inline-mention-editor.tsx @@ -696,7 +696,9 @@ export const InlineMentionEditor = forwardRef { // Arrow / Enter / Escape navigation for the active picker. const handleKeyDown = useCallback( (e: React.KeyboardEvent) => { + // While an IME composition is active (e.g. confirming a Japanese/Chinese/ + // Korean conversion), let the Enter/Arrow keys reach the IME instead of + // driving picker navigation/selection. + if (e.nativeEvent.isComposing) return; if (showPromptPicker) { if (e.key === "ArrowDown") { e.preventDefault(); diff --git a/surfsense_web/components/free-chat/free-composer.tsx b/surfsense_web/components/free-chat/free-composer.tsx index 162b906ad..d4523a4f9 100644 --- a/surfsense_web/components/free-chat/free-composer.tsx +++ b/surfsense_web/components/free-chat/free-composer.tsx @@ -99,7 +99,9 @@ export const FreeComposer: FC = () => { gate("mention documents"); return; } - if (e.key === "Enter" && !e.shiftKey) { + // Ignore Enter while an IME composition is active (e.g. confirming a + // Japanese/Chinese/Korean conversion) so it doesn't submit the message. + if (e.key === "Enter" && !e.shiftKey && !e.nativeEvent.isComposing) { e.preventDefault(); if (text.trim()) { aui.composer().send(); diff --git a/surfsense_web/components/providers/ZeroProvider.tsx b/surfsense_web/components/providers/ZeroProvider.tsx index 1a157a854..cfb8fe586 100644 --- a/surfsense_web/components/providers/ZeroProvider.tsx +++ b/surfsense_web/components/providers/ZeroProvider.tsx @@ -8,7 +8,7 @@ import { import { usePathname } from "next/navigation"; import { useEffect, useMemo, useRef, useState } from "react"; import { useSession } from "@/hooks/use-session"; -import { getDesktopAccessToken } from "@/lib/auth-fetch"; +import { authenticatedFetch, getDesktopAccessToken } from "@/lib/auth-fetch"; import { handleUnauthorized, isPublicRoute, refreshSession } from "@/lib/auth-utils"; import { buildBackendUrl } from "@/lib/env-config"; import type { Context } from "@/types/zero"; @@ -31,26 +31,14 @@ function getCacheURL() { } async function fetchZeroContext(isDesktop: boolean): Promise { - const headers: HeadersInit = {}; - let desktopAuth: string | undefined; - - if (isDesktop) { - const token = await getDesktopAccessToken(); - if (!token) return null; - desktopAuth = token; - headers.Authorization = `Bearer ${token}`; - } - - const response = await fetch(buildBackendUrl("/zero/context"), { - credentials: "include", - headers, + const response = await authenticatedFetch(buildBackendUrl("/zero/context"), { + skipAuthRedirect: true, }); - if (!response.ok) return null; return { context: (await response.json()) as ZeroContext, - desktopAuth, + desktopAuth: isDesktop ? (await getDesktopAccessToken()) || undefined : undefined, }; } @@ -106,7 +94,7 @@ function ZeroAuthSync({ isDesktop }: { isDesktop: boolean }) { } if (isDesktop) { - const newToken = await getDesktopAccessToken(); + const newToken = await getDesktopAccessToken({ forceRefresh: true }); if (!newToken) { handleUnauthorized(); return; @@ -262,7 +250,7 @@ export function ZeroProvider({ children }: { children: React.ReactNode }) { const pathname = usePathname(); const isDesktop = typeof window !== "undefined" && !!window.electronAPI; - if (!isDesktop && isPublicRoute(pathname)) { + if (isPublicRoute(pathname)) { return <>{children}; } diff --git a/surfsense_web/components/tool-ui/sandbox-execute.tsx b/surfsense_web/components/tool-ui/sandbox-execute.tsx index 535968908..2bac3e1ca 100644 --- a/surfsense_web/components/tool-ui/sandbox-execute.tsx +++ b/surfsense_web/components/tool-ui/sandbox-execute.tsx @@ -16,7 +16,7 @@ import { z } from "zod"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"; -import { getDesktopAccessToken } from "@/lib/auth-fetch"; +import { authenticatedFetch } from "@/lib/auth-fetch"; import { buildBackendUrl } from "@/lib/env-config"; import { cn } from "@/lib/utils"; @@ -157,14 +157,10 @@ function truncateCommand(command: string, maxLen = 80): string { // ============================================================================ async function downloadSandboxFile(threadId: string, filePath: string, fileName: string) { - const token = await getDesktopAccessToken(); const url = buildBackendUrl(`/api/v1/threads/${threadId}/sandbox/download`, { path: filePath, }); - const res = await fetch(url, { - headers: token ? { Authorization: `Bearer ${token}` } : undefined, - credentials: "include", - }); + const res = await authenticatedFetch(url); if (!res.ok) { throw new Error(`Download failed: ${res.statusText}`); } diff --git a/surfsense_web/hooks/use-session.ts b/surfsense_web/hooks/use-session.ts index 6bb10456f..207475eae 100644 --- a/surfsense_web/hooks/use-session.ts +++ b/surfsense_web/hooks/use-session.ts @@ -1,6 +1,7 @@ "use client"; import { useCallback, useEffect, useState } from "react"; +import { authenticatedFetch } from "@/lib/auth-fetch"; import { buildBackendUrl } from "@/lib/env-config"; type SessionState = @@ -8,15 +9,6 @@ type SessionState = | { status: "authenticated"; authenticated: true; accessExpiresAt: number | null } | { status: "unauthenticated"; authenticated: false; accessExpiresAt: null }; -async function getSessionHeaders(): Promise { - if (typeof window === "undefined" || !window.electronAPI?.getAccessToken) { - return {}; - } - - const token = await window.electronAPI.getAccessToken(); - return token ? { Authorization: `Bearer ${token}` } : {}; -} - export function useSession() { const [state, setState] = useState({ status: "loading", @@ -26,9 +18,8 @@ export function useSession() { const refresh = useCallback(async () => { try { - const response = await fetch(buildBackendUrl("/auth/session"), { - credentials: "include", - headers: await getSessionHeaders(), + const response = await authenticatedFetch(buildBackendUrl("/auth/session"), { + skipAuthRedirect: true, }); if (!response.ok) { setState({ diff --git a/surfsense_web/lib/apis/base-api.service.ts b/surfsense_web/lib/apis/base-api.service.ts index 5afb291ba..0cc5224e2 100644 --- a/surfsense_web/lib/apis/base-api.service.ts +++ b/surfsense_web/lib/apis/base-api.service.ts @@ -1,4 +1,5 @@ import type { ZodType } from "zod"; +import { getDesktopAccessToken } from "@/lib/auth-fetch"; import { buildBackendUrl } from "@/lib/env-config"; import { getClientPlatform } from "../agent-filesystem"; import { handleUnauthorized, refreshSession } from "../auth-utils"; @@ -59,11 +60,6 @@ class BaseApiService { return typeof window !== "undefined" && !!window.electronAPI; } - private async getDesktopAccessToken(): Promise { - if (!this.isDesktopClient) return ""; - return (await window.electronAPI?.getAccessToken?.()) || ""; - } - async request( url: string, responseSchema?: ZodType, @@ -90,7 +86,7 @@ class BaseApiService { this.noAuthPrefixes.some((prefix) => url.startsWith(prefix)) || /^\/api\/v1\/invites\/[^/]+\/info$/.test(url); const desktopAccessToken = - this.isDesktopClient && !isNoAuthEndpoint ? await this.getDesktopAccessToken() : ""; + this.isDesktopClient && !isNoAuthEndpoint ? (await getDesktopAccessToken()) || "" : ""; const defaultOptions: RequestOptions = { headers: { ...(desktopAccessToken ? { Authorization: `Bearer ${desktopAccessToken}` } : {}), @@ -174,7 +170,9 @@ class BaseApiService { } else if (!isNoAuthEndpoint && !isRefreshRetryBlocked(refreshRetryKey)) { const refreshed = await refreshSession(); if (refreshed) { - const newToken = this.isDesktopClient ? await this.getDesktopAccessToken() : ""; + const newToken = this.isDesktopClient + ? (await getDesktopAccessToken({ forceRefresh: true })) || "" + : ""; return this.request(url, responseSchema, { ...mergedOptions, headers: { diff --git a/surfsense_web/lib/apis/notes-api.service.ts b/surfsense_web/lib/apis/notes-api.service.ts deleted file mode 100644 index eac3d96ed..000000000 --- a/surfsense_web/lib/apis/notes-api.service.ts +++ /dev/null @@ -1,147 +0,0 @@ -import { z } from "zod"; -import { ValidationError } from "../error"; -import { baseApiService } from "./base-api.service"; - -// Request/Response schemas -const createNoteRequest = z.object({ - search_space_id: z.number(), - title: z.string().min(1), - source_markdown: z.string().optional(), -}); - -const createNoteResponse = z.object({ - id: z.number(), - title: z.string(), - document_type: z.string(), - content: z.string(), - content_hash: z.string(), - unique_identifier_hash: z.string().nullable(), - document_metadata: z.record(z.string(), z.any()).nullable(), - search_space_id: z.number(), - created_at: z.string(), - updated_at: z.string().nullable(), -}); - -const getNotesRequest = z.object({ - search_space_id: z.number(), - skip: z.number().optional(), - page: z.number().optional(), - page_size: z.number().optional(), -}); - -const noteItem = z.object({ - id: z.number(), - title: z.string(), - document_type: z.string(), - content: z.string(), - content_hash: z.string(), - unique_identifier_hash: z.string().nullable(), - document_metadata: z.record(z.string(), z.any()).nullable(), - search_space_id: z.number(), - created_at: z.string(), - updated_at: z.string().nullable(), -}); - -const getNotesResponse = z.object({ - items: z.array(noteItem), - total: z.number(), - page: z.number(), - page_size: z.number(), - has_more: z.boolean(), -}); - -const deleteNoteRequest = z.object({ - search_space_id: z.number(), - note_id: z.number(), -}); - -const deleteNoteResponse = z.object({ - message: z.string(), - note_id: z.number(), -}); - -// Type exports -export type CreateNoteRequest = z.infer; -export type CreateNoteResponse = z.infer; -export type GetNotesRequest = z.infer; -export type GetNotesResponse = z.infer; -export type NoteItem = z.infer; -export type DeleteNoteRequest = z.infer; -export type DeleteNoteResponse = z.infer; - -class NotesApiService { - /** - * Create a new note - */ - createNote = async (request: CreateNoteRequest) => { - const parsedRequest = createNoteRequest.safeParse(request); - - if (!parsedRequest.success) { - console.error("Invalid request:", parsedRequest.error); - const errorMessage = parsedRequest.error.issues.map((issue) => issue.message).join(", "); - throw new ValidationError(`Invalid request: ${errorMessage}`); - } - - const { search_space_id, title, source_markdown } = parsedRequest.data; - - // Send both title and source_markdown in request body - const body = { - title, - ...(source_markdown !== undefined && { source_markdown }), - }; - - return baseApiService.post( - `/api/v1/search-spaces/${search_space_id}/notes`, - createNoteResponse, - { body } - ); - }; - - /** - * Get list of notes - */ - getNotes = async (request: GetNotesRequest) => { - const parsedRequest = getNotesRequest.safeParse(request); - - if (!parsedRequest.success) { - console.error("Invalid request:", parsedRequest.error); - const errorMessage = parsedRequest.error.issues.map((issue) => issue.message).join(", "); - throw new ValidationError(`Invalid request: ${errorMessage}`); - } - - const { search_space_id, skip, page, page_size } = parsedRequest.data; - - // Build query params - const params = new URLSearchParams(); - if (skip !== undefined) params.append("skip", String(skip)); - if (page !== undefined) params.append("page", String(page)); - if (page_size !== undefined) params.append("page_size", String(page_size)); - - return baseApiService.get( - `/api/v1/search-spaces/${search_space_id}/notes?${params.toString()}`, - getNotesResponse - ); - }; - - /** - * Delete a note - */ - deleteNote = async (request: DeleteNoteRequest) => { - const parsedRequest = deleteNoteRequest.safeParse(request); - - if (!parsedRequest.success) { - console.error("Invalid request:", parsedRequest.error); - const errorMessage = parsedRequest.error.issues.map((issue) => issue.message).join(", "); - throw new ValidationError(`Invalid request: ${errorMessage}`); - } - - const { search_space_id, note_id } = parsedRequest.data; - - return baseApiService.delete( - `/api/v1/search-spaces/${search_space_id}/notes/${note_id}`, - deleteNoteResponse - ); - }; -} - -export const notesApiService = new NotesApiService(); diff --git a/surfsense_web/lib/auth-fetch.ts b/surfsense_web/lib/auth-fetch.ts index 20b236854..a79777825 100644 --- a/surfsense_web/lib/auth-fetch.ts +++ b/surfsense_web/lib/auth-fetch.ts @@ -3,6 +3,16 @@ import { handleUnauthorized, isDesktopClient, refreshSession } from "@/lib/auth- let desktopAccessToken: string | null = null; let didSubscribeToDesktopAuth = false; +type DesktopAccessTokenOptions = { + forceRefresh?: boolean; +}; + +type AuthenticatedFetchOptions = RequestInit & { + skipAuthRedirect?: boolean; + skipRefresh?: boolean; + forceDesktopTokenRefresh?: boolean; +}; + function subscribeToDesktopAuth(): void { if (didSubscribeToDesktopAuth || typeof window === "undefined" || !window.electronAPI) { return; @@ -17,10 +27,12 @@ function subscribeToDesktopAuth(): void { }); } -export async function getDesktopAccessToken(): Promise { +export async function getDesktopAccessToken( + options: DesktopAccessTokenOptions = {} +): Promise { if (!isDesktopClient()) return null; subscribeToDesktopAuth(); - if (desktopAccessToken) return desktopAccessToken; + if (desktopAccessToken && !options.forceRefresh) return desktopAccessToken; const token = (await window.electronAPI?.getAccessToken?.()) || null; desktopAccessToken = token; return token; @@ -34,42 +46,61 @@ export function getAuthHeaders(additionalHeaders?: Record): Reco }; } +async function fetchWithAuth( + url: string, + options: RequestInit, + { forceDesktopTokenRefresh = false }: { forceDesktopTokenRefresh?: boolean } = {} +): Promise { + const headers = new Headers(options.headers); + const token = await getDesktopAccessToken({ forceRefresh: forceDesktopTokenRefresh }); + if (token) { + headers.set("Authorization", `Bearer ${token}`); + } + + return fetch(url, { + ...options, + headers, + credentials: options.credentials ?? "include", + }); +} + export async function authenticatedFetch( url: string, - options?: RequestInit & { skipAuthRedirect?: boolean; skipRefresh?: boolean } + options: AuthenticatedFetchOptions = {} ): Promise { - const { skipAuthRedirect = false, skipRefresh = false, ...fetchOptions } = options || {}; - const token = await getDesktopAccessToken(); - const headers = { - ...(fetchOptions.headers as Record), - ...(token ? { Authorization: `Bearer ${token}` } : {}), - }; + const { + skipAuthRedirect = false, + skipRefresh = false, + forceDesktopTokenRefresh = false, + ...fetchOptions + } = options; - const response = await fetch(url, { - ...fetchOptions, - headers, - credentials: "include", + const response = await fetchWithAuth(url, fetchOptions, { + forceDesktopTokenRefresh, }); - if (response.status === 401 && !skipAuthRedirect) { - if (!skipRefresh) { - const refreshed = await refreshSession(); - if (refreshed) { - const newToken = await getDesktopAccessToken(); - return fetch(url, { - ...fetchOptions, - headers: { - ...(fetchOptions.headers as Record), - ...(newToken ? { Authorization: `Bearer ${newToken}` } : {}), - }, - credentials: "include", - }); - } - } + if (response.status !== 401) { + return response; + } + let unauthorizedResponse = response; + if (!skipRefresh) { + const refreshed = await refreshSession(); + if (refreshed) { + const retryResponse = await fetchWithAuth(url, fetchOptions, { + forceDesktopTokenRefresh: true, + }); + if (retryResponse.status !== 401) { + return retryResponse; + } + unauthorizedResponse = retryResponse; + } + } + + if (!skipAuthRedirect) { handleUnauthorized(); throw new Error("Unauthorized: Redirecting to login page"); } - return response; + return unauthorizedResponse; } diff --git a/surfsense_web/lib/auth-utils.ts b/surfsense_web/lib/auth-utils.ts index 47b2f043f..7372c8b66 100644 --- a/surfsense_web/lib/auth-utils.ts +++ b/surfsense_web/lib/auth-utils.ts @@ -188,9 +188,23 @@ async function doRefreshSession(): Promise { } } +let refreshPromise: Promise | null = null; + export async function refreshSession(): Promise { - if (typeof navigator !== "undefined" && "locks" in navigator) { - return navigator.locks.request("ss-token-refresh", () => doRefreshSession()); + if (refreshPromise) { + return refreshPromise; + } + + refreshPromise = (async () => { + if (typeof navigator !== "undefined" && "locks" in navigator) { + return navigator.locks.request("ss-token-refresh", () => doRefreshSession()); + } + return doRefreshSession(); + })(); + + try { + return await refreshPromise; + } finally { + refreshPromise = null; } - return doRefreshSession(); } diff --git a/surfsense_web/lib/env-config.ts b/surfsense_web/lib/env-config.ts index 8c671029c..c60ada372 100644 --- a/surfsense_web/lib/env-config.ts +++ b/surfsense_web/lib/env-config.ts @@ -8,9 +8,9 @@ import packageJson from "../package.json"; -// Build-time fallback for packaged clients. Docker runtime reads plain AUTH_TYPE -// through the runtime config provider first, then falls back to this baked value. -export const BUILD_TIME_AUTH_TYPE = process.env.NEXT_PUBLIC_AUTH_TYPE || "GOOGLE"; +// Build-time fallback for packaged clients. Docker/runtime deployments default to +// local auth unless AUTH_TYPE/NEXT_PUBLIC_AUTH_TYPE explicitly selects Google. +export const BUILD_TIME_AUTH_TYPE = process.env.NEXT_PUBLIC_AUTH_TYPE || "LOCAL"; // Backend API URL. An empty string is valid in proxy mode and means // same-origin relative requests (e.g. /api/v1/... and /auth/...). diff --git a/surfsense_web/lib/runtime-auth-config.ts b/surfsense_web/lib/runtime-auth-config.ts index 9e8d1921d..d619c5957 100644 --- a/surfsense_web/lib/runtime-auth-config.ts +++ b/surfsense_web/lib/runtime-auth-config.ts @@ -4,7 +4,7 @@ export type RuntimeAuthUiMode = "GOOGLE" | "LOCAL"; export function resolveRuntimeAuthUiMode( value: string | null | undefined, - fallback: string | null | undefined = "GOOGLE" + fallback: string | null | undefined = "LOCAL" ): RuntimeAuthUiMode { const candidate = value?.trim().toUpperCase(); if (candidate === "GOOGLE") return "GOOGLE";