Merge commit '683a7d3bc2' into feat/rename-searchspace-to-workspace

docker/proxy/Caddyfile

@@ -31,6 +31,10 @@
 		flush_interval -1
 	}
 
+	# Zero sync auth context is a backend (FastAPI) endpoint. More specific than
+	# /zero/*, so Caddy's matcher-specificity sort routes it here, not to zero-cache.
+	reverse_proxy /zero/context backend:8000
+
 	# Zero accepts a single path-component base URL (Zero >= 0.6).
 	# Preserve /zero so browser cacheURL can be ${SURFSENSE_PUBLIC_URL}/zero.
 	reverse_proxy /zero/* zero-cache:4848

docker/scripts/install.ps1

@@ -330,11 +330,13 @@ Write-Info "Installation directory: $InstallDir"
 
 New-Item -ItemType Directory -Path "$InstallDir\scripts" -Force | Out-Null
 New-Item -ItemType Directory -Path "$InstallDir\searxng" -Force | Out-Null
+New-Item -ItemType Directory -Path "$InstallDir\proxy" -Force | Out-Null
 
 $Files = @(
     @{ Src = "docker/docker-compose.yml";                Dest = "docker-compose.yml" }
     @{ Src = "docker/docker-compose.gpu.yml";            Dest = "docker-compose.gpu.yml" }
     @{ Src = "docker/.env.example";                      Dest = ".env.example" }
+    @{ Src = "docker/proxy/Caddyfile";                   Dest = "proxy/Caddyfile" }
     @{ Src = "docker/postgresql.conf";                   Dest = "postgresql.conf" }
     @{ Src = "docker/scripts/migrate-database.ps1";      Dest = "scripts/migrate-database.ps1" }
     @{ Src = "docker/searxng/settings.yml";              Dest = "searxng/settings.yml" }

plans/backend/00-umbrella-plan.md

@@ -40,7 +40,7 @@ flowchart TD
 - MCP-availability audit complete: BookStack (community MCP servers), Elasticsearch (official Elastic Agent Builder MCP), and Luma (community MCP servers) all have MCP available, so none are `DISABLED` — they're tagged `MIGRATING` (turned off for MVP like the other branded natives, pending the post-MVP MCP re-point).
 - `Pipeline` and `PipelineRun` are new first-class tables. A Pipeline references a connector + config + schedule + KB destination. File upload creates/uses a pipeline and registers a run; uploads always save to KB.
 - The chat agent gets read-only access to pipeline run history (pipelines + their recent runs/status) as context, so it can reason about what was fetched, when, and whether runs succeeded — even for data not saved to the KB.
-- Deferred (post-MVP): platform scraper implementations, public pay-as-you-go API for Type-1 connectors, public MCP server exposing the KB.
+- Deferred (post-MVP): platform scraper implementations (**Phase 8 — Platform actors**, public-data-only first; built in-house on the Phase-3 fetch core), public pay-as-you-go API for Type-1 connectors, public MCP server exposing the KB. Logged-in/account-based scraping deferred beyond that.
 
 ## Platform connector research list (deferred build, MVP = "coming soon")
 
@@ -69,14 +69,18 @@ flowchart TD
 - Consolidate API to `/workspaces` and fix the `/searchspaces` vs `/search-spaces` inconsistency.
 - High-touch files: `routes/search_spaces_routes.py`, `routes/rbac_routes.py`, `utils/rbac.py` (`check_search_space_access`), `schemas/search_space.py`, plus `search_space_id` threading through agents/Redis keys/storage paths (`documents/{id}/...`).
 
-### Phase 3 — WebURL Crawler & Crawl Billing (backend) [`subplans: 03a–03d`]
+### Phase 3 — WebURL Crawler & Crawl Billing (backend) [`subplans: 03a–03f`]
 
-The Universal WebURL Crawler is the flagship Type-1 data source (the moat). This phase hardens it on a single framework (Scrapling), generalizes proxy support, introduces pay-as-you-go crawl credits, and (deferred) adds opt-in captcha solving. It is broken into focused subplans:
+The Universal WebURL Crawler is the flagship Type-1 data source (**the moat**). This phase hardens it into an **in-house, best-effort "undetectable, captcha-bypassing" crawler** on a single framework (Scrapling): it standardizes the fetch layer, generalizes proxy support, introduces pay-as-you-go crawl credits, adds **stealth hardening** (geoip coherence, persistent profiles, headed/Xvfb, fonts, humanization, a block classifier + per-domain strategy memory), and adds **opt-in captcha solving**. All tiers plug in behind a single `FetchStrategy` seam returning `CrawlOutcome`, so callers never depend on *how* a page was fetched — that seam is what lets the moat grow (and lets a **deferred paid-unblocker tier** drop in later by config). **Strategy decision (recorded in the log):** CloakBrowser is rejected on licensing and external unblocker APIs are deferred; we hold an in-house bypass moat for ~4–6 months, then move hostile targets to a paid tier if demand/maintenance justifies it. **Logged-in/account-based bypass is out of scope** (public data only this MVP). It is broken into focused subplans:
+
+> **Sequencing within Phase 3 (critical path vs hardening).** Only **`03a` + `03b` + `03c`** are on the **MVP critical path** — Phases 4–7 (connector taxonomy, pipelines, execution) consume the `03a` `CrawlOutcome`/`FetchStrategy` contract, the `03b` proxy provider, and the `03c` `WebCrawlCreditService` billing seam. **`03d` (captcha), `03e` (stealth hardening), and `03f` (test harness) are hardening/measurement that nothing downstream imports** — they tune the *same* seam behind the *same* contract, so they can land **in parallel with, or after, Phases 4–7** without blocking the pivot. Recommended build order: `03a → 03b → 03c` (then proceed to Phase 4), with `03e → 03d → 03f` slotted in whenever crawler robustness is prioritized. The dependency notes inside `03d`/`03e`/`03f` (each requires `03a`/`03b`) still hold; this only frees them from gating Phase 4+.
 
 - **`03a-crawler-core.md`** — Standardize the fetch layer on Scrapling. **Remove Firecrawl entirely** (no other frameworks). Define crisp per-URL success/empty/failure semantics, keep Trafilatura extraction, and expose a single billable "successful crawl" signal (one unit per URL that yields usable content, regardless of how many internal fallback tiers ran).
 - **`03b-proxy-expansion.md`** — Add a BYO `CustomProxyProvider` (the only new provider — **no branded vendors**) alongside `anonymous_proxies`, selectable via a **single, app-wide** `Config.PROXY_PROVIDER`. Add bounded client-side rotation+retry via Scrapling's `ProxyRotator`/`is_proxy_error` **only** when the active provider is pool-backed (`CUSTOM_PROXY_URLS`); single-endpoint providers (incl. `anonymous_proxies`) stay the default and no-op the retry. **No per-connector/per-crawl selection** (one provider app-wide); a per-pipeline override is left as a no-op seam for Phase 5/6.
 - **`03c-crawl-billing.md`** — Charge crawl credits at **$1 / 1000 successful requests = 1000 micro-USD per successful crawl**, drawn from the existing credit wallet (`credit_micros_balance`), gated by a new `WEB_CRAWL_CREDIT_BILLING_ENABLED` flag (off for self-hosted). Two surfaces: **connector/pipeline crawls** billed to the **workspace owner** via a dedicated `WebCrawlCreditService` (mirrors `EtlCreditService`'s gate → `check_credits` → `charge_credits`, **not** `billable_call`); **chat scrapes** fold their crawl cost into the chat turn's existing bill (turn accumulator). No DB migration (uses the existing free-form `web_crawl` usage_type).
-- **`03d-captcha-solving.md`** *(DEFERRED — sequenced last, non-MVP-blocking)* — Covers the captcha types Scrapling does **not** (reCAPTCHA v2/v3, hCaptcha, image) via `captchatools`. `captchatools` is **itself** the provider registry (`new_harvester(solving_site=…)` across capmonster/2captcha/anticaptcha/capsolver/captchaai), so we do **not** rebuild a provider hierarchy — our layer is thin: config resolution + a StealthyFetcher `page_action` that detects the sitekey, harvests a token, and injects it. Scrapling already handles Cloudflare Turnstile (`03a`). Flags the **billing asymmetry** (solvers charge per *attempt*, `03c` bills per *success*) for resolution at build time. Requires a paid solver account.
+- **`03e-stealth-hardening.md`** — The in-house "undetectable" layer on top of Scrapling's default patchright-Chromium stealth. **Geoip coherence** (match browser `locale`/`timezone_id` to the proxy's exit geo), **fingerprint flags** (`hide_canvas`/`block_webrtc`), **persistent per-domain profiles** (`user_data_dir`), **headed execution under Xvfb**, **real fonts** in the worker image, and **DIY behavioral humanization** via `page_action` (the Chromium engine has no built-in `humanize`). Adds a **block classifier** (label Cloudflare/DataDome/Kasada/captcha/empty from the response) + **per-domain strategy memory** (Redis, no migration) so the ladder learns the known-good tier per domain. Defines — but does **not** build — the **deferred paid-unblocker `FetchStrategy`** (ZenRows/ScrapFly/Bright Data) as the config-flagged escape hatch, with its own (later) billing. Honest ceiling: defeats Cloudflare + the moderate long tail, **not** top-tier behavioral fingerprinting (DataDome/Kasada/reCAPTCHA-Enterprise) — that's the deferred tier.
+- **`03d-captcha-solving.md`** *(ACTIVE — sequenced last in Phase 3, after `03e`)* — Covers the captcha types Scrapling does **not** (reCAPTCHA v2/v3, hCaptcha, image) via `captchatools`, **opt-in + off by default**. `captchatools` is **itself** the provider registry (`new_harvester(solving_site=…)` across capmonster/2captcha/anticaptcha/capsolver/captchaai), so we do **not** rebuild a provider hierarchy — our layer is thin: config resolution + a StealthyFetcher `page_action` that detects the sitekey, harvests a token (egressing from the **same** proxy IP as the crawl), and injects it. Scrapling already handles Cloudflare Turnstile (`03a`), and `page_action` runs **after** `solve_cloudflare`, so the tiers compose. The **billing asymmetry** is now **resolved**: a **separate per-attempt** `web_crawl_captcha` unit (solvers charge per attempt regardless of crawl success), attached via a `WebCrawlCreditService` seam in `03c`. `ErrNoBalance` stops solving (no retry loop → avoids IP bans). Requires a paid solver account.
+- **`03f-undetectability-testing.md`** *(MANUAL-only — no CI gating; sequenced last)* — A **manual scorecard harness** that drives the real Scrapling tiers against the industry-standard detection + sandbox sites (modeled on CloakBrowser's `bin/cloaktest` suite) to **quantify the free-stack ceiling** over time. Two labeled axes: **Suite S (stealth/anti-bot)** — browser-tier (`bot.sannysoft`, `bot.incolumitas`, CreepJS, `deviceandbrowserinfo`, FingerprintJS demo, reCAPTCHA-v3 score, `fingerprint-scan`/Castle.js, `browserscan`), HTTP/TLS-tier JA3/JA4 parity (`tls.peet.ws`, **informational** not a gate), and proxy/leak checks (`httpbin/ip`, WebRTC/DNS); **Suite E (extraction correctness)** — toscrape/scrapethissite sandboxes for the HTTP vs JS (DynamicFetcher) tiers. Reuses `03d`'s `page_action`+closure-cell for JS-object verdicts. Adopts CloakBrowser's bars as **aspirational** (sannysoft 0 fails, CreepJS ≤30%, reCAPTCHA ≥0.7) while recording **our actual numbers as the committed baseline**; the scorecard is the documented **trigger** for flipping `03e`'s deferred paid-unblocker tier.
 
 ### Phase 4 — Connector two-type restructure (backend) [`subplans: 04a–04b`]
 
@@ -114,6 +118,13 @@ Split into two independently testable subplans:
 - A small migration adds a partial unique index `ON pipelines(workspace_id) WHERE connector_id IS NULL` (enforces the singleton + race-safe get-or-create).
 - KB-save-secondary: the opt-in `save_to_kb` + destination folder for connector pipelines already shipped in Phases 5–6; Phase 7 only records the inverse invariant (uploads always KB) and guards the connector default stays `False`.
 
+### Phase 8 — Platform actors (FUTURE — post-MVP, public data only) [`subplan: TBD`]
+
+NOT planned in this umbrella; recorded so the Phase-3 architecture stays aimed at it. Once the hardened fetch core (Phases 3a–3e) is solid, **platform actors** layer **on top** of it: per-platform structured extractors (Google Maps/Local, LinkedIn public profiles/companies, Amazon products, etc. — see "Platform connector research list" above), built in-house "Apify-style" rather than via third-party paid actors. They reuse the existing machinery end-to-end: each actor is a **Type-1 data-source connector** (Phase 4 taxonomy), runs via **Pipelines** (Phases 5–6, manual/cron), saves to the **KB** (opt-in folder), and is **billed per run** (extending `03c`/`06`). They consume the `03a` `FetchStrategy` core (proxies + `03e` hardening + `03d` captcha) under their own extractors.
+
+- **Public data only** at first — discovery/extraction of publicly visible pages. **Logged-in/account-based bypass is explicitly deferred** beyond Phase 8's first cut; it needs sticky/static proxies + credential management (`03b` static-proxy hand-off) and is the higher-risk, later workstream.
+- The **deferred paid-unblocker tier** (`03e §8`) is the fallback for any platform whose anti-bot exceeds the in-house ceiling.
+
 ## Deferred — Frontend & client phases (separate umbrella, planned LATER)
 
 These are recorded for continuity but are NOT planned in this umbrella. They start once the backend phases above are working.
@@ -140,10 +151,14 @@ These are recorded for continuity but are NOT planned in this umbrella. They sta
 - Phase 4 search APIs: all 5 enum values dropped (`SERPER_API`/`TAVILY_API`/`SEARXNG_API`/`LINKUP_API`/`BAIDU_SEARCH_API`) in 04b. Survivors (SearXNG/Linkup/Baidu) become PLATFORM providers keyed by env (Linkup/Baidu keys move from per-connector `config` to env — app-wide, not per-workspace). 04b carries a destructive migration deleting the 5 connector types' rows.
 - Phase 4 structure: split into 04a (taxonomy/gating/MCP-fix, no migration) and 04b (search repurposing + source-discovery endpoint, with migration); intended order 04a -> 04b (both orders safe).
 - Rename transition policy: HARD CUTOVER of the external API (paths + JSON field names) in Phase 2 — no backward-compat aliases. Rationale: the frontend is (re)built against the corrected backend later, so there is no old client to keep alive; backend correctness is verified via the test suite + OpenAPI rather than the existing UI.
-- WebURL Crawler framework: STANDARDIZE on Scrapling; **remove Firecrawl entirely** (no other scraping frameworks now or planned). Scrapling's `StealthyFetcher` handles Cloudflare; captcha-tools (deferred) covers the rest.
+- WebURL Crawler framework: STANDARDIZE on Scrapling; **remove Firecrawl entirely** (no other scraping frameworks now or planned). Scrapling's `StealthyFetcher` (patchright-Chromium as of 0.4.9 — **not** Camoufox) handles Cloudflare; `03e` stealth-hardening minimizes challenges; captcha-tools (`03d`) covers the rest. All fetch tiers sit behind a `FetchStrategy` seam returning `CrawlOutcome` (callers never depend on the tier).
 - Crawl billing: reuse the existing credit wallet (`credit_micros_balance`) with a new `web_crawl` usage_type. Price: **$1 / 1000 successful requests** (1000 micro-USD per success). Connector/pipeline crawls bill the **workspace owner**; chat scrapes fold their crawl cost into the already-billed chat turn. Gated by `WEB_CRAWL_CREDIT_BILLING_ENABLED` (off for self-hosted); no DB migration required.
 - Billable unit: one unit per URL that returns usable extracted content, regardless of how many internal fallback tiers were attempted (not per HTTP fetch, not per URL-processed).
-- Captcha solving (captcha-tools): DEFERRED to the last Phase-3 subplan (`03d`); non-MVP-blocking.
+- Captcha solving (captcha-tools): **ACTIVE** (no longer deferred) — sequenced last in Phase 3 (`03d`), **after** `03e` hardening, **opt-in + off by default**. Cloudflare stays in-framework (`03a`); reCAPTCHA/hCaptcha/image use `captchatools`. **Billing asymmetry RESOLVED → option (a): a separate per-attempt `web_crawl_captcha` unit** (`WEB_CRAWL_CAPTCHA_*` knobs on `WebCrawlCreditService`), since solvers charge per attempt regardless of crawl success. `ErrNoBalance` halts solving (no retry-loop IP bans).
+- Crawler stealth strategy (the moat): **CloakBrowser REJECTED** (source-patched Chromium binary requires an OEM/SaaS license incompatible with our model). **External unblocker APIs DEFERRED** (ZenRows/ScrapFly/Bright Data) — pre-wired as a config-flagged `FetchStrategy` (`03e §8`) but not built. Plan: maintain an **in-house bypass moat for ~4–6 months** (Scrapling stealth + residential proxies + `03e` hardening + `03d` captcha), then move hostile/top-tier-fingerprinted targets (DataDome/Kasada/reCAPTCHA-Enterprise) to a paid tier if demand/maintenance justifies it. Realistic ceiling acknowledged: in-house beats Cloudflare + the moderate long tail, not top-tier behavioral fingerprinting.
+- Authenticated/logged-in scraping: **OUT OF SCOPE this MVP** (public data only). Sticky/static proxies + credential management are deferred and paired with the future platform actors (`03b` static-proxy hand-off + Phase 8).
+- Phase 3 stealth-hardening subplan `03e` ADDED: geoip locale/tz coherence, `hide_canvas`/`block_webrtc`, persistent per-domain profiles, headed+Xvfb, fonts, DIY humanization (`page_action`; Chromium engine has no built-in `humanize`), a block classifier, and per-domain strategy memory (Redis, no migration).
+- Phase 3 test-harness subplan `03f` ADDED: **manual-only** (no CI/automated gating now) undetectability + extraction scorecard, modeled on CloakBrowser's `bin/cloaktest`. Two labeled axes (Suite S stealth + Suite E extraction) so they scale independently. Drives the **real** Scrapling tiers (browser + curl_cffi HTTP/TLS), reuses `03d`'s `page_action`+closure-cell for JS-object verdicts. **TLS JA3/JA4 parity = informational axis, not a hard gate.** Adopt CloakBrowser bars as aspirational; record our actual free-stack numbers as the committed baseline. The scorecard is the documented evidence/trigger for flipping `03e`'s deferred paid-unblocker tier.
 - Roadmap: WebURL Crawler & Crawl Billing inserted as the new Phase 3; connector two-type → Phase 4; pipelines → Phases 5/6/7.
 - Phase 5 pipelines data model: two new tables `pipelines` (mutable) + `pipeline_runs` (append-only), modeled on `automations`/`automation_runs`; ORM lives in `db.py` next to connectors/folders. `connector_id` nullable (NULL = Phase-7 Uploads), eligibility enforced at create via 04a's `is_pipeline_eligible`. Schedule = `schedule_cron` + `schedule_timezone` (default UTC) + `next_scheduled_at` (cron, matching automations). `pipeline_runs` pre-includes `charged_micros`/`crawls_*`/`result_blob_key` so Phase 6 needs no extra migration. Both tables published to Zero **full-row** (like folders/connectors). Routes reuse `CONNECTORS_*` permissions. Phase 5 ships the data model + API surface only; the `/run` endpoint enqueues a Phase-6 task stub.
 - Phase 7 uploads-as-pipeline: a **singleton "Uploads" pipeline** per workspace (`connector_id NULL`, `save_to_kb=true`), lazily get-or-created (race-safe via a partial unique index `ON pipelines(workspace_id) WHERE connector_id IS NULL`). Each `fileupload`/`folder-upload` request writes a **terminal audit `PipelineRun(trigger=upload, status=succeeded, documents_indexed=<accepted count>)`** — uploads are **route-recorded, not engine-executed** (Phase 6 fails NULL-connector runs by design; existing upload code stays the executor). Best-effort via an **inner** try/except (never 5xx the upload — the route's outer handler would otherwise 500 an already-committed upload). No crawl billing (uploads aren't crawls; `charged_micros` NULL). Per-file ETL truth stays on `Document.status`; accurate roll-up needs the deferred `documents.pipeline_run_id` provenance. Connector `save_to_kb` default stays `False` (opt-in for connectors, mandatory for uploads). Phase 7 also **guards Phase-5's generic CRUD** against the system Uploads pipeline: `POST /pipelines` rejects `connector_id=None` (supersedes Phase 5's permissive create), `/run` and schedule-`PUT` reject NULL-connector pipelines, and Phase 6's scheduler `_claim_due` filters `connector_id IS NOT NULL` as a backstop (so the Uploads pipeline can never be manually-run or scheduled into perpetually-failing runs). See `07-upload-pipeline-kb.md`.
@@ -158,7 +173,9 @@ These are recorded for continuity but are NOT planned in this umbrella. They sta
 | 3 | `03a-crawler-core.md` | drafted |
 | 3 | `03b-proxy-expansion.md` | drafted |
 | 3 | `03c-crawl-billing.md` | drafted |
-| 3 | `03d-captcha-solving.md` | drafted (deferred — last) |
+| 3 | `03e-stealth-hardening.md` | drafted |
+| 3 | `03d-captcha-solving.md` | drafted (active — sequenced after 03e) |
+| 3 | `03f-undetectability-testing.md` | drafted (manual scorecard — sequenced last) |
 | 4 | `04a-connector-category.md` | drafted |
 | 4 | `04b-source-discovery.md` | drafted |
 | 5 | `05-pipelines-model.md` | drafted |

plans/backend/03a-crawler-core.md

@@ -1,7 +1,7 @@
 # Phase 3a — WebURL Crawler core (Scrapling-only) + success semantics
 
 > Part of **Phase 3 — WebURL Crawler & Crawl Billing**. See `00-umbrella-plan.md`.
-> Sibling subplans: `03b-proxy-expansion.md`, `03c-crawl-billing.md`, `03d-captcha-solving.md` (deferred).
+> Sibling subplans: `03b-proxy-expansion.md`, `03c-crawl-billing.md`, `03e-stealth-hardening.md`, `03d-captcha-solving.md`, `03f-undetectability-testing.md` (manual scorecard).
 
 > **Implementation note (applies to all Phase-3 plans).** Phase 3 lands **after** Phases 1–2, which rename `SearchSpace`→`Workspace` and `search_space_id`→`workspace_id` everywhere. Citations below use **today's** names so they stay greppable against current code; when implementing, the live code will already say `workspace_*` — map accordingly. Apply every edit by **symbol/grep** (e.g. `firecrawl_api_key`, `crawl_url`, `FIRECRAWL_API_KEY`), **not** by the absolute line numbers cited here — Phase 2's rename (and `03a`'s own Firecrawl removal) shift line numbers.
 
@@ -11,7 +11,7 @@ Make the Universal WebURL Crawler a **single-framework (Scrapling) component** w
 
 Two hard requirements from the decisions log:
 
-1. **Remove Firecrawl entirely.** No other scraping framework now or planned. Scrapling's `StealthyFetcher` can bypass Cloudflare Turnstile when invoked with `solve_cloudflare=True` (see tier design below); captcha-tools (`03d`, deferred) covers the rest.
+1. **Remove Firecrawl entirely.** No other scraping framework now or planned. Scrapling's `StealthyFetcher` can bypass Cloudflare Turnstile when invoked with `solve_cloudflare=True` (see tier design below); captcha-tools (`03d`) covers reCAPTCHA/hCaptcha, and `03e` adds the broader stealth-hardening that makes the crawler "undetectable" as far as free tooling reaches.
 2. **One billable unit = one URL that yields usable extracted content**, regardless of how many internal fallback tiers ran. `03a` must expose that signal; `03c` meters it.
 
 This subplan does NOT touch proxy rotation (→ `03b`), credit metering (→ `03c`), or captcha (→ `03d`).
@@ -25,7 +25,11 @@ This subplan does NOT touch proxy rotation (→ `03b`), credit metering (→ `03
 1. **Firecrawl** (premium, if `firecrawl_api_key` set) — `_crawl_with_firecrawl()` (lines 89–100, 223–272), imports `from firecrawl import AsyncFirecrawlApp` (line 22).
 2. Scrapling `AsyncFetcher` (static HTTP, curl_cffi) — `_crawl_with_async_fetcher()` (lines 274–310).
 3. Scrapling `DynamicFetcher` (browser, run in a thread) — `_crawl_with_dynamic()` (lines 312–339).
-4. Scrapling `StealthyFetcher` (Camoufox anti-bot, run in a thread) — `_crawl_with_stealthy()` (lines 341–369).
+4. Scrapling `StealthyFetcher` (patchright-Chromium anti-bot, run in a thread) — `_crawl_with_stealthy()` (lines 341–369).
+
+> **Engine note (do not say "Camoufox").** As of the pinned `scrapling[fetchers]>=0.4.9` (`pyproject.toml:91`), `StealthyFetcher` is **"completely stealthy built on top of Chromium"** (`references/Scrapling/scrapling/fetchers/stealth_chrome.py:8`) driven by **patchright** (`references/Scrapling/scrapling/engines/_browsers/_stealth.py:8–9`), **not** Camoufox — Scrapling removed Camoufox (zero matches in the 0.4.9 tree; `uv.lock` carries `patchright`, no `camoufox`). The default Playwright `channel` is `"chromium"` (patchright), or `"chrome"` when `real_chrome=True` (`_browsers/_base.py:469`). Stealth = the compiled-in flag set `DEFAULT_ARGS + STEALTH_ARGS` (`engines/constants.py:24–99`, incl. `--disable-blink-features=AutomationControlled` `:94`) + a persistent context by default (`_stealth.py:90–93`) — this is **runtime/config-level** stealth, which sets the realistic ceiling (see `03e`).
+>
+> **Scrub the in-code "Camoufox" mentions too.** The connector still carries stale Camoufox wording in its own docstrings — the module header tier list (`webcrawler_connector.py:10–12`) and `_crawl_with_stealthy`'s docstring (`:343` "StealthyFetcher (Camoufox)"). Fix these in the 03a refactor so the code matches reality (patchright-Chromium).
 
 Extraction is Trafilatura HTML→markdown in `_build_result()` (lines 371–469). Every Scrapling tier passes `proxy=get_proxy_url()` (lines 287, 329, 359). `crawl_url()` returns a tuple `(result_dict | None, error | None)`; `result_dict` has `content` / `metadata` / `crawler_type`.
 
@@ -57,7 +61,7 @@ Firecrawl's API key is plumbed end-to-end and must be removed everywhere:
 
 ### Runtime/deps already in place
 
-- `Dockerfile:112–115` runs `RUN scrapling install` to fetch patchright Chromium + Camoufox; the `scrapling[fetchers]` extra pulls playwright/patchright. **No new install step needed** once Firecrawl is gone.
+- `Dockerfile:112–115` runs `RUN scrapling install`; the `scrapling[fetchers]` extra pulls playwright/patchright. **No new install step needed** once Firecrawl is gone. *(Accuracy fix: the Dockerfile comment says "patchright Chromium + Camoufox", but `scrapling install` in 0.4.9 only fetches Chromium — `references/Scrapling/scrapling/cli.py:122,131` runs `playwright install chromium` + `install-deps chromium`, no Camoufox. Drop the stale "+ Camoufox" wording from the comment when touching this file. `03e` may add `install-deps` extras for fonts/Xvfb.)*
 - Proxy is read via `app/utils/proxy/get_proxy_url()` (`__init__.py:13`), backed by the `PROXY_PROVIDER` registry (`config/__init__.py:983`). `03a` leaves this single-URL model untouched (rotation is `03b`).
 
 ## Target design
@@ -66,9 +70,9 @@ Firecrawl's API key is plumbed end-to-end and must be removed everywhere:
 
 `crawl_url()` becomes a 3-tier ladder, preserving the existing thread-offload + `NotImplementedError` handling for the browser tiers (Windows `SelectorEventLoop` cannot spawn subprocesses — lines 134–141, 161–168):
 
-1. `AsyncFetcher.get(...)` — fast static HTTP.
+1. `AsyncFetcher.get(...)` — fast static HTTP. **TLS gap to close:** the current call passes `stealthy_headers=True` but **not** `impersonate` (`webcrawler_connector.py:284–289`), so its TLS ClientHello is curl_cffi's *default* JA3 — trivially bot-flagged and incoherent with the browser tiers' UA. Add an `impersonate="chrome"` profile here (Scrapling's static engine accepts it — `references/Scrapling/scrapling/engines/static.py:36–47`). Tracked as a lever in `03e §2b` and validated by `03f §S3`.
 2. `DynamicFetcher.fetch(...)` — full browser (via `asyncio.to_thread`).
-3. `StealthyFetcher.fetch(...)` — Camoufox anti-bot, last resort. Enable Cloudflare solving here by passing **`solve_cloudflare=True`** — a documented `StealthyFetcher.fetch` kwarg ("Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response", `references/Scrapling/scrapling/fetchers/stealth_chrome.py:38`; it's a `StealthSession` TypedDict key passed via `**kwargs`). The current stealthy call (connector lines 354–360) passes `headless`/`network_idle`/`block_ads`/`proxy` but **not** `solve_cloudflare`, so this is a real behavior add. (Note: `solve_cloudflare` runs the full browser challenge loop, so it's correctly scoped to the last-resort tier only.)
+3. `StealthyFetcher.fetch(...)` — patchright-Chromium anti-bot, last resort. Enable Cloudflare solving here by passing **`solve_cloudflare=True`** — a documented `StealthyFetcher.fetch` kwarg ("Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response", `references/Scrapling/scrapling/fetchers/stealth_chrome.py:38`; it's a `StealthSession` TypedDict key passed via `**kwargs`). The current stealthy call (connector lines 354–360) passes `headless`/`network_idle`/`block_ads`/`proxy` but **not** `solve_cloudflare`, so this is a real behavior add. (Note: `solve_cloudflare` runs the full browser challenge loop, so it's correctly scoped to the last-resort tier only.)
 
 Trafilatura extraction (`_build_result`) and `format_to_structured_document()` are unchanged.
 
@@ -83,7 +87,7 @@ class CrawlOutcomeStatus(str, Enum):
     FAILED  = "failed"    # invalid URL or every tier errored
 ```
 
-`crawl_url()` returns a small dataclass `CrawlOutcome(status, result, error, tier)` — **commit to the dataclass** (not a tuple): `03c` keys billing off `status == SUCCESS`, and Phase 6's fetch-only path (`06-pipelines-exec.md`) consumes `outcome.status` / `outcome.result` / `outcome.error` as attributes, so a tuple form would break that consumer. The **billable success predicate is single-sourced**: `status == CrawlOutcomeStatus.SUCCESS`.
+`crawl_url()` returns a small dataclass `CrawlOutcome(status, result, error, tier)` — **commit to the dataclass** (not a tuple): `03c` keys billing off `status == SUCCESS`, and Phase 6's fetch-only path (`06-pipelines-exec.md`) consumes `outcome.status` / `outcome.result` / `outcome.error` as attributes, so a tuple form would break that consumer. The **billable success predicate is single-sourced**: `status == CrawlOutcomeStatus.SUCCESS`. Picking a dataclass (over a tuple) also leaves room for later subplans to **append fields without breaking callers** — `03d` adds `captcha_attempts` / `captcha_solved` for per-attempt billing, and `03e`'s block classifier can attach a `block_type`. (This is distinct from the indexer's positional return, which must stay 2-tuple — see the wrapper note below.)
 
 | Outcome | When | Billable (`03c`)? | Document status (indexer) |
 |---------|------|-------------------|---------------------------|
@@ -93,6 +97,17 @@ class CrawlOutcomeStatus(str, Enum):
 
 **Billing-policy note for `03c`:** success is the *crawl* succeeding (we fetched + extracted), independent of downstream KB dedupe. The indexer currently marks unchanged content as `skipped` (`webcrawler_indexer.py:341–347`) and cross-connector duplicates as `failed` (`:350–369`) — those still represent a **successful crawl** and should bill. `03c` must count `CrawlOutcomeStatus.SUCCESS`, not `documents_indexed`. Flagging here; final call lives in `03c`.
 
+### Extensibility seam (the tier ladder is a strategy chain)
+
+The 3-tier ladder is the first instance of a deliberate **`FetchStrategy` seam**: an *ordered list of strategies*, each `(url, ctx) -> CrawlOutcome`, tried in order until one returns `SUCCESS`. `crawl_url()` owns the chain; **every caller depends only on `CrawlOutcome`, never on which strategy produced it** (the indexer, the chat tool, and `03c` metering already do — keep that invariant sacred). This is what lets the moat grow without rework:
+
+- `03d` (captcha) attaches by escalating to the StealthyFetcher strategy with a `page_action` token-injector — a *parameterization* of the last tier, not a new caller contract.
+- `03e` (stealth-hardening) tunes/adds strategies (humanize, headed/Xvfb, persistent profiles, fingerprint flags) **behind the same return type**.
+- A future **paid-unblocker tier** (deferred — see `03e`) is just one more strategy appended last, flippable by config; no caller changes.
+- Future **platform actors** (Phase 8) reuse the same fetch strategies under their own structured extractors.
+
+MVP scope here is only the 3 Scrapling tiers + the `CrawlOutcome` contract; the seam is a design constraint (keep tiers pluggable + callers outcome-only), **not** a call for a heavyweight Strategy framework now.
+
 ### Success counter for the indexer
 
 Add an explicit `crawls_succeeded` counter in `index_crawled_urls()` incremented whenever `crawl_url` returns `SUCCESS` (right after line 297's call, before the dedupe/unchanged branches), and surface it in the task-success metadata (lines 455–466). `03c` meters against this **in-function** counter (it charges inside the indexer — the count does not need to escape via the return).
@@ -113,7 +128,7 @@ Drop the Firecrawl-only `formats=["markdown"]` arg (markdown is already the Traf
 
 ## Risks / trade-offs
 
-- **Loss of a managed fallback.** Firecrawl was a hosted last resort for hostile anti-bot sites. Mitigation: `StealthyFetcher` + Cloudflare solving now, `03b` proxy rotation, `03d` captcha solving. Acceptable per the decisions log (single-framework intent).
+- **Loss of a managed fallback.** Firecrawl was a hosted last resort for hostile anti-bot sites. Mitigation: the in-house stack — `StealthyFetcher` + Cloudflare solving (this plan), `03b` proxy rotation, `03e` stealth-hardening (humanize/headed/profiles/fonts + block-classifier), `03d` captcha solving — plus a **deferred paid-unblocker tier** behind the seam for the hostile residual. Realistic ceiling: this defeats Cloudflare + the long tail of moderate anti-bot, but runtime-level (patchright) stealth does **not** reliably beat top-tier fingerprinting (DataDome/Kasada/reCAPTCHA-Enterprise) — that's the deferred paid tier's job (`03e`). Acceptable per the decisions log (single-framework intent + in-house moat).
 - **Browser tiers on dev/Windows.** `DynamicFetcher`/`StealthyFetcher` need subprocess support; the existing `to_thread` + `NotImplementedError` guards (lines 134–141, 161–168) are preserved so static-only crawling still works in `uvicorn --reload`.
 - **`uv.lock` churn.** Removing `firecrawl-py` requires a lockfile regen + image rebuild; no new runtime deps are added.
 
@@ -121,5 +136,6 @@ Drop the Firecrawl-only `formats=["markdown"]` arg (markdown is already the Traf
 
 - Proxy provider expansion + rotation → `03b`.
 - Crawl credit metering on `CrawlOutcomeStatus.SUCCESS` → `03c`.
-- reCAPTCHA/hCaptcha solving via captcha-tools → `03d` (deferred). Cloudflare Turnstile stays in-framework (Scrapling).
+- reCAPTCHA/hCaptcha solving via captcha-tools → `03d` (**now active**, sequenced after `03e`). Cloudflare Turnstile stays in-framework (Scrapling).
+- Stealth-hardening (humanize, headed/Xvfb, persistent profiles, fonts, geoip locale/tz, block-classifier + per-domain strategy memory) and the deferred paid-unblocker tier → `03e`.
 - Whether ad-hoc **chat** scrapes are billed (vs only pipeline crawls) → decided in `03c`.

plans/backend/03b-proxy-expansion.md

@@ -1,7 +1,7 @@
 # Phase 3b — Proxy provider expansion + rotation
 
 > Part of **Phase 3 — WebURL Crawler & Crawl Billing**. See `00-umbrella-plan.md`.
-> Depends on `03a-crawler-core.md` (Scrapling-only crawler). Siblings: `03c-crawl-billing.md`, `03d-captcha-solving.md` (deferred).
+> Depends on `03a-crawler-core.md` (Scrapling-only crawler). Siblings: `03c-crawl-billing.md`, `03e-stealth-hardening.md`, `03d-captcha-solving.md`.
 
 > **Implementation note.** Same convention as `03a`: citations use **today's** names (`search_space_id`/`SearchSpace`) — map to `workspace_id`/`Workspace` post Phases 1–2. Crucially, the `webcrawler_connector.py` line numbers below (e.g. the three tiers at 287/329/359) and the `scrape_webpage.py` lines **predate `03a`'s Firecrawl removal + `crawl_url` refactor**, so they will have moved by the time `03b` is implemented. Locate code by **symbol/grep**, not absolute lines.
 
@@ -123,4 +123,6 @@ Because there is one global provider and rotation lives **inside** it, the crawl
 - Branded-vendor provider subclasses → not planned (use `CustomProxyProvider`).
 - **Static / sticky-session proxies (future).** A later capability will add **static proxy** support — sticky IPs held for the duration of a session — most likely paired with **authenticated/account-based scraping** to bypass logged-in platforms (the deferred platform connectors: LinkedIn, Instagram, etc.). This is a *different axis* from the rotating pool here: rotation maximizes IP diversity, whereas account bypass needs IP **stability** so a session/cookie stays bound to one IP. It is additive to this design — a new `ProxyProvider` (or a "sticky" mode/flag on `CustomProxyProvider`) registered under a new `PROXY_PROVIDER` key, with no change to the zero-arg getter contract — and stays consistent with the single-provider model (the active provider would be the static one when that workflow is selected). Build it alongside the platform connectors, not in Phase 3.
 - Crawl credit metering (proxy cost is absorbed into the flat `$1 / 1000 successful` price, **not** metered separately) → `03c`.
-- Captcha solving → `03d` (deferred).
+- Captcha solving → `03d`.
+- **Geoip fingerprint coherence** (matching browser `locale`/`timezone_id` to the proxy's exit geo) → `03e`. 03b only owns *selecting* the proxy; `03e` consumes the chosen endpoint's geo. The provider's `RESIDENTIAL_PROXY_LOCATION` (`config/__init__.py`) is one input; resolving the actual exit IP's geo is `03e`'s job.
+- **Surfacing the crawl's chosen endpoint into the strategy.** Both `03d` (IP-bound captcha solves) and `03e` (geoip + sticky reuse) need the **exact** endpoint a crawl used, *not* a fresh `get_proxy_url()` call (which rotates on a pool-backed `CustomProxyProvider`). The capture-once seam lives in `03a`'s `FetchStrategy` context; 03b only guarantees the rotating getter and notes the consumers.

plans/backend/03c-crawl-billing.md

@@ -1,7 +1,7 @@
 # Phase 3c — Crawl credit billing ($1 / 1000 successful requests)
 
 > Part of **Phase 3 — WebURL Crawler & Crawl Billing**. See `00-umbrella-plan.md`.
-> Depends on `03a-crawler-core.md` (the `CrawlOutcomeStatus.SUCCESS` signal + `crawls_succeeded` counter). Sibling: `03b-proxy-expansion.md`. `03d` is deferred.
+> Depends on `03a-crawler-core.md` (the `CrawlOutcomeStatus.SUCCESS` signal + `crawls_succeeded` counter). Siblings: `03b-proxy-expansion.md`, `03e-stealth-hardening.md`, `03d-captcha-solving.md` (now active). `03d` extends this service with a **separate per-attempt captcha unit** (see §"Captcha billing seam" + `03d §4`).
 
 ## Objective
 
@@ -49,6 +49,8 @@ New `app/services/web_crawl_credit_service.py` — a dedicated service (independ
 
 A separate `CreditMeterService` generalization is deliberately deferred until a third per-unit biller appears — exposing the price/flag as statics already lets the chat surface share the math without coupling the two flows.
 
+**Captcha billing seam (for `03d`).** `03d` (now active) adds a **separate per-attempt** captcha unit to this same service — `captcha_solves_to_micros(n) = n * config.WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE`, gated by `WEB_CRAWL_CAPTCHA_BILLING_ENABLED` (both new knobs, default FALSE), with `usage_type="web_crawl_captcha"`. It is **independent** of crawl SUCCESS because solvers charge per *attempt* regardless of crawl outcome (the asymmetry in the §"Risks" proxy/captcha note). Build the `WebCrawlCreditService` statics so `03d` can add one more `*_to_micros` static + one charge path without restructuring — no need to implement the captcha methods in `03c`, just leave the shape open.
+
 ### 2. Wiring in `webcrawler_indexer.py`
 
 1. **Resolve the owner.** Bill the **workspace owner** (the product concept), not the triggering `user_id`: `owner_user_id = (select SearchSpace.user_id where id == search_space_id)`. (Mirrors how `billable_calls._resolve_agent_billing_for_search_space` (func `:441`) reads `search_space.user_id` at `billable_calls.py:499` — but we only need the owner id, so a direct `select` is enough; no need to call that LLM-model-resolving helper.)
@@ -117,7 +119,8 @@ One unit per `CrawlOutcomeStatus.SUCCESS` — a URL that yielded usable extracte
 - **Session coupling.** `charge_credits` commits the indexer session; placing it after the final doc commit avoids entangling a debit with mid-run document state.
 - **Gating asymmetry (intended).** The **indexer** path bills whenever `WEB_CRAWL_CREDIT_BILLING_ENABLED` is on and the owner has credit — no "premium tier" requirement (matches ETL, which gates only on its own flag). The **chat** path additionally requires the turn to be premium, because it piggybacks the turn's reserve/finalize machinery. Same price, two different gate sets — call this out in the credit-status UI copy (frontend phase).
 - **No cross-surface double charge.** A chat scrape calls `crawl_url` directly and never enters `webcrawler_indexer`, so a given crawl is billed by exactly one surface.
-- **Proxy/captcha cost.** Absorbed into the flat $1/1000 (margin), **not** metered separately (see `03b`, and `03d` for captcha which has its own per-solve upstream cost — revisit there).
+- **Proxy cost.** Absorbed into the flat $1/1000 (margin), **not** metered separately (see `03b`).
+- **Captcha cost.** **Metered separately** as a per-attempt unit (`03d §4`, decided: option (a)) because the solver charges per attempt regardless of crawl success — it cannot ride the per-success flat rate. The `captcha_solves_to_micros` seam above is where it attaches.
 
 ## Resolved decisions (this pass)
 
@@ -128,5 +131,5 @@ One unit per `CrawlOutcomeStatus.SUCCESS` — a URL that yielded usable extracte
 ## Out of scope (hand-offs)
 
 - Per-**pipeline** run accounting / `charged_micros` persistence for idempotency → Phases 5–7. Specifically, **Phase 6 moves pipeline-run billing out of the indexer to the run level**: it adds a `bill: bool = True` param to this indexer wiring and calls `index_crawled_urls(bill=False)` from the pipeline engine, which then does its own `check_credits` + `charge_credits(crawls_succeeded)` and stamps `PipelineRun.charged_micros` (idempotency). So this phase's in-indexer billing keeps serving the **connector `/index` + periodic** paths; the "pipeline crawls" half of the objective is refined by `06`. Structure the §2 wiring so it's easy to gate behind that flag.
-- Captcha-solver upstream cost pass-through → `03d` (deferred).
+- Captcha-solver upstream cost pass-through → `03d` (now active; separate per-attempt `web_crawl_captcha` unit via the seam above).
 - Surfacing crawl spend in the credit-status UI → frontend umbrella.

plans/backend/03d-captcha-solving.md

@@ -1,11 +1,16 @@
-# Phase 3d — Captcha solving (DEFERRED — sequenced last, non-MVP-blocking)
+# Phase 3d — Captcha solving (reCAPTCHA / hCaptcha / image)
 
 > Part of **Phase 3 — WebURL Crawler & Crawl Billing**. See `00-umbrella-plan.md`.
-> **Status: deferred.** Build only after `03a`/`03b`/`03c` ship and there's a real need. Depends on `03a` (the StealthyFetcher tier) and the app-wide proxy provider from `03b` (env-selected `get_active_provider()`, accessed via the zero-arg `get_proxy_url()`). Touches the crawl billing model from `03c`.
+> **Status: ACTIVE (sequenced last in Phase 3).** No longer deferred — captcha bypass is now an explicit goal of the "undetectable, captcha-bypassing universal WebURL crawler." Build **after** `03a` (StealthyFetcher tier + `FetchStrategy` seam + `CrawlOutcome`), `03b` (app-wide proxy provider, accessed via the zero-arg `get_proxy_url()`), `03c` (crawl wallet), and `03e` (stealth-hardening — solving is the *last* resort once humanization + fingerprint coherence have minimized challenges).
+> **Decision recap (this pass):** Cloudflare stays in-framework (free, `03a`); reCAPTCHA/hCaptcha/image use a **paid** solver via `captchatools`, **opt-in + off by default**, and are billed as a **separate per-attempt unit** (option (a) below — now decided, not deferred). Logged-in/account-gated captcha flows are **out of scope** (no authenticated scraping in this MVP — see umbrella Phase 8 / `03b` static-proxy hand-off).
 
-## Why deferred
+## Why this is the last tier, not the first
 
-Scrapling already solves **Cloudflare Turnstile/Interstitial** via `solve_cloudflare=True` on the StealthyFetcher tier (`03a`; `references/Scrapling/scrapling/fetchers/stealth_chrome.py:38,90`). That covers the most common interstitial. The remaining captcha types (reCAPTCHA v2/v3, hCaptcha, image) need a **paid third-party solver**, add latency (10–60s/solve), have <100% success, and carry target-ToS/legal considerations. None of that is MVP-blocking, so it's sequenced last.
+A solve is the **most expensive and least reliable** move available (paid per attempt, 10–60 s latency, <100% success, ToS-sensitive). So the crawler must exhaust the cheaper, in-house levers first; captcha solving only fires when everything else has already failed:
+
+1. `03e` humanization + fingerprint coherence (geoip locale/tz, persistent profile, headed/Xvfb) → **avoid** triggering a challenge at all.
+2. `03a` `solve_cloudflare=True` on StealthyFetcher → free Cloudflare Turnstile/Interstitial.
+3. **This subplan** → only the residual reCAPTCHA v2/v3 + hCaptcha + image challenges, and only when `CAPTCHA_SOLVING_ENABLED`.
 
 ## Scope boundary
 
@@ -13,46 +18,89 @@ Scrapling already solves **Cloudflare Turnstile/Interstitial** via `solve_cloudf
 | --- | --- | --- |
 | Cloudflare Turnstile / Interstitial | Scrapling `solve_cloudflare=True` | `03a`, StealthyFetcher tier |
 | reCAPTCHA v2/v3, hCaptcha, image | **`captchatools`** (this subplan) | StealthyFetcher `page_action` |
+| DataDome / Kasada / reCAPTCHA-**Enterprise** | **none in the free stack** | deferred paid-unblocker tier (`03e`) |
 
 ## Grounding (libraries verified)
 
-- **`captchatools` is itself the provider registry.** `new_harvester(api_key, solving_site, sitekey, captcha_url, captcha_type="v2"|"v3"|"hcaptcha"|"image", ...)` → `.get_token(proxy=…, proxy_type=…, user_agent=…, b64_img=…)` returns a **token string** (`references/Captcha-Tools/README.md:28–47`). `solving_site` ∈ {`capmonster`,`2captcha`,`anticaptcha`,`capsolver`,`captchaai`} (`:113–127`). Errors: `ErrNoBalance`, `ErrWrongAPIKey`, `ErrWrongSitekey`, … via `captchatools.exceptions` (`:136–155`). **Implication:** we do **not** rebuild a multi-provider class hierarchy (unlike `03b`'s proxy registry) — `captchatools` already dispatches across the 5 services. Our layer is thin: config resolution + page detection/injection glue.
+- **`captchatools` is itself the provider registry.** `new_harvester(api_key, solving_site, sitekey, captcha_url, captcha_type="v2"|"v3"|"hcaptcha"|"image", invisible_captcha=…, min_score=…, action=…)` → `.get_token(proxy=…, proxy_type=…, user_agent=…, b64_img=…)` returns a **token string** (`references/Captcha-Tools/README.md:28–47`). `solving_site` ∈ {`capmonster`,`2captcha`,`anticaptcha`,`capsolver`,`captchaai`} (`:113–127`). Errors: `ErrNoBalance`, `ErrWrongAPIKey`, `ErrWrongSitekey`, `ErrIncorrectCapType`, `ErrNoHarvester` via `captchatools.exceptions` (`:136–155`). **Implication:** we do **not** rebuild a multi-provider class hierarchy (unlike `03b`'s proxy registry) — `captchatools` already dispatches across the 5 services. Our layer is thin: config resolution + page detection/injection glue + billing.
 - **`captchatools` only harvests a token; it does not inject it.** The caller must drop the token into the page (`g-recaptcha-response` / `h-captcha-response` textarea, or invoke the JS callback) and submit.
-- **Injection requires a browser page**, so this only works on the **StealthyFetcher** tier. Scrapling exposes `page_action`: "a function that takes the `page` object, runs after navigation, and does the automation you need" (`stealth_chrome.py:30,82`). AsyncFetcher (HTTP) and DynamicFetcher cannot solve interactive captchas — a captcha hit there must escalate to StealthyFetcher.
+- **Injection requires a live browser page**, so this only works on the **StealthyFetcher** tier. Scrapling exposes `page_action`: "a function that takes the `page` object, runs after navigation, and does the automation you need" (`references/Scrapling/scrapling/fetchers/stealth_chrome.py:30,82`; engine docstrings `_browsers/_stealth.py:50,191,326,466`). The HTTP (`AsyncFetcher`) and `DynamicFetcher` tiers cannot solve interactive captchas — a captcha hit there must **escalate to StealthyFetcher** (`FetchStrategy` seam from `03a`; the ladder already ends there).
+- **Ordering is already correct in Scrapling.** `solve_cloudflare` runs **before** `page_action` (sync `_stealth.py:253–254` then `:258–260`; async `:529–530` then `:534–536`). So Cloudflare is cleared first, *then* our injector runs — exactly the tier order we want. `page_action` exceptions are caught + logged by Scrapling (`:262` / `:538`), so our factory must also return a clear signal (it can't rely on raising to abort the fetch).
 
-## Sketch (when built)
+## Target design
 
-1. **Config layer** (`app/utils/captcha/`, mirroring `app/utils/proxy/`'s config resolution from `03b`):
+### 1. Config layer (`app/utils/captcha/`, mirroring `app/utils/proxy/` from `03b`)
-   - `CaptchaConfig` = `(enabled, solving_site, api_key)`, resolved from **env only** — one app-wide config, mirroring `03b`'s env-only single-provider model (`get_active_provider()`); **no per-connector config** (that path was dropped in `03b`).
-   - Env: `CAPTCHA_SOLVING_ENABLED` (default FALSE), `CAPTCHA_SOLVER_PROVIDER`, `CAPTCHA_SOLVER_API_KEY`. Off by default → zero captcha attempts (and zero solver cost).
-2. **Detection + injection `page_action` factory** — builds a callback passed to `StealthyFetcher.async_fetch(..., page_action=…)`:
-   - Detect sitekey in DOM (`.g-recaptcha[data-sitekey]`, `.h-captcha[data-sitekey]`, reCAPTCHA-v3 via `grecaptcha.execute`).
-   - Harvest: `new_harvester(solving_site, api_key, sitekey, captcha_url=page.url, captcha_type=…).get_token(proxy=<the exact proxy endpoint used for THIS crawl>, user_agent=<page UA>)` — see the IP-binding caveat below.
-   - Inject token + dispatch events / invoke callback; submit; wait for navigation.
-3. **Crawler escalation**: only the StealthyFetcher tier attempts solving; a captcha detected on a lower tier escalates to StealthyFetcher (the ladder already ends there per `03a`).
-4. **Dependency**: add `captchatools` to `pyproject.toml` (build-time only).
 
-## The billing asymmetry (the hard part — decide at build time)
+- `CaptchaConfig = (enabled, solving_site, api_key, captcha_type_default, min_score, max_attempts, timeout_s)`, resolved from **env only** — one app-wide config, mirroring `03b`'s env-only single-provider model (`get_active_provider()`); **no per-connector config** (that path was dropped in `03b`).
+- Env knobs (next to the proxy + crawl-billing knobs in `config/__init__.py`):
+  - `CAPTCHA_SOLVING_ENABLED` (default `FALSE`) — off ⇒ zero solve attempts and zero solver cost.
+  - `CAPTCHA_SOLVER_PROVIDER` (e.g. `capsolver`), `CAPTCHA_SOLVER_API_KEY`.
+  - `CAPTCHA_MAX_ATTEMPTS_PER_URL` (default `1`), `CAPTCHA_SOLVE_TIMEOUT_S` (default `120`).
+- A `captcha_enabled()` static (mirrors `WebCrawlCreditService.billing_enabled()`) so callers gate cheaply before constructing anything.
 
-`03c` bills the workspace owner **per successful crawl** (`CrawlOutcomeStatus.SUCCESS`), and absorbs proxy cost into the flat $1/1000. **Captcha is different**: the solver charges **per attempt** (~$1–3 / 1000, type-dependent) **regardless of whether the crawl ultimately succeeds**. So a failed solve = real upstream cost with **no billable success** — proxy's absorb-it model doesn't transfer cleanly.
+### 2. Detection + injection `page_action` factory
 
-Options (resolve when this is actually built):
+Builds the **sync** callable passed to `StealthyFetcher.fetch(..., page_action=…)` (03a runs StealthyFetcher via `asyncio.to_thread`, so the sync engine path `_stealth.py:258–262` applies — the factory returns a plain `def(page): ...`, not a coroutine):
-- **(a) Separate per-solve charge** — meter each solve attempt as its own unit (e.g. `web_crawl_captcha` usage_type, its own `*_MICROS_PER_*` knob), independent of crawl SUCCESS. Most cost-honest; bills even on failed solves (matching the upstream charge).
-- **(b) Higher crawl price when captcha enabled** — absorb into a fatter flat rate; simplest UX, but cross-subsidizes failed solves and easy-vs-hard pages unevenly.
-- **(c) Cost-plus pass-through** — meter the solver's reported cost × margin.
 
-Recommendation leaning **(a)** (separate per-attempt unit) because the upstream cost is per-attempt and significant, but defer the final call.
+1. **Detect** the challenge + sitekey in the DOM:
+   - reCAPTCHA v2: `.g-recaptcha[data-sitekey]` (or iframe `src*="recaptcha"`).
+   - hCaptcha: `.h-captcha[data-sitekey]` (or iframe `src*="hcaptcha"`).
+   - reCAPTCHA v3: presence of `grecaptcha.execute` + a known `action` (config/site-mapped).
+   - If no sitekey found → **no-op return** (nothing to solve; let `wait_selector`/extraction proceed).
+2. **Bind the proxy/UA** (the IP-coherence caveat below): harvest with the **exact endpoint used for this crawl tier** —
+   `new_harvester(solving_site, api_key, sitekey, captcha_url=page.url, captcha_type=…, min_score=…, action=…).get_token(proxy=<this crawl's endpoint, reformatted ip:port:user:pass>, proxy_type="HTTP", user_agent=<page UA>)` (`README.md:44–46,107–110`).
+3. **Inject + submit**: set the response token into `g-recaptcha-response` / `h-captcha-response` (make textarea visible if needed), dispatch `input`/`change`, invoke the JS callback when present (`grecaptcha.getResponse` flows / `___grecaptcha_cfg` callbacks), then submit the form and `page.wait_for_load_state`. For v3 (no widget) the token is consumed by the site's own `execute()` flow.
+4. **Signal outcome** via a mutable closure cell (since Scrapling swallows `page_action` exceptions at `:262` **and discards its return value** at `:260`/`:536`): record `solved: bool` + `attempts: int` into a captured `dict` the crawler reads after `fetch()` returns, so billing (§4) and the retry cap can act on it. **This `page_action`+closure-cell helper is shared with `03f`'s test harness** (which uses the same mechanism to read JS-object verdicts like CreepJS `window.Fingerprint`) — factor it once.
+
+### 3. Crawler escalation (uses the `03a` `FetchStrategy` seam)
+
+- Only the **StealthyFetcher** strategy attempts solving. A captcha detected on a lower tier (HTTP/DynamicFetcher) returns non-`SUCCESS`, and the ladder already escalates to StealthyFetcher (`03a`), which this time is constructed with the captcha `page_action` when `captcha_enabled()`.
+- **Attempt cap:** at most `CAPTCHA_MAX_ATTEMPTS_PER_URL` solves per URL so one hostile page can't burn unbounded solver credit. The closure cell's `attempts` enforces it.
+- No new caller *contract*: the strategy still returns `CrawlOutcome` (`03a` invariant). Captcha is a *parameterization* of the last tier, not a new return shape.
+- **But billing needs the attempt count to escape the closure cell.** §4 charges per attempt on **both** the indexer and chat paths, so the count can't stay buried inside `page_action`'s closure — the crawler must copy it onto the outcome. Add **`captcha_attempts: int = 0`** (and `captcha_solved: bool = False`) as **dataclass fields on `CrawlOutcome`** (`03a`). This is safe: `03a`'s "don't widen the return" rule is about the *indexer's* positional `(total_processed, error)` tuple unpacked by length, **not** about `crawl_url`'s dataclass — adding fields to a dataclass breaks no consumer. The indexer sums `outcome.captcha_attempts`; the chat tool reads it off the same outcome.
+
+### 4. Billing — separate per-attempt unit (option (a), DECIDED)
+
+`03c` bills per **successful crawl** (`CrawlOutcomeStatus.SUCCESS`) and absorbs proxy cost into the flat $1/1000. **Captcha can't ride that model**: the solver charges **per attempt regardless of whether the crawl ultimately succeeds**, so a failed solve = real upstream cost with no billable crawl success. We therefore meter solves **independently**:
+
+- New static on `WebCrawlCreditService` (defined in `03c`, knobs added here): `captcha_solves_to_micros(n) = n * config.WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE`.
+- Each **attempt** (not each success) is charged when `CAPTCHA_SOLVING_ENABLED` **and** `WEB_CRAWL_CAPTCHA_BILLING_ENABLED`:
+  - **Indexer/pipeline path:** after the crawl loop, debit `captcha_solves_to_micros(total_attempts)` — where `total_attempts = Σ outcome.captcha_attempts` over the batch — from the **workspace owner** (same owner resolution as `03c §2`) and `record_token_usage(usage_type="web_crawl_captcha", …, cost_micros=…, call_details={"attempts": n, "solved": k})`. Added before the charge commit, same one-transaction pattern as `03c`.
+  - **Chat-scrape path:** fold `captcha_solves_to_micros(outcome.captcha_attempts)` into the turn accumulator with `call_kind="web_crawl_captcha"` (same mechanism as `03c §3`), so it settles with the premium turn; non-premium/anonymous turns record-but-don't-debit.
+- **No pre-block on solves** (unlike the crawl batch): attempts are bounded by `CAPTCHA_MAX_ATTEMPTS_PER_URL × len(urls)`, an upper bound the indexer can optionally pre-check against the wallet if we want symmetry with `03c §2` (recommended: pre-check the combined crawl + worst-case captcha estimate so a run can't strand mid-batch on solver insolvency).
+- New env (next to `03c`'s knobs): `WEB_CRAWL_CAPTCHA_BILLING_ENABLED` (default `FALSE`), `WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE` (default e.g. `3000` = $3/1000 attempts; type-dependent, set with margin over the solver's per-attempt price). **No DB migration** — `web_crawl_captcha` is just another free-form `TokenUsage.usage_type` value (`db.py` `usage_type String(50)`).
+
+### 5. Dependency + Docker
+
+- Add `captchatools` to `pyproject.toml` (build-time only). It's a thin HTTP client for the solver services — no browser binaries.
+- No `Dockerfile` change beyond the dependency (browser already installed by `03a`/`03e`).
+
+## Error handling (must not loop or silently burn credit)
+
+- `ErrNoBalance` / `ErrWrongAPIKey` (`README.md:134,136–143`) → **stop solving for the rest of the run** (set a process/run flag), surface a clear "captcha solver out of balance / misconfigured" message via `log_task_failure`, and fall through to a non-`SUCCESS` `CrawlOutcome`. The README explicitly warns balance-exhausted loops can get the **IP temporarily banned** (`:132–134`) — never retry on `ErrNoBalance`.
+- `ErrWrongSitekey` / `ErrIncorrectCapType` → log + skip this URL's solve (likely detection bug, not transient); count the attempt for billing only if the solver actually charged.
+- Timeout (`CAPTCHA_SOLVE_TIMEOUT_S`) → abort the solve, count one attempt, return non-`SUCCESS`.
 
 ## Risks / considerations
 
 - **Solver-tier only.** No captcha solving on the HTTP/DynamicFetcher tiers; must escalate to StealthyFetcher (slower, browser-backed).
-- **Latency & flakiness.** Solves take 10–60s and aren't guaranteed; tune timeouts and a max-attempts cap so a single URL can't burn unbounded solver credit.
+- **Latency & flakiness.** Solves take 10–60 s and aren't guaranteed; the `CAPTCHA_MAX_ATTEMPTS_PER_URL` + `CAPTCHA_SOLVE_TIMEOUT_S` caps keep a single URL from burning unbounded solver credit.
-- **Solver-account balance.** `ErrNoBalance`/`ErrWrongAPIKey` (`README.md:136–155`) must surface clearly and disable solving rather than loop.
+- **Proxy coherence (rotating-pool caveat).** Many captchas IP-bind the token, so the solver must egress from the **same IP** as the crawl. Under `03b`'s single app-wide provider this is automatic for single-endpoint providers (`anonymous_proxies`, single-URL `custom`). But a **pool-backed `CustomProxyProvider`** returns the *next* endpoint on each zero-arg `get_proxy_url()` call — so the `page_action` must **reuse the endpoint actually used for this crawl tier** (the crawler captures it once and passes it into the factory), NOT call `get_proxy_url()` again (which would rotate to a different IP). This is the same "surface the chosen endpoint" seam `03e`/`03d` both rely on — build it once in `03a`'s strategy context.
-- **Proxy coherence (rotating-pool caveat).** Some captchas IP-bind the token, so the solver must egress from the **same IP** as the crawl. Under `03b`'s single app-wide provider this is automatic for single-endpoint providers (`anonymous_proxies`, single-URL `custom`). But a **pool-backed `CustomProxyProvider`** returns the *next* endpoint on each zero-arg `get_proxy_url()` call — so the `page_action` must **reuse the endpoint actually used for this crawl tier** (capture it once, pass it to `get_token(proxy=…)`), NOT call `get_proxy_url()` again (which would rotate to a different IP). This needs a small seam to surface the crawl's chosen endpoint into the `page_action` — note it when `03d` is built on top of `03b`'s rotation.
+- **Enterprise captchas are out of reach.** reCAPTCHA **Enterprise**, DataDome, and Kasada use behavioral + device signals the token-injection model doesn't satisfy reliably; those route to the deferred paid-unblocker tier (`03e`), not here.
-- **Policy / ToS.** Automated captcha solving may violate a target site's terms; gate behind the explicit `CAPTCHA_SOLVING_ENABLED` flag and treat as an opt-in, owner-acknowledged capability.
+- **Policy / ToS.** Automated captcha solving may violate a target site's terms; gate behind the explicit `CAPTCHA_SOLVING_ENABLED` flag and treat as an opt-in, owner-acknowledged capability. Public data only (no logged-in bypass).
+
+## Work items
+
+1. **`app/utils/captcha/`**: `CaptchaConfig` + env resolution + `captcha_enabled()` static.
+2. **`page_action` factory**: detection (v2/v3/hcaptcha/image sitekey), harvest via `captchatools` with bound proxy/UA, inject+submit, outcome closure cell.
+3. **Crawler wiring**: construct StealthyFetcher with the factory when `captcha_enabled()`; enforce per-URL attempt cap; surface the crawl's chosen proxy endpoint into the factory.
+4. **Billing**: `WebCrawlCreditService.captcha_solves_to_micros` + the two knobs; debit per-attempt on the indexer path (owner) and fold into the turn on the chat path; `record_token_usage(usage_type="web_crawl_captcha")`.
+5. **Config + docs**: all env knobs in `Config` + `.env.example` (commented, hosted=TRUE / self-hosted=FALSE), plus the `captchatools` dependency.
+6. **Tests**: solving disabled → zero attempts, zero solver cost (self-hosted); v2 detected → harvest+inject+submit path invoked with the **crawl's** proxy endpoint (not a re-rotated one); `ErrNoBalance` → stops solving + no retry loop; attempt cap honored; billing enabled → per-attempt debit even when the crawl ultimately fails; chat path folds captcha micros into a premium turn and record-only on anonymous; `web_crawl_captcha` `TokenUsage` rows written.
 
 ## Out of scope
 
-- Anything in `03a`/`03b`/`03c`.
+- Anything owned by `03a`/`03b`/`03c`/`03e`.
 - Cloudflare Turnstile (already handled by Scrapling in `03a`).
-- Final captcha billing model — chosen at build time (see options above).
+- Logged-in / account-gated captcha flows (no authenticated scraping this MVP → umbrella Phase 8 + `03b` static-proxy hand-off).
+- Enterprise/behavioral anti-bot (DataDome/Kasada/reCAPTCHA-Enterprise) → deferred paid-unblocker tier (`03e`).

plans/backend/03e-stealth-hardening.md

@@ -0,0 +1,119 @@
+# Phase 3e — Stealth hardening (the in-house "undetectable" moat)
+
+> Part of **Phase 3 — WebURL Crawler & Crawl Billing**. See `00-umbrella-plan.md`.
+> Depends on `03a` (Scrapling StealthyFetcher tier + `FetchStrategy` seam + `CrawlOutcome`) and `03b` (app-wide proxy provider). Precedes `03d` (captcha) in the escalation order — hardening **avoids** challenges; captcha solving is the paid last resort for the ones we can't avoid. Touches `03c` only via the deferred paid-unblocker tier (its own billing, decided if/when built).
+
+> **Implementation note.** Same convention as `03a`–`03d`: citations use **today's** names (`search_space_id`/`SearchSpace`) and predate `03a`'s `crawl_url` refactor; locate code by **symbol/grep**, not absolute lines. Scrapling references point at the on-disk (gitignored) `references/Scrapling/` checkout pinned to `scrapling[fetchers]>=0.4.9` (`pyproject.toml:91`).
+
+## Objective
+
+Push the Universal WebURL Crawler as far toward "undetectable" as the **free / in-house stack** allows, so we hold a scraping moat for the next ~4–6 months **without** a third-party unblocker (ZenRows/ScrapFly/Bright Data) or a source-patched browser (CloakBrowser — rejected on licensing). Everything here is **runtime/config-level** stealth layered on Scrapling's patchright-Chromium (`03a` engine note): geoip fingerprint coherence, persistent profiles, headed execution, real fonts, and behavioral humanization — plus a **block classifier** + **per-domain strategy memory** so the ladder learns and the cheap tiers get skipped once a domain's working strategy is known. The paid-unblocker tier is defined here as a **deferred** `FetchStrategy` (the explicit escape hatch for when in-house maintenance gets too costly), not built now.
+
+## The realistic ceiling (be honest with downstream devs)
+
+Scrapling already ships strong **runtime** stealth by default: `DEFAULT_ARGS + STEALTH_ARGS` (`references/Scrapling/scrapling/engines/constants.py:24–99`, incl. `--disable-blink-features=AutomationControlled` `:94`), a persistent context (`_browsers/_stealth.py:91–93,366–368`), and `navigator.webdriver` masking via patchright. With `03b` residential proxies + this plan's coherence/humanization on top, the crawler reliably handles **Cloudflare** (via `03a` `solve_cloudflare`) and the long tail of **moderate** anti-bot.
+
+What the free stack does **not** reliably beat: **DataDome, Kasada, reCAPTCHA-Enterprise**, and other behavioral/device-fingerprint systems. Those generally require C++ source-level browser patches (CloakBrowser's 58-patch approach — out on licensing) or a commercial unblocker. For **competitive-intelligence targets** these top-tier defenses are the exception, not the rule, so the in-house moat is a sound 4–6 month bet — with the deferred paid tier (§8) as the pre-wired escape hatch. **Do not promise "beats everything."**
+
+## Levers (all behind the `03a` `FetchStrategy` seam — callers stay outcome-only)
+
+### 1. Geoip fingerprint coherence (match the browser to the proxy exit IP)
+
+A residential IP in Berlin behind an `en-US`/`America/New_York` browser is an instant tell. Make the fingerprint cohere with the proxy's exit geo:
+
+- Resolve the proxy exit IP → country/locale/timezone (lightweight geoip; e.g. a bundled MaxmindLite/`geoip2` DB or the proxy provider's location knob `RESIDENTIAL_PROXY_LOCATION` from `03b`).
+- Pass the matched values into StealthyFetcher: `locale=` (drives `navigator.language` + `Accept-Language` — `stealth_chrome.py:34–35`) and `timezone_id=` (`:36`). These flow into the Playwright context (`_browsers/_base.py:440–441`).
+- This needs the crawl's **chosen proxy endpoint** surfaced into the strategy (the same seam `03d` needs for IP-bound solves) — capture it once in `03a`'s strategy context, don't re-call `get_proxy_url()` (which rotates on a pool-backed provider, `03b`).
+
+### 2. Fingerprint flags Scrapling already exposes (turn them on)
+
+- `hide_canvas=True` — random noise on canvas ops to defeat canvas fingerprinting (`stealth_chrome.py:40`).
+- `block_webrtc=True` — forces WebRTC to respect the proxy, preventing the **real local IP leak** that unmasks proxied browsers (`:41`).
+- `google_search=True` (default) — sets a Google referer so the first hit looks like organic arrival (`:45`); override per-need via `extra_headers` (`:46`).
+- `additional_args=` — last-priority Playwright context overrides for anything not surfaced as a first-class param (`:51`).
+
+### 2b. HTTP-tier TLS fingerprint (the AsyncFetcher tier — `impersonate`)
+
+The cheap static tier (`03a` tier 1) is the **first** thing every crawl hits, yet it currently sets `stealthy_headers=True` but **no** `impersonate` (`webcrawler_connector.py:284–289`), so its TLS ClientHello is curl_cffi's default JA3 — a non-browser signature that fingerprinting WAFs flag before the body even loads. Pass an `impersonate="chrome"` profile (Scrapling's static engine selects a matching curl_cffi browser profile — `references/Scrapling/scrapling/engines/static.py:36–47`) so the HTTP tier's **JA3/JA4/HTTP-2** matches a real Chrome and coheres with the browser tiers' UA. Cheap, safe, default-on. `03f §S3` is the test that validates parity against `tls.peet.ws`.
+
+### 3. Persistent per-domain profiles (look like a returning human)
+
+Scrapling defaults to a **temporary** user-data dir (fresh = suspicious). Use `user_data_dir=` (`stealth_chrome.py:48`; `launch_persistent_context` `_stealth.py:93,366–368`) to keep a **persistent profile per domain** (or per domain+proxy-geo), so cookies/localStorage/site-trust carry across crawls and the browser presents as a returning visitor rather than a brand-new incognito session. Store profiles under a configured dir (`shared_tmp`-style volume so API + worker share them).
+
+### 4. Headed execution under Xvfb (defeat headless tells)
+
+`headless` defaults to hidden (`stealth_chrome.py:19/71`). Many WAFs flag headless Chromium. Run **headful** (`headless=False`) inside a virtual framebuffer (**Xvfb**) in the Docker worker so the browser is "visible" to itself but needs no real display. Gate behind a config flag (off by default for self-hosted, on for hosted workers that have Xvfb installed).
+
+### 5. Real fonts (canvas/emoji hash realism)
+
+A minimal container has almost no fonts, making canvas/emoji fingerprint hashes obviously synthetic. Install real font packages (DejaVu, Liberation, Noto incl. CJK + emoji) in the worker image so font-enumeration + canvas hashes resemble a real desktop. `Dockerfile` change only (`apt-get install fonts-*`).
+
+### 6. Behavioral humanization (DIY — Scrapling has no `humanize` for the Chromium engine)
+
+Unlike Camoufox, the patchright-Chromium StealthyFetcher exposes **no built-in `humanize`** (verified: zero matches in the StealthyFetcher param set). So humanization is custom, injected via the existing hooks:
+
+- `page_action=` (runs after navigation, `stealth_chrome.py:30`; engine `_stealth.py:258–262`) — randomized mouse moves/curves, scrolls, hover-before-click, and small think-time delays before extraction. This is the **same hook `03d` uses** for token injection, so the two compose (humanize → optional captcha solve).
+- `init_script=` (JS executed on page creation, `:33`) — early shims for any residual JS tells not covered by patchright.
+- Tunable `wait`/`network_idle` (`:29,27`) so dwell time isn't robotically constant.
+
+### 7. Block classifier + per-domain strategy memory (the "learning ladder")
+
+Two small in-house pieces make the ladder smart instead of brute-force:
+
+- **Block classifier** — inspect each `Response` (status + body/cookie markers) and label the outcome: `OK` / `CLOUDFLARE` / `CAPTCHA_RECAPTCHA` / `CAPTCHA_HCAPTCHA` / `DATADOME` / `KASADA` / `RATE_LIMITED` / `EMPTY`. (Markers: `"Just a moment"` + `cf-mitigated` → Cloudflare; `datadome` cookie/script → DataDome; `g-recaptcha`/`h-captcha` sitekeys → captcha; etc.) The label drives escalation (which tier/lever next), routes captcha types to `03d`, and feeds the memory below. It also makes `CrawlOutcome.status` decisions principled instead of "empty == fail".
+- **Per-domain strategy memory** — cache the **strategy that last succeeded per domain** (e.g. Redis key `crawl:strategy:{domain}` with TTL, reusing the existing Redis the indexer already uses for `indexing_locks`). Next crawl of that domain starts at the known-good tier/lever set, skipping cheaper tiers that always fail there — fewer requests, lower latency, less proxy/captcha spend. No DB migration (Redis, best-effort, self-healing on miss).
+
+## 8. Deferred: paid-unblocker tier (the escape hatch, NOT built now)
+
+The moat strategy is explicit: **maintain in-house bypass for ~4–6 months, then move hostile targets to a paid unblocker if demand/maintenance justifies it.** That switch is **evidence-driven, not a guess** — `03f`'s manual scorecard quantifies the free-stack ceiling over time and is the documented trigger for flipping this tier. Pre-wire the seam so the switch is a config flip, not a refactor:
+
+- Define (but do not implement) a `PaidUnblockerStrategy` that satisfies the `03a` `FetchStrategy` contract `(url, ctx) -> CrawlOutcome`, appended **last** in the ladder and active only when an env flag + API key are set.
+- It would call an external unblocker (ZenRows/ScrapFly/Bright Data Web Unlocker) for the residual `DATADOME`/`KASADA`/`reCAPTCHA-Enterprise` domains the block classifier flags as unreachable in-house.
+- **Its own billing** (cost-plus pass-through, decided at build time) — separate from `03c`'s flat crawl unit and `03d`'s per-solve unit, because unblocker pricing is per-request and provider-specific.
+- Until built, those domains simply return non-`SUCCESS` (free under `03c`). This keeps the umbrella's "WebURL Crawler is the moat" honest while bounding our maintenance risk.
+
+## Config / env changes
+
+Add (all default OFF / conservative; next to the `03b`/`03c` knobs in `config/__init__.py` + `.env.example`):
+
+- `CRAWL_GEOIP_MATCH_ENABLED` (default FALSE) + optional geoip DB path.
+- `CRAWL_HIDE_CANVAS` / `CRAWL_BLOCK_WEBRTC` (default TRUE — cheap, safe).
+- `CRAWL_PERSISTENT_PROFILES_DIR` (unset → Scrapling's temp default; set → per-domain profiles).
+- `CRAWL_HEADED_XVFB_ENABLED` (default FALSE; requires Xvfb in the image).
+- `CRAWL_HUMANIZE_ENABLED` (default TRUE) + dwell/jitter bounds.
+- `CRAWL_STRATEGY_MEMORY_TTL_S` (default e.g. `86400`; 0 → disabled).
+- `CRAWL_PAID_UNBLOCKER_ENABLED` (default FALSE) + provider/key (deferred tier).
+
+## Docker changes (`surfsense_backend/Dockerfile`)
+
+- Install **Xvfb** + a font set (`fonts-dejavu`, `fonts-liberation`, `fonts-noto`, `fonts-noto-cjk`, `fonts-noto-color-emoji`) in the worker image. (Note from `03a`: also drop the stale "+ Camoufox" comment near `:112`; `scrapling install` only fetches Chromium.)
+- Headed runs need the browser launched under `xvfb-run` (or an Xvfb display in the worker entrypoint), gated by `CRAWL_HEADED_XVFB_ENABLED`.
+
+## Work items
+
+1. **Geoip coherence** — resolve proxy exit geo → `locale`/`timezone_id`; thread the crawl's chosen endpoint into the strategy context (shared seam with `03d`).
+2. **Fingerprint flags** — wire `hide_canvas`/`block_webrtc`/`google_search`/`extra_headers`/`additional_args` from config into the StealthyFetcher tier. Also add `impersonate="chrome"` to the AsyncFetcher (HTTP) tier (§2b) so its TLS coheres. **Centralize this into a single per-tier kwargs builder** (one function that returns the StealthyFetcher / AsyncFetcher kwargs from config) — the crawler *and* `03f`'s harness both import it, so the scorecard grades the exact browser we ship (no test-vs-prod drift).
+3. **Persistent profiles** — per-domain `user_data_dir` under `CRAWL_PERSISTENT_PROFILES_DIR` (shared volume).
+4. **Headed + Xvfb** — `headless=False` path gated by flag; Xvfb + fonts in the worker image.
+5. **Humanization** — a `page_action` humanizer (mouse/scroll/dwell) composing with `03d`'s injector; optional `init_script` shims.
+6. **Block classifier** — `classify(response) -> BlockType`, used by the ladder + `CrawlOutcome.status` + `03d` routing.
+7. **Per-domain strategy memory** — Redis read/write around the ladder; start at known-good tier; self-heal on miss.
+8. **Paid-unblocker seam** — define the deferred `FetchStrategy` + config flag only (no provider integration).
+9. **Instrumentation** — log `(domain, block_type, winning_strategy, attempts, latency)` for tuning the ladder.
+10. **Config + Docker + tests**.
+
+## Risks / trade-offs
+
+- **Arms race / maintenance.** Fingerprint bypasses rot as WAFs update; this is exactly the cost the deferred paid tier (§8) hedges. Instrumentation (work item 9) plus **`03f`'s scorecard** are what tell us when in-house upkeep stops being worth it.
+- **Headed/Xvfb cost.** Headful browsers use more CPU/RAM than headless; gate per-flag and only escalate to headed when the block classifier says cheaper tiers fail for a domain (per-domain memory keeps it from being the default).
+- **Profile growth.** Persistent profiles accumulate disk; add a size/TTL cap and periodic prune.
+- **Geoip accuracy.** A coarse country→locale map is fine; over-fitting per-city tz isn't worth it. Wrong-but-coherent beats default-mismatched.
+- **No silver bullet.** Reiterate the ceiling (§"realistic ceiling") in any user-facing copy: the crawler is "best-effort undetectable," not guaranteed.
+
+## Out of scope (hand-offs)
+
+- Cloudflare solving (`03a`), reCAPTCHA/hCaptcha solving + its per-solve billing (`03d`), proxy rotation (`03b`), flat crawl billing (`03c`).
+- **Logged-in / account-based bypass** (sticky/static proxies + credential management) — deferred to the platform-actor work (umbrella Phase 8 + `03b` static-proxy hand-off). Public data only this MVP.
+- Building the paid-unblocker provider integration — deferred (§8 leaves only the seam + flag).
+- **Measuring** undetectability (the scorecard that grades these levers) → `03f` (manual harness). This plan *builds* the levers; `03f` *tests* them.
+- Platform-specific structured extractors (Google Maps, LinkedIn public, …) — these sit **on top** of this hardened fetch core as Phase-8 actors; this plan only delivers the core they depend on.

plans/backend/03f-undetectability-testing.md

@@ -0,0 +1,124 @@
+# Phase 3f — Undetectability & extraction test harness (manual scorecard)
+
+> Part of **Phase 3 — WebURL Crawler & Crawl Billing**. See `00-umbrella-plan.md`.
+> **Status: ACTIVE, sequenced last in Phase 3** (after `03a`–`03e` ship — the harness measures what those built). **Manual-only** — no CI/automated gating for now (resolved decision); it's a dev-run scorecard, not a build gate. Depends on `03a` (the Scrapling tiers + `FetchStrategy`/`CrawlOutcome`), `03b` (proxy provider), `03e` (the stealth levers being tested), and reuses the `page_action` + closure-cell mechanism shared with `03d`.
+
+> **Convention note.** This is **dev/operator tooling**, not a product code path, so it's untouched by the Phase 1–2 rename (no `search_space_id`/`workspace_id` concern). Citations to the gitignored reference checkouts (`references/CloakBrowser/`, `references/Scrapling/`) are pinned to what's on disk; locate code by **symbol/grep** if lines drift.
+
+## Objective
+
+A **manual, repeatable scorecard** that answers one question: *how undetectable (and how correct) is our Universal WebURL Crawler right now?* It drives the real Scrapling tiers against the industry's standard detection + sandbox sites, parses each site's verdict, and prints a pass/fail + numeric scorecard. Its real job is to **quantify the free-stack ceiling** (`03e`) over time so we know — with evidence — when fingerprint maintenance stops being worth it and we should flip the deferred paid-unblocker tier.
+
+It is explicitly **two axes, two suites** (kept separate so each scales independently):
+
+- **Suite S — Stealth / anti-bot** (is the crawler detected?): browser-tier fingerprint/bot tests, HTTP/TLS-tier fingerprint, proxy/leak verification.
+- **Suite E — Extraction correctness** (does the crawler get the content right?): scraping sandboxes for the HTTP vs JS (DynamicFetcher) tiers + Trafilatura quality.
+
+## How CloakBrowser tests (the pattern we copy)
+
+CloakBrowser's suite is a **driven-browser verdict-scraper**, not unit tests (`references/CloakBrowser/examples/stealth_test.py`, run via `bin/cloaktest`). The repeated shape:
+
+1. Launch a real page with **proxy + geoip**, toggleable **headed/headless** (`stealth_test.py:231` `launch(headless=…, proxy=…, geoip=True)`).
+2. `page.goto(site, wait_until="networkidle")` then **sleep** — scores compute async (Castle.js ~20 s `fingerprint_scan_test.py:31`; CreepJS ~30 s `:96`; reCAPTCHA polls up to 30 s `stealth_test.py:156–164`).
+3. `page.evaluate(js)` to **parse the verdict** from the rendered DOM (sannysoft table `stealth_test.py:32–45`) or from internal JS objects (`window.Fingerprint.headless` `fingerprint_scan_test.py:117–127`).
+4. Apply an explicit **pass threshold** + screenshot (`stealth_test.py:169–217`).
+
+Their shipped bars (we adopt as **aspirational targets**, see Scorecard): sannysoft **0 fails**; `bot.incolumitas` ≤ `{WEBDRIVER, connectionRTT}` known-FPs; `browserscan` **0 Abnormal**; `deviceandbrowserinfo` `isBot=false`; FingerprintJS demo **not blocked**; reCAPTCHA v3 **≥0.7** (they hit 0.9); CreepJS **headless ≤30% / stealth ≤30%** (`fingerprint_scan_test.py:112–114,166–171`).
+
+## The bridge to our Scrapling crawler
+
+We are **not** a single browser; we're the `03a` tier ladder. The harness therefore tests **per tier**, and extracts verdicts two ways:
+
+- **DOM/JSON-rendered verdicts** → just `StealthyFetcher.fetch(url, …)` (or `Fetcher.get` for JSON endpoints) and parse the **returned post-JS page** with Scrapling's selector (`load_dom` is on by default — `references/Scrapling/scrapling/fetchers/stealth_chrome.py:43`). Covers sannysoft, incolumitas, deviceandbrowserinfo, the reCAPTCHA score text, and every JSON endpoint (`tls.peet.ws/api/all`, `httpbin/headers`).
+- **Internal JS-object verdicts** (CreepJS `window.Fingerprint`, Castle.js score node) → a **`page_action`** that runs `page.evaluate()` and stashes the result into a **closure cell**, because Scrapling **discards `page_action`'s return value** (sync `_stealth.py:260`, async `:536`). **This is the exact same `page_action`+closure-cell plumbing as `03d`'s token injector** — building the harness de-risks `03d` (and vice-versa); factor it once.
+
+## Suite S — Stealth / anti-bot
+
+### S1. Browser tier (StealthyFetcher — the "undetectable" tier)
+
+| Site | Signal | Extraction | Aspirational bar |
+|---|---|---|---|
+| `bot.sannysoft.com` | webdriver/chrome/plugins/UA leaks | DOM table | 0 fails |
+| `bot.incolumitas.com` | 30+ checks incl. behavioral | JSON-in-body | ≤ `{WEBDRIVER, connectionRTT}` |
+| `browserscan.net/bot-detection` | WebDriver/CDP/Navigator | DOM text | 0 Abnormal |
+| `deviceandbrowserinfo.com/are_you_a_bot` | fingerprint + behavioral | JSON `isBot` | `isBot=false` |
+| `abrahamjuliot.github.io/creepjs` | fingerprint **consistency/lies**, headless% | `window.Fingerprint` (page_action) | headless ≤30%, stealth ≤30% |
+| `fingerprint-scan.com` | Castle.js bot-risk + headless signals | DOM node + evaluate | low risk; headless signals 0 |
+| `demo.fingerprint.com/web-scraping` | **behavioral block** (click→blocked vs flights) | page_action click + DOM | not blocked |
+| `recaptcha-demo.appspot.com/...v3-request-scores.php` | Google server-verified human score | `wait_for_function` + regex (`recaptcha_score.py:22–28`) | score ≥0.7 |
+
+### S2. Per-property fingerprint detail (manual/debug — validates `03e` levers directly)
+
+- `browserleaks.com/canvas`, `/webgl`, `/fonts` — confirms `03e` `hide_canvas` + font packages.
+- `browserleaks.com/webrtc` + DNS leak — confirms `03e` `block_webrtc` (no real-IP leak through the proxy).
+
+### S3. HTTP/TLS tier (AsyncFetcher / `Fetcher` — curl_cffi impersonation)
+
+Our HTTP tier is `curl_cffi`-based and *can* impersonate a real Chrome TLS stack (`references/Scrapling/scrapling/fetchers/requests.py:29`; `engines/static.py:6–9,36–47`) — **but only if `impersonate=` is set.** It currently is **not** (`webcrawler_connector.py:284–289` passes `stealthy_headers=True` only), so today this tier's JA3 is curl_cffi's default and **will fail parity**. This row is therefore the *driver* for the `03e §2b` `impersonate="chrome"` lever: run it before (red) and after (green) that fix.
+
+- `tls.peet.ws/api/all` (+ `/api/clean`) — JSON **JA3/JA4/Akamai-HTTP2/PeetPrint**; diff against a real-Chrome baseline.
+- `httpbin.co/headers` (or httpbingo) — header set/order/UA sanity.
+
+**Recommendation (resolved): TLS parity is a first-class *axis* but an *informational threshold*, not a hard gate.** We record JA3/JA4 and flag drift from the Chrome baseline, but don't "fail" on it — curl_cffi impersonation is strong yet JA-hashes shift across versions, and this is a manual scorecard anyway. Treat a *mismatch* as a tuning signal (pick a closer `impersonate` profile), not a regression.
+
+### S4. Proxy / leak verification
+
+- `httpbin.org/ip` — exit IP == the proxy endpoint actually used (capture-once seam from `03b`/`03e`, not a re-rotated `get_proxy_url()`).
+- WebRTC/DNS (S2) — no real-IP leak.
+
+## Suite E — Extraction correctness (separate axis)
+
+Purpose-built, ToS-safe sandboxes — validates the HTTP vs **DynamicFetcher (JS)** tiers and Trafilatura output, independent of stealth:
+
+- `books.toscrape.com` — static catalog + pagination (HTTP tier; baseline extraction).
+- `quotes.toscrape.com` — has **`/js`** (JS-rendered), **`/js-delayed`** (`?delay=`), **scroll** (infinite), **login** (CSRF) variants → exercises the DynamicFetcher JS tier + `wait_selector`/`network_idle`.
+- `scrapethissite.com` — mixed structures for extraction robustness.
+
+Assertion style: known expected values (e.g. first book title, quote count per page) so extraction regressions are caught deterministically.
+
+## Scorecard & thresholds (resolved)
+
+- **Adopt CloakBrowser's bars as aspirational targets** (above), but the harness's primary output is **our actual measured numbers recorded as the baseline** — a committed `scorecard.md`/JSON snapshot per run (date, tier, proxy on/off, headed/headless, per-site result). Subsequent runs diff against the last baseline so we see drift (ours improving, or a WAF tightening).
+- Each row reports: site, tier, verdict, numeric (where applicable), PASS/FAIL vs aspirational bar, and screenshot path.
+- A run is summarized as `passed/total` per suite (like `stealth_test.py:314–317`), **never blocking** anything.
+
+## Harness design
+
+- Lives under dev tooling (e.g. `surfsense_backend/scripts/crawler_testbench/` or `tests/manual/` — not collected by the normal pytest run, since it hits the live internet + needs proxies). A thin CLI mirrors `bin/cloaktest`: `python -m ... [--proxy URL] [--headed] [--headless] [--suite S|E|all] [--no-screenshots]`.
+- **Reuse split (important — `crawl_url` is *not* a drop-in here):**
+  - **Suite E** drives the **real `crawl_url`** end-to-end — extraction correctness *is* the production path (auto-ladder + Trafilatura markdown is exactly what we want to assert).
+  - **Suite S** does **not** use `crawl_url`. Two reasons: (1) `crawl_url` auto-ladders and stops at the first `SUCCESS`, so a detection site might be answered by the cheap HTTP tier when we mean to grade the **StealthyFetcher** tier; (2) `crawl_url` returns Trafilatura **markdown** (`_build_result`), but verdict parsing needs the **raw post-JS DOM** (and `window.*` objects need a live page). So Suite S drives the **individual Scrapling fetchers directly**, per tier.
+  - **Avoid test-vs-prod drift:** Suite S must construct each fetcher from the **same centralized stealth-config builder** `03e` introduces (the single source of truth for `locale`/`timezone_id`/`hide_canvas`/`block_webrtc`/`impersonate`/profile/headed), **not** a hand-rolled kwargs set — otherwise the scorecard grades a browser we don't ship. (`03e` work item: expose that builder so both the crawler and this harness import it.)
+- Outputs: console summary + screenshots + the `scorecard` snapshot.
+- Runs with the app-wide proxy provider (`03b`) and `03e` levers on, so the scorecard reflects production fetch behavior — **except captcha solving, which Suite S forces OFF** (`CAPTCHA_SOLVING_ENABLED=false`): we measure the *unaided* stealth/score, and it avoids the `03d` injector firing paid solves against the reCAPTCHA-demo / FingerprintJS rows. Captcha solving is exercised separately (functional `03d` test), not in the stealth scorecard.
+
+## The ceiling-decision loop (why this matters)
+
+The scorecard is the evidence base for the moat strategy: re-run it (a) on a cadence and (b) whenever crawls start failing in the wild. When the **hard** rows (FingerprintJS demo, reCAPTCHA-v3 ≥0.7, and any DataDome/Kasada targets) degrade and `03e` levers can't recover them, that's the documented trigger to flip the **deferred paid-unblocker `FetchStrategy`** (`03e §8`). Without this harness, that decision is guesswork.
+
+## Config / dependencies
+
+- **No new production dependencies** — reuses Scrapling (already pinned) + the crawler. Optionally a tiny dev helper for the scorecard diff (or just stdlib `json`).
+- Needs **residential proxy creds** (the `03b` env) to be meaningful; document a "no-proxy" mode that still runs but is expected to fail the harder rows (datacenter IP).
+- A results dir (gitignored screenshots; committed `scorecard` snapshots).
+
+## Work items
+
+1. **Shared `page_action`+closure-cell helper** (factored with `03d`) for JS-object verdict extraction.
+2. **Suite S runners** (S1 browser, S3 TLS, S4 proxy-leak) with per-site parse + aspirational thresholds; S2 as manual debug links.
+3. **Suite E runners** against the toscrape/scrapethissite sandboxes with deterministic assertions.
+4. **CLI + scorecard** (snapshot writer + diff vs last baseline) mirroring `bin/cloaktest` ergonomics.
+5. **Docs**: a short runbook (how to run, read the scorecard, interpret the ceiling trigger).
+
+## Risks / trade-offs
+
+- **Flaky/rate-limited/changing sites.** Detection sites move DOM and tighten over time; the harness must tolerate parse misses (report `ERROR`, not crash — `stealth_test.py:298–300`) and is **manual** precisely so flakiness never blocks development.
+- **Proxy required for realism.** Without residential egress the hard rows fail by design (datacenter IP); document this so a red scorecard isn't misread.
+- **ToS.** These are public detection/sandbox sites intended for this purpose; keep volume low and don't hammer.
+- **Not a guarantee.** Passing sannysoft/CreepJS ≠ beating DataDome/Kasada; the scorecard's value is *trend + ceiling visibility*, not a green checkmark.
+
+## Out of scope (hand-offs)
+
+- **CI automation** — deferred (resolved: manual-only now). If revisited, it needs tolerant thresholds + proxy creds + nightly cadence; the scorecard JSON is designed to make that easy later.
+- The levers themselves (`03e`), captcha solving (`03d`), proxy provider (`03b`), billing (`03c`) — this plan only **measures** them.
+- The deferred paid-unblocker integration (`03e §8`) — the harness defines its *trigger*, not its build.

surfsense_backend/alembic/versions/168_harden_refresh_token_schema.py

@@ -16,35 +16,48 @@ depends_on: str | Sequence[str] | None = None
 
 def upgrade() -> None:
     op.execute(
-        "ALTER TABLE refresh_tokens "
+        "ALTER TABLE refresh_tokens ADD COLUMN IF NOT EXISTS "
-        "ADD COLUMN IF NOT EXISTS revoked_at TIMESTAMP WITH TIME ZONE"
+        "revoked_at TIMESTAMP WITH TIME ZONE"
     )
     op.execute(
-        "ALTER TABLE refresh_tokens "
+        "ALTER TABLE refresh_tokens ADD COLUMN IF NOT EXISTS "
-        "ADD COLUMN IF NOT EXISTS absolute_expiry TIMESTAMP WITH TIME ZONE"
+        "absolute_expiry TIMESTAMP WITH TIME ZONE"
     )
+
+    bind = op.get_bind()
+    is_revoked_exists = bind.execute(
+        sa.text(
+            """
+        SELECT EXISTS (
+            SELECT FROM information_schema.columns
+            WHERE table_schema = current_schema()
+              AND table_name = 'refresh_tokens'
+              AND column_name = 'is_revoked'
+        )
+        """
+        )
+    ).scalar()
+
+    if is_revoked_exists:
+        op.execute(
+            """
+            UPDATE refresh_tokens
+            SET revoked_at = NOW()
+            WHERE is_revoked = TRUE
+              AND revoked_at IS NULL
+            """
+        )
+
     op.execute(
-        """
+        "ALTER TABLE refresh_tokens ALTER COLUMN token_hash TYPE VARCHAR(64)"
-        DO $$
-        BEGIN
-            IF EXISTS (
-                SELECT 1 FROM information_schema.columns
-                WHERE table_name = 'refresh_tokens'
-                  AND column_name = 'is_revoked'
-            ) THEN
-                UPDATE refresh_tokens SET revoked_at = NOW() WHERE is_revoked = TRUE;
-            END IF;
-        END $$;
-        """
     )
-    op.execute("ALTER TABLE refresh_tokens ALTER COLUMN token_hash TYPE VARCHAR(64)")
     op.execute("ALTER TABLE refresh_tokens DROP COLUMN IF EXISTS is_revoked")
 
 
 def downgrade() -> None:
     op.execute(
-        "ALTER TABLE refresh_tokens "
+        "ALTER TABLE refresh_tokens ADD COLUMN IF NOT EXISTS "
-        "ADD COLUMN IF NOT EXISTS is_revoked BOOLEAN NOT NULL DEFAULT false"
+        "is_revoked BOOLEAN NOT NULL DEFAULT false"
     )
     op.execute(
         """
@@ -61,6 +74,8 @@ def downgrade() -> None:
         """
     )
     op.execute("ALTER TABLE refresh_tokens ALTER COLUMN is_revoked DROP DEFAULT")
-    op.execute("ALTER TABLE refresh_tokens ALTER COLUMN token_hash TYPE VARCHAR(256)")
+    op.execute(
+        "ALTER TABLE refresh_tokens ALTER COLUMN token_hash TYPE VARCHAR(256)"
+    )
     op.execute("ALTER TABLE refresh_tokens DROP COLUMN IF EXISTS absolute_expiry")
     op.execute("ALTER TABLE refresh_tokens DROP COLUMN IF EXISTS revoked_at")

surfsense_backend/app/config/__init__.py

@@ -762,7 +762,7 @@ class Config:
     TURNSTILE_SECRET_KEY = os.getenv("TURNSTILE_SECRET_KEY", "")
 
     # Auth
-    AUTH_TYPE = os.getenv("AUTH_TYPE")
+    AUTH_TYPE = os.getenv("AUTH_TYPE", "LOCAL")
     REGISTRATION_ENABLED = os.getenv("REGISTRATION_ENABLED", "TRUE").upper() == "TRUE"
 
     # Google OAuth
@@ -916,7 +916,7 @@ class Config:
 
     # JWT Token Lifetimes
     ACCESS_TOKEN_LIFETIME_SECONDS = int(
-        os.getenv("ACCESS_TOKEN_LIFETIME_SECONDS", str(30 * 60))  # 30 minutes
+        os.getenv("ACCESS_TOKEN_LIFETIME_SECONDS", str(60 * 60))  # 60 minutes
     )
     MIN_ISSUED_AT = int(os.getenv("MIN_ISSUED_AT", "0"))
     REFRESH_TOKEN_LIFETIME_SECONDS = int(

surfsense_backend/app/tasks/document_processors/_save.py

@@ -10,6 +10,7 @@ from app.utils.document_converters import (
     create_document_chunks,
     embed_text,
     generate_content_hash,
+    truncate_for_embedding,
 )
 
 from ._helpers import (
@@ -73,7 +74,9 @@ async def save_file_document(
             if should_skip:
                 return doc
 
-        document_content = f"File: {file_name}\n\n{markdown_content[:4000]}"
+        document_content = (
+            f"File: {file_name}\n\n{truncate_for_embedding(markdown_content)}"
+        )
         document_embedding = embed_text(document_content)
         chunks = await create_document_chunks(markdown_content)
         doc_metadata = {"FILE_NAME": file_name, "ETL_SERVICE": etl_service}

surfsense_backend/app/tasks/document_processors/markdown_processor.py

@@ -13,6 +13,7 @@ from app.utils.document_converters import (
     create_document_chunks,
     embed_text,
     generate_content_hash,
+    truncate_for_embedding,
 )
 
 from ._helpers import (
@@ -182,7 +183,9 @@ async def add_received_markdown_file_document(
                 return doc
             # Content changed - continue to update
 
-        summary_content = f"File: {file_name}\n\n{file_in_markdown[:4000]}"
+        summary_content = (
+            f"File: {file_name}\n\n{truncate_for_embedding(file_in_markdown)}"
+        )
         summary_embedding = embed_text(summary_content)
 
         # Process chunks

surfsense_backend/app/utils/blocknote_to_markdown.py

@@ -145,7 +145,7 @@ def _render_block(
         code_text = _render_inline_content(content) if content else ""
         lines.append(f"{prefix}```{language}")
         for code_line in code_text.split("\n"):
-            lines.append(f"{prefix}{code_line}")
+            lines.append(code_line)
         lines.append(f"{prefix}```")
 
     elif block_type == "table":

surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx

@@ -71,7 +71,7 @@ import { useMessagesSync } from "@/hooks/use-messages-sync";
 import { useThreadDetail, useThreadMessages } from "@/hooks/use-thread-queries";
 import { getAgentFilesystemSelection } from "@/lib/agent-filesystem";
 import { documentsApiService } from "@/lib/apis/documents-api.service";
-import { getDesktopAccessToken } from "@/lib/auth-fetch";
+import { authenticatedFetch } from "@/lib/auth-fetch";
 import { type ChatFlow, classifyChatError } from "@/lib/chat/chat-error-classifier";
 import { tagPreAcceptSendFailure, toHttpResponseError } from "@/lib/chat/chat-request-errors";
 import { getMentionDocKey } from "@/lib/chat/mention-doc-key";
@@ -932,14 +932,11 @@ export default function NewChatPage() {
 	// Cancel ongoing request
 	const cancelRun = useCallback(async () => {
 		if (threadId) {
-			const token = await getDesktopAccessToken();
 			try {
-				const response = await fetch(
+				const response = await authenticatedFetch(
 					buildBackendUrl(`/api/v1/threads/${threadId}/cancel-active-turn`),
 					{
 						method: "POST",
-						headers: token ? { Authorization: `Bearer ${token}` } : undefined,
-						credentials: "include",
 					}
 				);
 				if (response.ok) {
@@ -986,8 +983,6 @@ export default function NewChatPage() {
 
 			if (!userQuery.trim() && userImages.length === 0) return;
 
-			const token = await getDesktopAccessToken();
-
 			// Lazy thread creation: create thread on first message if it doesn't exist
 			let currentThreadId = threadId;
 			let isNewThread = false;
@@ -1159,13 +1154,11 @@ export default function NewChatPage() {
 				const hasThreadIds = mentionPayload.thread_ids.length > 0;
 
 				const response = await fetchWithTurnCancellingRetry(() =>
-					fetch(buildBackendUrl("/api/v1/new_chat"), {
+					authenticatedFetch(buildBackendUrl("/api/v1/new_chat"), {
 						method: "POST",
 						headers: {
 							"Content-Type": "application/json",
-							...(token ? { Authorization: `Bearer ${token}` } : {}),
 						},
-						credentials: "include",
 						body: JSON.stringify({
 							chat_id: currentThreadId,
 							user_query: userQuery.trim(),
@@ -1546,8 +1539,6 @@ export default function NewChatPage() {
 			stagedDecisionsByInterruptIdRef.current.clear();
 			setIsRunning(true);
 
-			const token = await getDesktopAccessToken();
-
 			const controller = new AbortController();
 			abortControllerRef.current = controller;
 
@@ -1648,13 +1639,11 @@ export default function NewChatPage() {
 					localFilesystemEnabled,
 				});
 				const response = await fetchWithTurnCancellingRetry(() =>
-					fetch(buildBackendUrl(`/api/v1/threads/${resumeThreadId}/resume`), {
+					authenticatedFetch(buildBackendUrl(`/api/v1/threads/${resumeThreadId}/resume`), {
 						method: "POST",
 						headers: {
 							"Content-Type": "application/json",
-							...(token ? { Authorization: `Bearer ${token}` } : {}),
 						},
-						credentials: "include",
 						body: JSON.stringify({
 							search_space_id: searchSpaceId,
 							decisions,
@@ -1986,8 +1975,6 @@ export default function NewChatPage() {
 				abortControllerRef.current = null;
 			}
 
-			const token = await getDesktopAccessToken();
-
 			// Extract the original user query BEFORE removing messages (for reload mode)
 			let userQueryToDisplay: string | undefined;
 			let originalUserMessageContent: ThreadMessageLike["content"] | null = null;
@@ -2105,13 +2092,11 @@ export default function NewChatPage() {
 					}
 				}
 				const response = await fetchWithTurnCancellingRetry(() =>
-					fetch(getRegenerateUrl(threadId), {
+					authenticatedFetch(getRegenerateUrl(threadId), {
 						method: "POST",
 						headers: {
 							"Content-Type": "application/json",
-							...(token ? { Authorization: `Bearer ${token}` } : {}),
 						},
-						credentials: "include",
 						body: JSON.stringify(requestBody),
 						signal: controller.signal,
 					})

surfsense_web/components/assistant-ui/inline-mention-editor.tsx

@@ -696,7 +696,9 @@ export const InlineMentionEditor = forwardRef<InlineMentionEditorRef, InlineMent
 				onKeyDown?.(e);
 				if (e.defaultPrevented) return;
 
-				if (e.key === "Enter" && !e.shiftKey) {
+				// Ignore Enter while an IME composition is active (e.g. confirming a
+				// Japanese/Chinese/Korean conversion) so it doesn't submit the message.
+				if (e.key === "Enter" && !e.shiftKey && !e.nativeEvent.isComposing) {
 					e.preventDefault();
 					onSubmit?.();
 					return;

surfsense_web/components/assistant-ui/thread.tsx

@@ -713,6 +713,10 @@ const Composer: FC = () => {
 	// Arrow / Enter / Escape navigation for the active picker.
 	const handleKeyDown = useCallback(
 		(e: React.KeyboardEvent) => {
+			// While an IME composition is active (e.g. confirming a Japanese/Chinese/
+			// Korean conversion), let the Enter/Arrow keys reach the IME instead of
+			// driving picker navigation/selection.
+			if (e.nativeEvent.isComposing) return;
 			if (showPromptPicker) {
 				if (e.key === "ArrowDown") {
 					e.preventDefault();

surfsense_web/components/free-chat/free-composer.tsx

@@ -99,7 +99,9 @@ export const FreeComposer: FC = () => {
 				gate("mention documents");
 				return;
 			}
-			if (e.key === "Enter" && !e.shiftKey) {
+			// Ignore Enter while an IME composition is active (e.g. confirming a
+			// Japanese/Chinese/Korean conversion) so it doesn't submit the message.
+			if (e.key === "Enter" && !e.shiftKey && !e.nativeEvent.isComposing) {
 				e.preventDefault();
 				if (text.trim()) {
 					aui.composer().send();

surfsense_web/components/providers/ZeroProvider.tsx

@@ -8,7 +8,7 @@ import {
 import { usePathname } from "next/navigation";
 import { useEffect, useMemo, useRef, useState } from "react";
 import { useSession } from "@/hooks/use-session";
-import { getDesktopAccessToken } from "@/lib/auth-fetch";
+import { authenticatedFetch, getDesktopAccessToken } from "@/lib/auth-fetch";
 import { handleUnauthorized, isPublicRoute, refreshSession } from "@/lib/auth-utils";
 import { buildBackendUrl } from "@/lib/env-config";
 import type { Context } from "@/types/zero";
@@ -31,26 +31,14 @@ function getCacheURL() {
 }
 
 async function fetchZeroContext(isDesktop: boolean): Promise<LoadedZeroContext | null> {
-	const headers: HeadersInit = {};
+	const response = await authenticatedFetch(buildBackendUrl("/zero/context"), {
-	let desktopAuth: string | undefined;
+		skipAuthRedirect: true,
-
-	if (isDesktop) {
-		const token = await getDesktopAccessToken();
-		if (!token) return null;
-		desktopAuth = token;
-		headers.Authorization = `Bearer ${token}`;
-	}
-
-	const response = await fetch(buildBackendUrl("/zero/context"), {
-		credentials: "include",
-		headers,
 	});
-
 	if (!response.ok) return null;
 
 	return {
 		context: (await response.json()) as ZeroContext,
-		desktopAuth,
+		desktopAuth: isDesktop ? (await getDesktopAccessToken()) || undefined : undefined,
 	};
 }
 
@@ -106,7 +94,7 @@ function ZeroAuthSync({ isDesktop }: { isDesktop: boolean }) {
 					}
 
 					if (isDesktop) {
-						const newToken = await getDesktopAccessToken();
+						const newToken = await getDesktopAccessToken({ forceRefresh: true });
 						if (!newToken) {
 							handleUnauthorized();
 							return;
@@ -262,7 +250,7 @@ export function ZeroProvider({ children }: { children: React.ReactNode }) {
 	const pathname = usePathname();
 	const isDesktop = typeof window !== "undefined" && !!window.electronAPI;
 
-	if (!isDesktop && isPublicRoute(pathname)) {
+	if (isPublicRoute(pathname)) {
 		return <>{children}</>;
 	}
 

surfsense_web/components/tool-ui/sandbox-execute.tsx

@@ -16,7 +16,7 @@ import { z } from "zod";
 import { Badge } from "@/components/ui/badge";
 import { Button } from "@/components/ui/button";
 import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible";
-import { getDesktopAccessToken } from "@/lib/auth-fetch";
+import { authenticatedFetch } from "@/lib/auth-fetch";
 import { buildBackendUrl } from "@/lib/env-config";
 import { cn } from "@/lib/utils";
 
@@ -157,14 +157,10 @@ function truncateCommand(command: string, maxLen = 80): string {
 // ============================================================================
 
 async function downloadSandboxFile(threadId: string, filePath: string, fileName: string) {
-	const token = await getDesktopAccessToken();
 	const url = buildBackendUrl(`/api/v1/threads/${threadId}/sandbox/download`, {
 		path: filePath,
 	});
-	const res = await fetch(url, {
+	const res = await authenticatedFetch(url);
-		headers: token ? { Authorization: `Bearer ${token}` } : undefined,
-		credentials: "include",
-	});
 	if (!res.ok) {
 		throw new Error(`Download failed: ${res.statusText}`);
 	}

surfsense_web/hooks/use-session.ts

@@ -1,6 +1,7 @@
 "use client";
 
 import { useCallback, useEffect, useState } from "react";
+import { authenticatedFetch } from "@/lib/auth-fetch";
 import { buildBackendUrl } from "@/lib/env-config";
 
 type SessionState =
@@ -8,15 +9,6 @@ type SessionState =
 	| { status: "authenticated"; authenticated: true; accessExpiresAt: number | null }
 	| { status: "unauthenticated"; authenticated: false; accessExpiresAt: null };
 
-async function getSessionHeaders(): Promise<HeadersInit> {
-	if (typeof window === "undefined" || !window.electronAPI?.getAccessToken) {
-		return {};
-	}
-
-	const token = await window.electronAPI.getAccessToken();
-	return token ? { Authorization: `Bearer ${token}` } : {};
-}
-
 export function useSession() {
 	const [state, setState] = useState<SessionState>({
 		status: "loading",
@@ -26,9 +18,8 @@ export function useSession() {
 
 	const refresh = useCallback(async () => {
 		try {
-			const response = await fetch(buildBackendUrl("/auth/session"), {
+			const response = await authenticatedFetch(buildBackendUrl("/auth/session"), {
-				credentials: "include",
+				skipAuthRedirect: true,
-				headers: await getSessionHeaders(),
 			});
 			if (!response.ok) {
 				setState({

surfsense_web/lib/apis/base-api.service.ts

@@ -1,4 +1,5 @@
 import type { ZodType } from "zod";
+import { getDesktopAccessToken } from "@/lib/auth-fetch";
 import { buildBackendUrl } from "@/lib/env-config";
 import { getClientPlatform } from "../agent-filesystem";
 import { handleUnauthorized, refreshSession } from "../auth-utils";
@@ -59,11 +60,6 @@ class BaseApiService {
 		return typeof window !== "undefined" && !!window.electronAPI;
 	}
 
-	private async getDesktopAccessToken(): Promise<string> {
-		if (!this.isDesktopClient) return "";
-		return (await window.electronAPI?.getAccessToken?.()) || "";
-	}
-
 	async request<T, R extends ResponseType = ResponseType.JSON>(
 		url: string,
 		responseSchema?: ZodType<T>,
@@ -90,7 +86,7 @@ class BaseApiService {
 				this.noAuthPrefixes.some((prefix) => url.startsWith(prefix)) ||
 				/^\/api\/v1\/invites\/[^/]+\/info$/.test(url);
 			const desktopAccessToken =
-				this.isDesktopClient && !isNoAuthEndpoint ? await this.getDesktopAccessToken() : "";
+				this.isDesktopClient && !isNoAuthEndpoint ? (await getDesktopAccessToken()) || "" : "";
 			const defaultOptions: RequestOptions = {
 				headers: {
 					...(desktopAccessToken ? { Authorization: `Bearer ${desktopAccessToken}` } : {}),
@@ -174,7 +170,9 @@ class BaseApiService {
 					} else if (!isNoAuthEndpoint && !isRefreshRetryBlocked(refreshRetryKey)) {
 						const refreshed = await refreshSession();
 						if (refreshed) {
-							const newToken = this.isDesktopClient ? await this.getDesktopAccessToken() : "";
+							const newToken = this.isDesktopClient
+								? (await getDesktopAccessToken({ forceRefresh: true })) || ""
+								: "";
 							return this.request(url, responseSchema, {
 								...mergedOptions,
 								headers: {

surfsense_web/lib/apis/notes-api.service.ts

@@ -1,147 +0,0 @@
-import { z } from "zod";
-import { ValidationError } from "../error";
-import { baseApiService } from "./base-api.service";
-
-// Request/Response schemas
-const createNoteRequest = z.object({
-	search_space_id: z.number(),
-	title: z.string().min(1),
-	source_markdown: z.string().optional(),
-});
-
-const createNoteResponse = z.object({
-	id: z.number(),
-	title: z.string(),
-	document_type: z.string(),
-	content: z.string(),
-	content_hash: z.string(),
-	unique_identifier_hash: z.string().nullable(),
-	document_metadata: z.record(z.string(), z.any()).nullable(),
-	search_space_id: z.number(),
-	created_at: z.string(),
-	updated_at: z.string().nullable(),
-});
-
-const getNotesRequest = z.object({
-	search_space_id: z.number(),
-	skip: z.number().optional(),
-	page: z.number().optional(),
-	page_size: z.number().optional(),
-});
-
-const noteItem = z.object({
-	id: z.number(),
-	title: z.string(),
-	document_type: z.string(),
-	content: z.string(),
-	content_hash: z.string(),
-	unique_identifier_hash: z.string().nullable(),
-	document_metadata: z.record(z.string(), z.any()).nullable(),
-	search_space_id: z.number(),
-	created_at: z.string(),
-	updated_at: z.string().nullable(),
-});
-
-const getNotesResponse = z.object({
-	items: z.array(noteItem),
-	total: z.number(),
-	page: z.number(),
-	page_size: z.number(),
-	has_more: z.boolean(),
-});
-
-const deleteNoteRequest = z.object({
-	search_space_id: z.number(),
-	note_id: z.number(),
-});
-
-const deleteNoteResponse = z.object({
-	message: z.string(),
-	note_id: z.number(),
-});
-
-// Type exports
-export type CreateNoteRequest = z.infer<typeof createNoteRequest>;
-export type CreateNoteResponse = z.infer<typeof createNoteResponse>;
-export type GetNotesRequest = z.infer<typeof getNotesRequest>;
-export type GetNotesResponse = z.infer<typeof getNotesResponse>;
-export type NoteItem = z.infer<typeof noteItem>;
-export type DeleteNoteRequest = z.infer<typeof deleteNoteRequest>;
-export type DeleteNoteResponse = z.infer<typeof deleteNoteResponse>;
-
-class NotesApiService {
-	/**
-	 * Create a new note
-	 */
-	createNote = async (request: CreateNoteRequest) => {
-		const parsedRequest = createNoteRequest.safeParse(request);
-
-		if (!parsedRequest.success) {
-			console.error("Invalid request:", parsedRequest.error);
-			const errorMessage = parsedRequest.error.issues.map((issue) => issue.message).join(", ");
-			throw new ValidationError(`Invalid request: ${errorMessage}`);
-		}
-
-		const { search_space_id, title, source_markdown } = parsedRequest.data;
-
-		// Send both title and source_markdown in request body
-		const body = {
-			title,
-			...(source_markdown !== undefined && { source_markdown }),
-		};
-
-		return baseApiService.post(
-			`/api/v1/search-spaces/${search_space_id}/notes`,
-			createNoteResponse,
-			{ body }
-		);
-	};
-
-	/**
-	 * Get list of notes
-	 */
-	getNotes = async (request: GetNotesRequest) => {
-		const parsedRequest = getNotesRequest.safeParse(request);
-
-		if (!parsedRequest.success) {
-			console.error("Invalid request:", parsedRequest.error);
-			const errorMessage = parsedRequest.error.issues.map((issue) => issue.message).join(", ");
-			throw new ValidationError(`Invalid request: ${errorMessage}`);
-		}
-
-		const { search_space_id, skip, page, page_size } = parsedRequest.data;
-
-		// Build query params
-		const params = new URLSearchParams();
-		if (skip !== undefined) params.append("skip", String(skip));
-		if (page !== undefined) params.append("page", String(page));
-		if (page_size !== undefined) params.append("page_size", String(page_size));
-
-		return baseApiService.get(
-			`/api/v1/search-spaces/${search_space_id}/notes?${params.toString()}`,
-			getNotesResponse
-		);
-	};
-
-	/**
-	 * Delete a note
-	 */
-	deleteNote = async (request: DeleteNoteRequest) => {
-		const parsedRequest = deleteNoteRequest.safeParse(request);
-
-		if (!parsedRequest.success) {
-			console.error("Invalid request:", parsedRequest.error);
-			const errorMessage = parsedRequest.error.issues.map((issue) => issue.message).join(", ");
-			throw new ValidationError(`Invalid request: ${errorMessage}`);
-		}
-
-		const { search_space_id, note_id } = parsedRequest.data;
-
-		return baseApiService.delete(
-			`/api/v1/search-spaces/${search_space_id}/notes/${note_id}`,
-			deleteNoteResponse
-		);
-	};
-}
-
-export const notesApiService = new NotesApiService();

surfsense_web/lib/auth-fetch.ts

@@ -3,6 +3,16 @@ import { handleUnauthorized, isDesktopClient, refreshSession } from "@/lib/auth-
 let desktopAccessToken: string | null = null;
 let didSubscribeToDesktopAuth = false;
 
+type DesktopAccessTokenOptions = {
+	forceRefresh?: boolean;
+};
+
+type AuthenticatedFetchOptions = RequestInit & {
+	skipAuthRedirect?: boolean;
+	skipRefresh?: boolean;
+	forceDesktopTokenRefresh?: boolean;
+};
+
 function subscribeToDesktopAuth(): void {
 	if (didSubscribeToDesktopAuth || typeof window === "undefined" || !window.electronAPI) {
 		return;
@@ -17,10 +27,12 @@ function subscribeToDesktopAuth(): void {
 	});
 }
 
-export async function getDesktopAccessToken(): Promise<string | null> {
+export async function getDesktopAccessToken(
+	options: DesktopAccessTokenOptions = {}
+): Promise<string | null> {
 	if (!isDesktopClient()) return null;
 	subscribeToDesktopAuth();
-	if (desktopAccessToken) return desktopAccessToken;
+	if (desktopAccessToken && !options.forceRefresh) return desktopAccessToken;
 	const token = (await window.electronAPI?.getAccessToken?.()) || null;
 	desktopAccessToken = token;
 	return token;
@@ -34,42 +46,61 @@ export function getAuthHeaders(additionalHeaders?: Record<string, string>): Reco
 	};
 }
 
+async function fetchWithAuth(
+	url: string,
+	options: RequestInit,
+	{ forceDesktopTokenRefresh = false }: { forceDesktopTokenRefresh?: boolean } = {}
+): Promise<Response> {
+	const headers = new Headers(options.headers);
+	const token = await getDesktopAccessToken({ forceRefresh: forceDesktopTokenRefresh });
+	if (token) {
+		headers.set("Authorization", `Bearer ${token}`);
+	}
+
+	return fetch(url, {
+		...options,
+		headers,
+		credentials: options.credentials ?? "include",
+	});
+}
+
 export async function authenticatedFetch(
 	url: string,
-	options?: RequestInit & { skipAuthRedirect?: boolean; skipRefresh?: boolean }
+	options: AuthenticatedFetchOptions = {}
 ): Promise<Response> {
-	const { skipAuthRedirect = false, skipRefresh = false, ...fetchOptions } = options || {};
+	const {
-	const token = await getDesktopAccessToken();
+		skipAuthRedirect = false,
-	const headers = {
+		skipRefresh = false,
-		...(fetchOptions.headers as Record<string, string>),
+		forceDesktopTokenRefresh = false,
-		...(token ? { Authorization: `Bearer ${token}` } : {}),
+		...fetchOptions
-	};
+	} = options;
 
-	const response = await fetch(url, {
+	const response = await fetchWithAuth(url, fetchOptions, {
-		...fetchOptions,
+		forceDesktopTokenRefresh,
-		headers,
-		credentials: "include",
 	});
 
-	if (response.status === 401 && !skipAuthRedirect) {
+	if (response.status !== 401) {
-		if (!skipRefresh) {
+		return response;
-			const refreshed = await refreshSession();
+	}
-			if (refreshed) {
-				const newToken = await getDesktopAccessToken();
-				return fetch(url, {
-					...fetchOptions,
-					headers: {
-						...(fetchOptions.headers as Record<string, string>),
-						...(newToken ? { Authorization: `Bearer ${newToken}` } : {}),
-					},
-					credentials: "include",
-				});
-			}
-		}
 
+	let unauthorizedResponse = response;
+	if (!skipRefresh) {
+		const refreshed = await refreshSession();
+		if (refreshed) {
+			const retryResponse = await fetchWithAuth(url, fetchOptions, {
+				forceDesktopTokenRefresh: true,
+			});
+			if (retryResponse.status !== 401) {
+				return retryResponse;
+			}
+			unauthorizedResponse = retryResponse;
+		}
+	}
+
+	if (!skipAuthRedirect) {
 		handleUnauthorized();
 		throw new Error("Unauthorized: Redirecting to login page");
 	}
 
-	return response;
+	return unauthorizedResponse;
 }

surfsense_web/lib/auth-utils.ts

@@ -188,9 +188,23 @@ async function doRefreshSession(): Promise<boolean> {
 	}
 }
 
+let refreshPromise: Promise<boolean> | null = null;
+
 export async function refreshSession(): Promise<boolean> {
-	if (typeof navigator !== "undefined" && "locks" in navigator) {
+	if (refreshPromise) {
-		return navigator.locks.request("ss-token-refresh", () => doRefreshSession());
+		return refreshPromise;
+	}
+
+	refreshPromise = (async () => {
+		if (typeof navigator !== "undefined" && "locks" in navigator) {
+			return navigator.locks.request("ss-token-refresh", () => doRefreshSession());
+		}
+		return doRefreshSession();
+	})();
+
+	try {
+		return await refreshPromise;
+	} finally {
+		refreshPromise = null;
 	}
-	return doRefreshSession();
 }

surfsense_web/lib/env-config.ts

@@ -8,9 +8,9 @@
 
 import packageJson from "../package.json";
 
-// Build-time fallback for packaged clients. Docker runtime reads plain AUTH_TYPE
+// Build-time fallback for packaged clients. Docker/runtime deployments default to
-// through the runtime config provider first, then falls back to this baked value.
+// local auth unless AUTH_TYPE/NEXT_PUBLIC_AUTH_TYPE explicitly selects Google.
-export const BUILD_TIME_AUTH_TYPE = process.env.NEXT_PUBLIC_AUTH_TYPE || "GOOGLE";
+export const BUILD_TIME_AUTH_TYPE = process.env.NEXT_PUBLIC_AUTH_TYPE || "LOCAL";
 
 // Backend API URL. An empty string is valid in proxy mode and means
 // same-origin relative requests (e.g. /api/v1/... and /auth/...).

surfsense_web/lib/runtime-auth-config.ts

@@ -4,7 +4,7 @@ export type RuntimeAuthUiMode = "GOOGLE" | "LOCAL";
 
 export function resolveRuntimeAuthUiMode(
 	value: string | null | undefined,
-	fallback: string | null | undefined = "GOOGLE"
+	fallback: string | null | undefined = "LOCAL"
 ): RuntimeAuthUiMode {
 	const candidate = value?.trim().toUpperCase();
 	if (candidate === "GOOGLE") return "GOOGLE";