From 147c69005179f117adfe4491a13d36294d925e8d Mon Sep 17 00:00:00 2001 From: DhruvTilva Date: Fri, 26 Jun 2026 00:04:41 +0530 Subject: [PATCH 01/19] fix: prevent spurious leading spaces in nested codeBlock interior lines When a codeBlock is nested inside an indented structure like a bulletListItem, the \_render_block\ function prepends the block's indentation \prefix\ to every line. However, for codeBlock elements, only the fence markers (the opening and closing \\\) should carry the block indentation. Interior code lines must not have the prefix prepended, because markdown parsers treat leading spaces inside a code fence as part of the code content. This fix removes the prefix from interior code lines to prevent code snippets stored in notes from gaining spurious whitespace. --- surfsense_backend/app/utils/blocknote_to_markdown.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/surfsense_backend/app/utils/blocknote_to_markdown.py b/surfsense_backend/app/utils/blocknote_to_markdown.py index 3731b4b3c..8c172c7bd 100644 --- a/surfsense_backend/app/utils/blocknote_to_markdown.py +++ b/surfsense_backend/app/utils/blocknote_to_markdown.py @@ -133,7 +133,7 @@ def _render_block( code_text = _render_inline_content(content) if content else "" lines.append(f"{prefix}```{language}") for code_line in code_text.split("\n"): - lines.append(f"{prefix}{code_line}") + lines.append(code_line) lines.append(f"{prefix}```") elif block_type == "table": From bf805fd81ef651df1a8d288d311b2d633c05a016 Mon Sep 17 00:00:00 2001 From: DhruvTilva Date: Fri, 26 Jun 2026 00:09:11 +0530 Subject: [PATCH 02/19] fix: use token-safe truncation for document embeddings The document saving logic used a hardcoded character slice ([:4000]) for the document summary content fed to the embedder. For UTF-8 documents (e.g., Arabic, Chinese, Japanese), characters above U+007F take multiple bytes but count as 1 character in a slice, potentially producing text that exceeds the token limit of the embedding model. Replaced the arbitrary slice with runcate_for_embedding(), which safely bounds the text using the embedding model's actual tokenizer. --- surfsense_backend/app/tasks/document_processors/_save.py | 5 ++++- .../app/tasks/document_processors/markdown_processor.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/surfsense_backend/app/tasks/document_processors/_save.py b/surfsense_backend/app/tasks/document_processors/_save.py index 3b9616cbd..9456e2f1c 100644 --- a/surfsense_backend/app/tasks/document_processors/_save.py +++ b/surfsense_backend/app/tasks/document_processors/_save.py @@ -10,6 +10,7 @@ from app.utils.document_converters import ( create_document_chunks, embed_text, generate_content_hash, + truncate_for_embedding, ) from ._helpers import ( @@ -73,7 +74,9 @@ async def save_file_document( if should_skip: return doc - document_content = f"File: {file_name}\n\n{markdown_content[:4000]}" + document_content = ( + f"File: {file_name}\n\n{truncate_for_embedding(markdown_content)}" + ) document_embedding = embed_text(document_content) chunks = await create_document_chunks(markdown_content) doc_metadata = {"FILE_NAME": file_name, "ETL_SERVICE": etl_service} diff --git a/surfsense_backend/app/tasks/document_processors/markdown_processor.py b/surfsense_backend/app/tasks/document_processors/markdown_processor.py index 19a4df87d..463951a64 100644 --- a/surfsense_backend/app/tasks/document_processors/markdown_processor.py +++ b/surfsense_backend/app/tasks/document_processors/markdown_processor.py @@ -13,6 +13,7 @@ from app.utils.document_converters import ( create_document_chunks, embed_text, generate_content_hash, + truncate_for_embedding, ) from ._helpers import ( @@ -182,7 +183,9 @@ async def add_received_markdown_file_document( return doc # Content changed - continue to update - summary_content = f"File: {file_name}\n\n{file_in_markdown[:4000]}" + summary_content = ( + f"File: {file_name}\n\n{truncate_for_embedding(file_in_markdown)}" + ) summary_embedding = embed_text(summary_content) # Process chunks From a80cb8c0605e73962bd68d89ae780ec71f4a471b Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Fri, 26 Jun 2026 18:52:42 +0530 Subject: [PATCH 03/19] feat: update authentication defaults to LOCAL - Changed default AUTH_TYPE in backend configuration to "LOCAL". - Updated frontend environment configuration to reflect the new default for packaged clients. - Adjusted runtime authentication resolution to use "LOCAL" as the fallback value. --- surfsense_backend/app/config/__init__.py | 2 +- surfsense_web/lib/env-config.ts | 6 +++--- surfsense_web/lib/runtime-auth-config.ts | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index 47e529741..199b9b579 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -762,7 +762,7 @@ class Config: TURNSTILE_SECRET_KEY = os.getenv("TURNSTILE_SECRET_KEY", "") # Auth - AUTH_TYPE = os.getenv("AUTH_TYPE") + AUTH_TYPE = os.getenv("AUTH_TYPE", "LOCAL") REGISTRATION_ENABLED = os.getenv("REGISTRATION_ENABLED", "TRUE").upper() == "TRUE" # Google OAuth diff --git a/surfsense_web/lib/env-config.ts b/surfsense_web/lib/env-config.ts index 8c671029c..c60ada372 100644 --- a/surfsense_web/lib/env-config.ts +++ b/surfsense_web/lib/env-config.ts @@ -8,9 +8,9 @@ import packageJson from "../package.json"; -// Build-time fallback for packaged clients. Docker runtime reads plain AUTH_TYPE -// through the runtime config provider first, then falls back to this baked value. -export const BUILD_TIME_AUTH_TYPE = process.env.NEXT_PUBLIC_AUTH_TYPE || "GOOGLE"; +// Build-time fallback for packaged clients. Docker/runtime deployments default to +// local auth unless AUTH_TYPE/NEXT_PUBLIC_AUTH_TYPE explicitly selects Google. +export const BUILD_TIME_AUTH_TYPE = process.env.NEXT_PUBLIC_AUTH_TYPE || "LOCAL"; // Backend API URL. An empty string is valid in proxy mode and means // same-origin relative requests (e.g. /api/v1/... and /auth/...). diff --git a/surfsense_web/lib/runtime-auth-config.ts b/surfsense_web/lib/runtime-auth-config.ts index 9e8d1921d..d619c5957 100644 --- a/surfsense_web/lib/runtime-auth-config.ts +++ b/surfsense_web/lib/runtime-auth-config.ts @@ -4,7 +4,7 @@ export type RuntimeAuthUiMode = "GOOGLE" | "LOCAL"; export function resolveRuntimeAuthUiMode( value: string | null | undefined, - fallback: string | null | undefined = "GOOGLE" + fallback: string | null | undefined = "LOCAL" ): RuntimeAuthUiMode { const candidate = value?.trim().toUpperCase(); if (candidate === "GOOGLE") return "GOOGLE"; From de4507f4139709237cbff144b8edeac5d866a47e Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Fri, 26 Jun 2026 21:00:55 +0530 Subject: [PATCH 04/19] feat: enhance session management in chat page - Implemented session refresh logic in the fetchWithTurnCancellingRetry function to handle 401 errors more gracefully. - Added a new import for refreshSession utility to facilitate session renewal. --- .../[search_space_id]/new-chat/[[...chat_id]]/page.tsx | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx index 9c3a7c617..407a9c53b 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx @@ -72,6 +72,7 @@ import { useThreadDetail, useThreadMessages } from "@/hooks/use-thread-queries"; import { getAgentFilesystemSelection } from "@/lib/agent-filesystem"; import { documentsApiService } from "@/lib/apis/documents-api.service"; import { getDesktopAccessToken } from "@/lib/auth-fetch"; +import { refreshSession } from "@/lib/auth-utils"; import { type ChatFlow, classifyChatError } from "@/lib/chat/chat-error-classifier"; import { tagPreAcceptSendFailure, toHttpResponseError } from "@/lib/chat/chat-request-errors"; import { getMentionDocKey } from "@/lib/chat/mention-doc-key"; @@ -688,11 +689,19 @@ export default function NewChatPage() { const fetchWithTurnCancellingRetry = useCallback(async (runFetch: () => Promise) => { const maxAttempts = 4; + let didRefreshAuth = false; for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { const response = await runFetch(); if (response.ok) { return response; } + if (response.status === 401 && !didRefreshAuth) { + didRefreshAuth = true; + const refreshed = await refreshSession(); + if (refreshed) { + continue; + } + } const error = await toHttpResponseError(response); const withMeta = error as Error & { errorCode?: string; retryAfterMs?: number }; const isTurnCancelling = withMeta.errorCode === "TURN_CANCELLING"; From 23c128dd0d494f80cd9164eaff94ecc1148cb366 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Fri, 26 Jun 2026 21:24:44 +0530 Subject: [PATCH 05/19] feat: improve fetchZeroContext with enhanced session handling - Refactored fetchZeroContext to include a buildHeaders function for better header management. - Added a request function to handle 401 errors and refresh sessions as needed. - Improved overall session management for desktop authentication. --- .../components/providers/ZeroProvider.tsx | 44 ++++++++++++++----- 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/surfsense_web/components/providers/ZeroProvider.tsx b/surfsense_web/components/providers/ZeroProvider.tsx index 1a157a854..05ff1b4b1 100644 --- a/surfsense_web/components/providers/ZeroProvider.tsx +++ b/surfsense_web/components/providers/ZeroProvider.tsx @@ -31,21 +31,41 @@ function getCacheURL() { } async function fetchZeroContext(isDesktop: boolean): Promise { - const headers: HeadersInit = {}; - let desktopAuth: string | undefined; + const buildHeaders = async ( + forceRefresh = false + ): Promise<{ headers: HeadersInit; desktopAuth?: string } | null> => { + const headers: HeadersInit = {}; - if (isDesktop) { - const token = await getDesktopAccessToken(); - if (!token) return null; - desktopAuth = token; - headers.Authorization = `Bearer ${token}`; + if (isDesktop) { + const token = await getDesktopAccessToken({ forceRefresh }); + if (!token) return null; + headers.Authorization = `Bearer ${token}`; + return { headers, desktopAuth: token }; + } + + return { headers }; + }; + + const request = async (forceRefresh = false) => { + const auth = await buildHeaders(forceRefresh); + if (!auth) return null; + const response = await fetch(buildBackendUrl("/zero/context"), { + credentials: "include", + headers: auth.headers, + }); + return { response, desktopAuth: auth.desktopAuth }; + }; + + let result = await request(); + if (result?.response.status === 401) { + const refreshed = await refreshSession(); + if (refreshed) { + result = await request(true); + } } - const response = await fetch(buildBackendUrl("/zero/context"), { - credentials: "include", - headers, - }); - + if (!result) return null; + const { response, desktopAuth } = result; if (!response.ok) return null; return { From 013fae6eba142e72cb3851faf7183861e40ffd73 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Fri, 26 Jun 2026 21:25:00 +0530 Subject: [PATCH 06/19] feat: enhance session handling in useSession and auth-fetch - Introduced fetchSession function to streamline session fetching logic. - Updated useSession to handle 401 errors by refreshing the session when necessary. - Modified getDesktopAccessToken to accept options for forced token refresh, improving desktop authentication flow. --- surfsense_web/hooks/use-session.ts | 19 +++++++++++++++---- surfsense_web/lib/auth-fetch.ts | 12 +++++++++--- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/surfsense_web/hooks/use-session.ts b/surfsense_web/hooks/use-session.ts index 6bb10456f..0012e8763 100644 --- a/surfsense_web/hooks/use-session.ts +++ b/surfsense_web/hooks/use-session.ts @@ -1,6 +1,7 @@ "use client"; import { useCallback, useEffect, useState } from "react"; +import { refreshSession } from "@/lib/auth-utils"; import { buildBackendUrl } from "@/lib/env-config"; type SessionState = @@ -17,6 +18,13 @@ async function getSessionHeaders(): Promise { return token ? { Authorization: `Bearer ${token}` } : {}; } +async function fetchSession(): Promise { + return fetch(buildBackendUrl("/auth/session"), { + credentials: "include", + headers: await getSessionHeaders(), + }); +} + export function useSession() { const [state, setState] = useState({ status: "loading", @@ -26,10 +34,13 @@ export function useSession() { const refresh = useCallback(async () => { try { - const response = await fetch(buildBackendUrl("/auth/session"), { - credentials: "include", - headers: await getSessionHeaders(), - }); + let response = await fetchSession(); + if (response.status === 401) { + const refreshed = await refreshSession(); + if (refreshed) { + response = await fetchSession(); + } + } if (!response.ok) { setState({ status: "unauthenticated", diff --git a/surfsense_web/lib/auth-fetch.ts b/surfsense_web/lib/auth-fetch.ts index 20b236854..4f512c2ca 100644 --- a/surfsense_web/lib/auth-fetch.ts +++ b/surfsense_web/lib/auth-fetch.ts @@ -3,6 +3,10 @@ import { handleUnauthorized, isDesktopClient, refreshSession } from "@/lib/auth- let desktopAccessToken: string | null = null; let didSubscribeToDesktopAuth = false; +type DesktopAccessTokenOptions = { + forceRefresh?: boolean; +}; + function subscribeToDesktopAuth(): void { if (didSubscribeToDesktopAuth || typeof window === "undefined" || !window.electronAPI) { return; @@ -17,10 +21,12 @@ function subscribeToDesktopAuth(): void { }); } -export async function getDesktopAccessToken(): Promise { +export async function getDesktopAccessToken( + options: DesktopAccessTokenOptions = {} +): Promise { if (!isDesktopClient()) return null; subscribeToDesktopAuth(); - if (desktopAccessToken) return desktopAccessToken; + if (desktopAccessToken && !options.forceRefresh) return desktopAccessToken; const token = (await window.electronAPI?.getAccessToken?.()) || null; desktopAccessToken = token; return token; @@ -55,7 +61,7 @@ export async function authenticatedFetch( if (!skipRefresh) { const refreshed = await refreshSession(); if (refreshed) { - const newToken = await getDesktopAccessToken(); + const newToken = await getDesktopAccessToken({ forceRefresh: true }); return fetch(url, { ...fetchOptions, headers: { From ef9b5b42a837b84715ab091e31cc50d46d84720d Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Fri, 26 Jun 2026 21:25:27 +0530 Subject: [PATCH 07/19] feat: update JWT token lifetime and enhance header management in chat page - Increased ACCESS_TOKEN_LIFETIME_SECONDS from 30 minutes to 60 minutes for improved session duration. - Introduced getRequestHeadersWithCurrentDesktopAuth function to streamline authorization header management across fetch requests in the chat page. --- surfsense_backend/app/config/__init__.py | 2 +- .../new-chat/[[...chat_id]]/page.tsx | 40 +++++++++---------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index 47e529741..63de28bfc 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -916,7 +916,7 @@ class Config: # JWT Token Lifetimes ACCESS_TOKEN_LIFETIME_SECONDS = int( - os.getenv("ACCESS_TOKEN_LIFETIME_SECONDS", str(30 * 60)) # 30 minutes + os.getenv("ACCESS_TOKEN_LIFETIME_SECONDS", str(60 * 60)) # 60 minutes ) MIN_ISSUED_AT = int(os.getenv("MIN_ISSUED_AT", "0")) REFRESH_TOKEN_LIFETIME_SECONDS = int( diff --git a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx index 407a9c53b..a9851cbbc 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx @@ -289,6 +289,16 @@ const TURN_CANCELLING_BACKOFF_FACTOR = 2; const TURN_CANCELLING_MAX_DELAY_MS = 1500; const RECENT_CANCEL_WINDOW_MS = 5_000; +async function getRequestHeadersWithCurrentDesktopAuth( + headers: Record = {} +): Promise> { + const token = await getDesktopAccessToken({ forceRefresh: true }); + return { + ...headers, + ...(token ? { Authorization: `Bearer ${token}` } : {}), + }; +} + function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } @@ -941,13 +951,12 @@ export default function NewChatPage() { // Cancel ongoing request const cancelRun = useCallback(async () => { if (threadId) { - const token = await getDesktopAccessToken(); try { const response = await fetch( buildBackendUrl(`/api/v1/threads/${threadId}/cancel-active-turn`), { method: "POST", - headers: token ? { Authorization: `Bearer ${token}` } : undefined, + headers: await getRequestHeadersWithCurrentDesktopAuth(), credentials: "include", } ); @@ -995,8 +1004,6 @@ export default function NewChatPage() { if (!userQuery.trim() && userImages.length === 0) return; - const token = await getDesktopAccessToken(); - // Lazy thread creation: create thread on first message if it doesn't exist let currentThreadId = threadId; let isNewThread = false; @@ -1167,13 +1174,12 @@ export default function NewChatPage() { const hasConnectorIds = mentionPayload.connector_ids.length > 0; const hasThreadIds = mentionPayload.thread_ids.length > 0; - const response = await fetchWithTurnCancellingRetry(() => + const response = await fetchWithTurnCancellingRetry(async () => fetch(buildBackendUrl("/api/v1/new_chat"), { method: "POST", - headers: { + headers: await getRequestHeadersWithCurrentDesktopAuth({ "Content-Type": "application/json", - ...(token ? { Authorization: `Bearer ${token}` } : {}), - }, + }), credentials: "include", body: JSON.stringify({ chat_id: currentThreadId, @@ -1555,8 +1561,6 @@ export default function NewChatPage() { stagedDecisionsByInterruptIdRef.current.clear(); setIsRunning(true); - const token = await getDesktopAccessToken(); - const controller = new AbortController(); abortControllerRef.current = controller; @@ -1656,13 +1660,12 @@ export default function NewChatPage() { const selection = await getAgentFilesystemSelection(searchSpaceId, { localFilesystemEnabled, }); - const response = await fetchWithTurnCancellingRetry(() => + const response = await fetchWithTurnCancellingRetry(async () => fetch(buildBackendUrl(`/api/v1/threads/${resumeThreadId}/resume`), { method: "POST", - headers: { + headers: await getRequestHeadersWithCurrentDesktopAuth({ "Content-Type": "application/json", - ...(token ? { Authorization: `Bearer ${token}` } : {}), - }, + }), credentials: "include", body: JSON.stringify({ search_space_id: searchSpaceId, @@ -1995,8 +1998,6 @@ export default function NewChatPage() { abortControllerRef.current = null; } - const token = await getDesktopAccessToken(); - // Extract the original user query BEFORE removing messages (for reload mode) let userQueryToDisplay: string | undefined; let originalUserMessageContent: ThreadMessageLike["content"] | null = null; @@ -2113,13 +2114,12 @@ export default function NewChatPage() { requestBody.revert_actions = true; } } - const response = await fetchWithTurnCancellingRetry(() => + const response = await fetchWithTurnCancellingRetry(async () => fetch(getRegenerateUrl(threadId), { method: "POST", - headers: { + headers: await getRequestHeadersWithCurrentDesktopAuth({ "Content-Type": "application/json", - ...(token ? { Authorization: `Bearer ${token}` } : {}), - }, + }), credentials: "include", body: JSON.stringify(requestBody), signal: controller.signal, From 9fc93e5e82475ed07e79fbd16db6be329064b54c Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Fri, 26 Jun 2026 22:03:59 +0530 Subject: [PATCH 08/19] fix(auth): centralize session refresh retry --- surfsense_web/lib/auth-utils.ts | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/surfsense_web/lib/auth-utils.ts b/surfsense_web/lib/auth-utils.ts index 47b2f043f..7372c8b66 100644 --- a/surfsense_web/lib/auth-utils.ts +++ b/surfsense_web/lib/auth-utils.ts @@ -188,9 +188,23 @@ async function doRefreshSession(): Promise { } } +let refreshPromise: Promise | null = null; + export async function refreshSession(): Promise { - if (typeof navigator !== "undefined" && "locks" in navigator) { - return navigator.locks.request("ss-token-refresh", () => doRefreshSession()); + if (refreshPromise) { + return refreshPromise; + } + + refreshPromise = (async () => { + if (typeof navigator !== "undefined" && "locks" in navigator) { + return navigator.locks.request("ss-token-refresh", () => doRefreshSession()); + } + return doRefreshSession(); + })(); + + try { + return await refreshPromise; + } finally { + refreshPromise = null; } - return doRefreshSession(); } From 4c257e912205019df38c3b2a44097b0b6b31e4e3 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Fri, 26 Jun 2026 22:04:23 +0530 Subject: [PATCH 09/19] refactor(auth): route session and zero context through shared fetch --- .../components/providers/ZeroProvider.tsx | 44 +++---------------- 1 file changed, 6 insertions(+), 38 deletions(-) diff --git a/surfsense_web/components/providers/ZeroProvider.tsx b/surfsense_web/components/providers/ZeroProvider.tsx index 05ff1b4b1..3f87ed007 100644 --- a/surfsense_web/components/providers/ZeroProvider.tsx +++ b/surfsense_web/components/providers/ZeroProvider.tsx @@ -8,7 +8,7 @@ import { import { usePathname } from "next/navigation"; import { useEffect, useMemo, useRef, useState } from "react"; import { useSession } from "@/hooks/use-session"; -import { getDesktopAccessToken } from "@/lib/auth-fetch"; +import { authenticatedFetch, getDesktopAccessToken } from "@/lib/auth-fetch"; import { handleUnauthorized, isPublicRoute, refreshSession } from "@/lib/auth-utils"; import { buildBackendUrl } from "@/lib/env-config"; import type { Context } from "@/types/zero"; @@ -31,46 +31,14 @@ function getCacheURL() { } async function fetchZeroContext(isDesktop: boolean): Promise { - const buildHeaders = async ( - forceRefresh = false - ): Promise<{ headers: HeadersInit; desktopAuth?: string } | null> => { - const headers: HeadersInit = {}; - - if (isDesktop) { - const token = await getDesktopAccessToken({ forceRefresh }); - if (!token) return null; - headers.Authorization = `Bearer ${token}`; - return { headers, desktopAuth: token }; - } - - return { headers }; - }; - - const request = async (forceRefresh = false) => { - const auth = await buildHeaders(forceRefresh); - if (!auth) return null; - const response = await fetch(buildBackendUrl("/zero/context"), { - credentials: "include", - headers: auth.headers, - }); - return { response, desktopAuth: auth.desktopAuth }; - }; - - let result = await request(); - if (result?.response.status === 401) { - const refreshed = await refreshSession(); - if (refreshed) { - result = await request(true); - } - } - - if (!result) return null; - const { response, desktopAuth } = result; + const response = await authenticatedFetch(buildBackendUrl("/zero/context"), { + skipAuthRedirect: true, + }); if (!response.ok) return null; return { context: (await response.json()) as ZeroContext, - desktopAuth, + desktopAuth: isDesktop ? (await getDesktopAccessToken()) || undefined : undefined, }; } @@ -126,7 +94,7 @@ function ZeroAuthSync({ isDesktop }: { isDesktop: boolean }) { } if (isDesktop) { - const newToken = await getDesktopAccessToken(); + const newToken = await getDesktopAccessToken({ forceRefresh: true }); if (!newToken) { handleUnauthorized(); return; From 652a25be37a9f733fbd149af886d63b7aac973b7 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Fri, 26 Jun 2026 22:04:33 +0530 Subject: [PATCH 10/19] refactor(chat): use shared authenticated fetch for chat requests --- .../new-chat/[[...chat_id]]/page.tsx | 52 +++++-------------- 1 file changed, 14 insertions(+), 38 deletions(-) diff --git a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx index a9851cbbc..64971c3bb 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx @@ -71,8 +71,7 @@ import { useMessagesSync } from "@/hooks/use-messages-sync"; import { useThreadDetail, useThreadMessages } from "@/hooks/use-thread-queries"; import { getAgentFilesystemSelection } from "@/lib/agent-filesystem"; import { documentsApiService } from "@/lib/apis/documents-api.service"; -import { getDesktopAccessToken } from "@/lib/auth-fetch"; -import { refreshSession } from "@/lib/auth-utils"; +import { authenticatedFetch } from "@/lib/auth-fetch"; import { type ChatFlow, classifyChatError } from "@/lib/chat/chat-error-classifier"; import { tagPreAcceptSendFailure, toHttpResponseError } from "@/lib/chat/chat-request-errors"; import { getMentionDocKey } from "@/lib/chat/mention-doc-key"; @@ -289,16 +288,6 @@ const TURN_CANCELLING_BACKOFF_FACTOR = 2; const TURN_CANCELLING_MAX_DELAY_MS = 1500; const RECENT_CANCEL_WINDOW_MS = 5_000; -async function getRequestHeadersWithCurrentDesktopAuth( - headers: Record = {} -): Promise> { - const token = await getDesktopAccessToken({ forceRefresh: true }); - return { - ...headers, - ...(token ? { Authorization: `Bearer ${token}` } : {}), - }; -} - function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } @@ -699,19 +688,11 @@ export default function NewChatPage() { const fetchWithTurnCancellingRetry = useCallback(async (runFetch: () => Promise) => { const maxAttempts = 4; - let didRefreshAuth = false; for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { const response = await runFetch(); if (response.ok) { return response; } - if (response.status === 401 && !didRefreshAuth) { - didRefreshAuth = true; - const refreshed = await refreshSession(); - if (refreshed) { - continue; - } - } const error = await toHttpResponseError(response); const withMeta = error as Error & { errorCode?: string; retryAfterMs?: number }; const isTurnCancelling = withMeta.errorCode === "TURN_CANCELLING"; @@ -952,12 +933,10 @@ export default function NewChatPage() { const cancelRun = useCallback(async () => { if (threadId) { try { - const response = await fetch( + const response = await authenticatedFetch( buildBackendUrl(`/api/v1/threads/${threadId}/cancel-active-turn`), { method: "POST", - headers: await getRequestHeadersWithCurrentDesktopAuth(), - credentials: "include", } ); if (response.ok) { @@ -1174,13 +1153,12 @@ export default function NewChatPage() { const hasConnectorIds = mentionPayload.connector_ids.length > 0; const hasThreadIds = mentionPayload.thread_ids.length > 0; - const response = await fetchWithTurnCancellingRetry(async () => - fetch(buildBackendUrl("/api/v1/new_chat"), { + const response = await fetchWithTurnCancellingRetry(() => + authenticatedFetch(buildBackendUrl("/api/v1/new_chat"), { method: "POST", - headers: await getRequestHeadersWithCurrentDesktopAuth({ + headers: { "Content-Type": "application/json", - }), - credentials: "include", + }, body: JSON.stringify({ chat_id: currentThreadId, user_query: userQuery.trim(), @@ -1660,13 +1638,12 @@ export default function NewChatPage() { const selection = await getAgentFilesystemSelection(searchSpaceId, { localFilesystemEnabled, }); - const response = await fetchWithTurnCancellingRetry(async () => - fetch(buildBackendUrl(`/api/v1/threads/${resumeThreadId}/resume`), { + const response = await fetchWithTurnCancellingRetry(() => + authenticatedFetch(buildBackendUrl(`/api/v1/threads/${resumeThreadId}/resume`), { method: "POST", - headers: await getRequestHeadersWithCurrentDesktopAuth({ + headers: { "Content-Type": "application/json", - }), - credentials: "include", + }, body: JSON.stringify({ search_space_id: searchSpaceId, decisions, @@ -2114,13 +2091,12 @@ export default function NewChatPage() { requestBody.revert_actions = true; } } - const response = await fetchWithTurnCancellingRetry(async () => - fetch(getRegenerateUrl(threadId), { + const response = await fetchWithTurnCancellingRetry(() => + authenticatedFetch(getRegenerateUrl(threadId), { method: "POST", - headers: await getRequestHeadersWithCurrentDesktopAuth({ + headers: { "Content-Type": "application/json", - }), - credentials: "include", + }, body: JSON.stringify(requestBody), signal: controller.signal, }) From 4b6bcaeb1bd3db5c7e9ddfee5aba95e025c965ae Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Fri, 26 Jun 2026 22:04:59 +0530 Subject: [PATCH 11/19] refactor(auth): reuse desktop token cache in API clients --- surfsense_web/components/tool-ui/sandbox-execute.tsx | 8 ++------ surfsense_web/lib/apis/base-api.service.ts | 12 +++++------- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/surfsense_web/components/tool-ui/sandbox-execute.tsx b/surfsense_web/components/tool-ui/sandbox-execute.tsx index 535968908..2bac3e1ca 100644 --- a/surfsense_web/components/tool-ui/sandbox-execute.tsx +++ b/surfsense_web/components/tool-ui/sandbox-execute.tsx @@ -16,7 +16,7 @@ import { z } from "zod"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"; -import { getDesktopAccessToken } from "@/lib/auth-fetch"; +import { authenticatedFetch } from "@/lib/auth-fetch"; import { buildBackendUrl } from "@/lib/env-config"; import { cn } from "@/lib/utils"; @@ -157,14 +157,10 @@ function truncateCommand(command: string, maxLen = 80): string { // ============================================================================ async function downloadSandboxFile(threadId: string, filePath: string, fileName: string) { - const token = await getDesktopAccessToken(); const url = buildBackendUrl(`/api/v1/threads/${threadId}/sandbox/download`, { path: filePath, }); - const res = await fetch(url, { - headers: token ? { Authorization: `Bearer ${token}` } : undefined, - credentials: "include", - }); + const res = await authenticatedFetch(url); if (!res.ok) { throw new Error(`Download failed: ${res.statusText}`); } diff --git a/surfsense_web/lib/apis/base-api.service.ts b/surfsense_web/lib/apis/base-api.service.ts index 5afb291ba..0cc5224e2 100644 --- a/surfsense_web/lib/apis/base-api.service.ts +++ b/surfsense_web/lib/apis/base-api.service.ts @@ -1,4 +1,5 @@ import type { ZodType } from "zod"; +import { getDesktopAccessToken } from "@/lib/auth-fetch"; import { buildBackendUrl } from "@/lib/env-config"; import { getClientPlatform } from "../agent-filesystem"; import { handleUnauthorized, refreshSession } from "../auth-utils"; @@ -59,11 +60,6 @@ class BaseApiService { return typeof window !== "undefined" && !!window.electronAPI; } - private async getDesktopAccessToken(): Promise { - if (!this.isDesktopClient) return ""; - return (await window.electronAPI?.getAccessToken?.()) || ""; - } - async request( url: string, responseSchema?: ZodType, @@ -90,7 +86,7 @@ class BaseApiService { this.noAuthPrefixes.some((prefix) => url.startsWith(prefix)) || /^\/api\/v1\/invites\/[^/]+\/info$/.test(url); const desktopAccessToken = - this.isDesktopClient && !isNoAuthEndpoint ? await this.getDesktopAccessToken() : ""; + this.isDesktopClient && !isNoAuthEndpoint ? (await getDesktopAccessToken()) || "" : ""; const defaultOptions: RequestOptions = { headers: { ...(desktopAccessToken ? { Authorization: `Bearer ${desktopAccessToken}` } : {}), @@ -174,7 +170,9 @@ class BaseApiService { } else if (!isNoAuthEndpoint && !isRefreshRetryBlocked(refreshRetryKey)) { const refreshed = await refreshSession(); if (refreshed) { - const newToken = this.isDesktopClient ? await this.getDesktopAccessToken() : ""; + const newToken = this.isDesktopClient + ? (await getDesktopAccessToken({ forceRefresh: true })) || "" + : ""; return this.request(url, responseSchema, { ...mergedOptions, headers: { From 3ce759f1d6b267ecaf9cd61e7062df0b881bb69b Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Fri, 26 Jun 2026 22:05:11 +0530 Subject: [PATCH 12/19] refactor(auth): streamline session handling with authenticatedFetch --- surfsense_web/hooks/use-session.ts | 28 ++--------- surfsense_web/lib/auth-fetch.ts | 79 ++++++++++++++++++++---------- 2 files changed, 56 insertions(+), 51 deletions(-) diff --git a/surfsense_web/hooks/use-session.ts b/surfsense_web/hooks/use-session.ts index 0012e8763..207475eae 100644 --- a/surfsense_web/hooks/use-session.ts +++ b/surfsense_web/hooks/use-session.ts @@ -1,7 +1,7 @@ "use client"; import { useCallback, useEffect, useState } from "react"; -import { refreshSession } from "@/lib/auth-utils"; +import { authenticatedFetch } from "@/lib/auth-fetch"; import { buildBackendUrl } from "@/lib/env-config"; type SessionState = @@ -9,22 +9,6 @@ type SessionState = | { status: "authenticated"; authenticated: true; accessExpiresAt: number | null } | { status: "unauthenticated"; authenticated: false; accessExpiresAt: null }; -async function getSessionHeaders(): Promise { - if (typeof window === "undefined" || !window.electronAPI?.getAccessToken) { - return {}; - } - - const token = await window.electronAPI.getAccessToken(); - return token ? { Authorization: `Bearer ${token}` } : {}; -} - -async function fetchSession(): Promise { - return fetch(buildBackendUrl("/auth/session"), { - credentials: "include", - headers: await getSessionHeaders(), - }); -} - export function useSession() { const [state, setState] = useState({ status: "loading", @@ -34,13 +18,9 @@ export function useSession() { const refresh = useCallback(async () => { try { - let response = await fetchSession(); - if (response.status === 401) { - const refreshed = await refreshSession(); - if (refreshed) { - response = await fetchSession(); - } - } + const response = await authenticatedFetch(buildBackendUrl("/auth/session"), { + skipAuthRedirect: true, + }); if (!response.ok) { setState({ status: "unauthenticated", diff --git a/surfsense_web/lib/auth-fetch.ts b/surfsense_web/lib/auth-fetch.ts index 4f512c2ca..a79777825 100644 --- a/surfsense_web/lib/auth-fetch.ts +++ b/surfsense_web/lib/auth-fetch.ts @@ -7,6 +7,12 @@ type DesktopAccessTokenOptions = { forceRefresh?: boolean; }; +type AuthenticatedFetchOptions = RequestInit & { + skipAuthRedirect?: boolean; + skipRefresh?: boolean; + forceDesktopTokenRefresh?: boolean; +}; + function subscribeToDesktopAuth(): void { if (didSubscribeToDesktopAuth || typeof window === "undefined" || !window.electronAPI) { return; @@ -40,42 +46,61 @@ export function getAuthHeaders(additionalHeaders?: Record): Reco }; } +async function fetchWithAuth( + url: string, + options: RequestInit, + { forceDesktopTokenRefresh = false }: { forceDesktopTokenRefresh?: boolean } = {} +): Promise { + const headers = new Headers(options.headers); + const token = await getDesktopAccessToken({ forceRefresh: forceDesktopTokenRefresh }); + if (token) { + headers.set("Authorization", `Bearer ${token}`); + } + + return fetch(url, { + ...options, + headers, + credentials: options.credentials ?? "include", + }); +} + export async function authenticatedFetch( url: string, - options?: RequestInit & { skipAuthRedirect?: boolean; skipRefresh?: boolean } + options: AuthenticatedFetchOptions = {} ): Promise { - const { skipAuthRedirect = false, skipRefresh = false, ...fetchOptions } = options || {}; - const token = await getDesktopAccessToken(); - const headers = { - ...(fetchOptions.headers as Record), - ...(token ? { Authorization: `Bearer ${token}` } : {}), - }; + const { + skipAuthRedirect = false, + skipRefresh = false, + forceDesktopTokenRefresh = false, + ...fetchOptions + } = options; - const response = await fetch(url, { - ...fetchOptions, - headers, - credentials: "include", + const response = await fetchWithAuth(url, fetchOptions, { + forceDesktopTokenRefresh, }); - if (response.status === 401 && !skipAuthRedirect) { - if (!skipRefresh) { - const refreshed = await refreshSession(); - if (refreshed) { - const newToken = await getDesktopAccessToken({ forceRefresh: true }); - return fetch(url, { - ...fetchOptions, - headers: { - ...(fetchOptions.headers as Record), - ...(newToken ? { Authorization: `Bearer ${newToken}` } : {}), - }, - credentials: "include", - }); - } - } + if (response.status !== 401) { + return response; + } + let unauthorizedResponse = response; + if (!skipRefresh) { + const refreshed = await refreshSession(); + if (refreshed) { + const retryResponse = await fetchWithAuth(url, fetchOptions, { + forceDesktopTokenRefresh: true, + }); + if (retryResponse.status !== 401) { + return retryResponse; + } + unauthorizedResponse = retryResponse; + } + } + + if (!skipAuthRedirect) { handleUnauthorized(); throw new Error("Unauthorized: Redirecting to login page"); } - return response; + return unauthorizedResponse; } From f14c471a0315a8f24caff66cb87a75183fc6d914 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Fri, 26 Jun 2026 22:15:48 +0530 Subject: [PATCH 13/19] refactor(ZeroProvider): simplify route handling by removing desktop check for public routes --- surfsense_web/components/providers/ZeroProvider.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/surfsense_web/components/providers/ZeroProvider.tsx b/surfsense_web/components/providers/ZeroProvider.tsx index 3f87ed007..cfb8fe586 100644 --- a/surfsense_web/components/providers/ZeroProvider.tsx +++ b/surfsense_web/components/providers/ZeroProvider.tsx @@ -250,7 +250,7 @@ export function ZeroProvider({ children }: { children: React.ReactNode }) { const pathname = usePathname(); const isDesktop = typeof window !== "undefined" && !!window.electronAPI; - if (!isDesktop && isPublicRoute(pathname)) { + if (isPublicRoute(pathname)) { return <>{children}; } From ab7d138b32a253c8524552d625f20aa9ac1ed67d Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Fri, 26 Jun 2026 11:30:06 -0700 Subject: [PATCH 14/19] feat(hotpatch): fix migration 168 --- .../168_harden_refresh_token_schema.py | 72 +++++++++++-------- 1 file changed, 41 insertions(+), 31 deletions(-) diff --git a/surfsense_backend/alembic/versions/168_harden_refresh_token_schema.py b/surfsense_backend/alembic/versions/168_harden_refresh_token_schema.py index fc14c8d73..1e902ea58 100644 --- a/surfsense_backend/alembic/versions/168_harden_refresh_token_schema.py +++ b/surfsense_backend/alembic/versions/168_harden_refresh_token_schema.py @@ -17,35 +17,49 @@ depends_on: str | Sequence[str] | None = None def upgrade() -> None: - op.add_column( - "refresh_tokens", - sa.Column("revoked_at", sa.TIMESTAMP(timezone=True), nullable=True), - ) - op.add_column( - "refresh_tokens", - sa.Column("absolute_expiry", sa.TIMESTAMP(timezone=True), nullable=True), + op.execute( + "ALTER TABLE refresh_tokens ADD COLUMN IF NOT EXISTS " + "revoked_at TIMESTAMP WITH TIME ZONE" ) op.execute( - """ - UPDATE refresh_tokens - SET revoked_at = NOW() - WHERE is_revoked = TRUE - """ + "ALTER TABLE refresh_tokens ADD COLUMN IF NOT EXISTS " + "absolute_expiry TIMESTAMP WITH TIME ZONE" ) - op.alter_column( - "refresh_tokens", - "token_hash", - existing_type=sa.String(length=256), - type_=sa.String(length=64), - existing_nullable=False, + + bind = op.get_bind() + is_revoked_exists = bind.execute( + sa.text( + """ + SELECT EXISTS ( + SELECT FROM information_schema.columns + WHERE table_schema = current_schema() + AND table_name = 'refresh_tokens' + AND column_name = 'is_revoked' + ) + """ + ) + ).scalar() + + if is_revoked_exists: + op.execute( + """ + UPDATE refresh_tokens + SET revoked_at = NOW() + WHERE is_revoked = TRUE + AND revoked_at IS NULL + """ + ) + + op.execute( + "ALTER TABLE refresh_tokens ALTER COLUMN token_hash TYPE VARCHAR(64)" ) - op.drop_column("refresh_tokens", "is_revoked") + op.execute("ALTER TABLE refresh_tokens DROP COLUMN IF EXISTS is_revoked") def downgrade() -> None: - op.add_column( - "refresh_tokens", - sa.Column("is_revoked", sa.Boolean(), nullable=False, server_default="false"), + op.execute( + "ALTER TABLE refresh_tokens ADD COLUMN IF NOT EXISTS " + "is_revoked BOOLEAN NOT NULL DEFAULT false" ) op.execute( """ @@ -54,13 +68,9 @@ def downgrade() -> None: WHERE revoked_at IS NOT NULL """ ) - op.alter_column("refresh_tokens", "is_revoked", server_default=None) - op.alter_column( - "refresh_tokens", - "token_hash", - existing_type=sa.String(length=64), - type_=sa.String(length=256), - existing_nullable=False, + op.execute("ALTER TABLE refresh_tokens ALTER COLUMN is_revoked DROP DEFAULT") + op.execute( + "ALTER TABLE refresh_tokens ALTER COLUMN token_hash TYPE VARCHAR(256)" ) - op.drop_column("refresh_tokens", "absolute_expiry") - op.drop_column("refresh_tokens", "revoked_at") + op.execute("ALTER TABLE refresh_tokens DROP COLUMN IF EXISTS absolute_expiry") + op.execute("ALTER TABLE refresh_tokens DROP COLUMN IF EXISTS revoked_at") From b4e8aa0a1cd0268638c7010a2e4438d21f92ea89 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Fri, 26 Jun 2026 12:00:50 -0700 Subject: [PATCH 15/19] feat(docker): add proxy directory and Caddyfile to installation script --- docker/scripts/install.ps1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/scripts/install.ps1 b/docker/scripts/install.ps1 index 23b14c3c4..761fef80e 100644 --- a/docker/scripts/install.ps1 +++ b/docker/scripts/install.ps1 @@ -330,11 +330,13 @@ Write-Info "Installation directory: $InstallDir" New-Item -ItemType Directory -Path "$InstallDir\scripts" -Force | Out-Null New-Item -ItemType Directory -Path "$InstallDir\searxng" -Force | Out-Null +New-Item -ItemType Directory -Path "$InstallDir\proxy" -Force | Out-Null $Files = @( @{ Src = "docker/docker-compose.yml"; Dest = "docker-compose.yml" } @{ Src = "docker/docker-compose.gpu.yml"; Dest = "docker-compose.gpu.yml" } @{ Src = "docker/.env.example"; Dest = ".env.example" } + @{ Src = "docker/proxy/Caddyfile"; Dest = "proxy/Caddyfile" } @{ Src = "docker/postgresql.conf"; Dest = "postgresql.conf" } @{ Src = "docker/scripts/migrate-database.ps1"; Dest = "scripts/migrate-database.ps1" } @{ Src = "docker/searxng/settings.yml"; Dest = "searxng/settings.yml" } From 2e4959f947a14cdb9e78400b0a65650d08cb23b9 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Fri, 26 Jun 2026 12:24:12 -0700 Subject: [PATCH 16/19] fix(docker): add reverse proxy for zero sync auth context in Caddyfile --- docker/proxy/Caddyfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker/proxy/Caddyfile b/docker/proxy/Caddyfile index 534a8c2c2..abc5ff199 100644 --- a/docker/proxy/Caddyfile +++ b/docker/proxy/Caddyfile @@ -31,6 +31,10 @@ flush_interval -1 } + # Zero sync auth context is a backend (FastAPI) endpoint. More specific than + # /zero/*, so Caddy's matcher-specificity sort routes it here, not to zero-cache. + reverse_proxy /zero/context backend:8000 + # Zero accepts a single path-component base URL (Zero >= 0.6). # Preserve /zero so browser cacheURL can be ${SURFSENSE_PUBLIC_URL}/zero. reverse_proxy /zero/* zero-cache:4848 From 24ff4b7d937ed6ccf32c17ee654b1106cb90908c Mon Sep 17 00:00:00 2001 From: yagitoshiro Date: Sun, 28 Jun 2026 13:13:46 +0900 Subject: [PATCH 17/19] fix(web): don't submit chat on Enter during IME composition The chat composers submit on Enter without checking whether an IME composition is active. For Japanese/Chinese/Korean input, pressing Enter to confirm a conversion fires the submit handler and sends a half-typed message. Guard the Enter handlers with e.nativeEvent.isComposing in the main chat editor, the anonymous free composer, and the mention/prompt picker key navigation. --- .../components/assistant-ui/inline-mention-editor.tsx | 4 +++- surfsense_web/components/assistant-ui/thread.tsx | 4 ++++ surfsense_web/components/free-chat/free-composer.tsx | 4 +++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/surfsense_web/components/assistant-ui/inline-mention-editor.tsx b/surfsense_web/components/assistant-ui/inline-mention-editor.tsx index 5fc942e54..ee2d1c873 100644 --- a/surfsense_web/components/assistant-ui/inline-mention-editor.tsx +++ b/surfsense_web/components/assistant-ui/inline-mention-editor.tsx @@ -696,7 +696,9 @@ export const InlineMentionEditor = forwardRef { // Arrow / Enter / Escape navigation for the active picker. const handleKeyDown = useCallback( (e: React.KeyboardEvent) => { + // While an IME composition is active (e.g. confirming a Japanese/Chinese/ + // Korean conversion), let the Enter/Arrow keys reach the IME instead of + // driving picker navigation/selection. + if (e.nativeEvent.isComposing) return; if (showPromptPicker) { if (e.key === "ArrowDown") { e.preventDefault(); diff --git a/surfsense_web/components/free-chat/free-composer.tsx b/surfsense_web/components/free-chat/free-composer.tsx index 162b906ad..d4523a4f9 100644 --- a/surfsense_web/components/free-chat/free-composer.tsx +++ b/surfsense_web/components/free-chat/free-composer.tsx @@ -99,7 +99,9 @@ export const FreeComposer: FC = () => { gate("mention documents"); return; } - if (e.key === "Enter" && !e.shiftKey) { + // Ignore Enter while an IME composition is active (e.g. confirming a + // Japanese/Chinese/Korean conversion) so it doesn't submit the message. + if (e.key === "Enter" && !e.shiftKey && !e.nativeEvent.isComposing) { e.preventDefault(); if (text.trim()) { aui.composer().send(); From 9b2f880e3cace3e58008f099495427954c835fc7 Mon Sep 17 00:00:00 2001 From: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> Date: Mon, 29 Jun 2026 00:49:30 -0700 Subject: [PATCH 18/19] refactor: delete unused notesApiService orphan API module --- surfsense_web/lib/apis/notes-api.service.ts | 147 -------------------- 1 file changed, 147 deletions(-) delete mode 100644 surfsense_web/lib/apis/notes-api.service.ts diff --git a/surfsense_web/lib/apis/notes-api.service.ts b/surfsense_web/lib/apis/notes-api.service.ts deleted file mode 100644 index eac3d96ed..000000000 --- a/surfsense_web/lib/apis/notes-api.service.ts +++ /dev/null @@ -1,147 +0,0 @@ -import { z } from "zod"; -import { ValidationError } from "../error"; -import { baseApiService } from "./base-api.service"; - -// Request/Response schemas -const createNoteRequest = z.object({ - search_space_id: z.number(), - title: z.string().min(1), - source_markdown: z.string().optional(), -}); - -const createNoteResponse = z.object({ - id: z.number(), - title: z.string(), - document_type: z.string(), - content: z.string(), - content_hash: z.string(), - unique_identifier_hash: z.string().nullable(), - document_metadata: z.record(z.string(), z.any()).nullable(), - search_space_id: z.number(), - created_at: z.string(), - updated_at: z.string().nullable(), -}); - -const getNotesRequest = z.object({ - search_space_id: z.number(), - skip: z.number().optional(), - page: z.number().optional(), - page_size: z.number().optional(), -}); - -const noteItem = z.object({ - id: z.number(), - title: z.string(), - document_type: z.string(), - content: z.string(), - content_hash: z.string(), - unique_identifier_hash: z.string().nullable(), - document_metadata: z.record(z.string(), z.any()).nullable(), - search_space_id: z.number(), - created_at: z.string(), - updated_at: z.string().nullable(), -}); - -const getNotesResponse = z.object({ - items: z.array(noteItem), - total: z.number(), - page: z.number(), - page_size: z.number(), - has_more: z.boolean(), -}); - -const deleteNoteRequest = z.object({ - search_space_id: z.number(), - note_id: z.number(), -}); - -const deleteNoteResponse = z.object({ - message: z.string(), - note_id: z.number(), -}); - -// Type exports -export type CreateNoteRequest = z.infer; -export type CreateNoteResponse = z.infer; -export type GetNotesRequest = z.infer; -export type GetNotesResponse = z.infer; -export type NoteItem = z.infer; -export type DeleteNoteRequest = z.infer; -export type DeleteNoteResponse = z.infer; - -class NotesApiService { - /** - * Create a new note - */ - createNote = async (request: CreateNoteRequest) => { - const parsedRequest = createNoteRequest.safeParse(request); - - if (!parsedRequest.success) { - console.error("Invalid request:", parsedRequest.error); - const errorMessage = parsedRequest.error.issues.map((issue) => issue.message).join(", "); - throw new ValidationError(`Invalid request: ${errorMessage}`); - } - - const { search_space_id, title, source_markdown } = parsedRequest.data; - - // Send both title and source_markdown in request body - const body = { - title, - ...(source_markdown !== undefined && { source_markdown }), - }; - - return baseApiService.post( - `/api/v1/search-spaces/${search_space_id}/notes`, - createNoteResponse, - { body } - ); - }; - - /** - * Get list of notes - */ - getNotes = async (request: GetNotesRequest) => { - const parsedRequest = getNotesRequest.safeParse(request); - - if (!parsedRequest.success) { - console.error("Invalid request:", parsedRequest.error); - const errorMessage = parsedRequest.error.issues.map((issue) => issue.message).join(", "); - throw new ValidationError(`Invalid request: ${errorMessage}`); - } - - const { search_space_id, skip, page, page_size } = parsedRequest.data; - - // Build query params - const params = new URLSearchParams(); - if (skip !== undefined) params.append("skip", String(skip)); - if (page !== undefined) params.append("page", String(page)); - if (page_size !== undefined) params.append("page_size", String(page_size)); - - return baseApiService.get( - `/api/v1/search-spaces/${search_space_id}/notes?${params.toString()}`, - getNotesResponse - ); - }; - - /** - * Delete a note - */ - deleteNote = async (request: DeleteNoteRequest) => { - const parsedRequest = deleteNoteRequest.safeParse(request); - - if (!parsedRequest.success) { - console.error("Invalid request:", parsedRequest.error); - const errorMessage = parsedRequest.error.issues.map((issue) => issue.message).join(", "); - throw new ValidationError(`Invalid request: ${errorMessage}`); - } - - const { search_space_id, note_id } = parsedRequest.data; - - return baseApiService.delete( - `/api/v1/search-spaces/${search_space_id}/notes/${note_id}`, - deleteNoteResponse - ); - }; -} - -export const notesApiService = new NotesApiService(); From d12f39a7e2dba39d1eea60850af9a5d1d01593d3 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Mon, 29 Jun 2026 17:50:24 -0700 Subject: [PATCH 19/19] feat: updated plans with test statergy --- plans/backend/00-umbrella-plan.md | 31 +++-- plans/backend/03a-crawler-core.md | 34 +++-- plans/backend/03b-proxy-expansion.md | 6 +- plans/backend/03c-crawl-billing.md | 9 +- plans/backend/03d-captcha-solving.md | 106 +++++++++++----- plans/backend/03e-stealth-hardening.md | 119 ++++++++++++++++++ plans/backend/03f-undetectability-testing.md | 124 +++++++++++++++++++ 7 files changed, 379 insertions(+), 50 deletions(-) create mode 100644 plans/backend/03e-stealth-hardening.md create mode 100644 plans/backend/03f-undetectability-testing.md diff --git a/plans/backend/00-umbrella-plan.md b/plans/backend/00-umbrella-plan.md index fce43f137..060865f41 100644 --- a/plans/backend/00-umbrella-plan.md +++ b/plans/backend/00-umbrella-plan.md @@ -40,7 +40,7 @@ flowchart TD - MCP-availability audit complete: BookStack (community MCP servers), Elasticsearch (official Elastic Agent Builder MCP), and Luma (community MCP servers) all have MCP available, so none are `DISABLED` — they're tagged `MIGRATING` (turned off for MVP like the other branded natives, pending the post-MVP MCP re-point). - `Pipeline` and `PipelineRun` are new first-class tables. A Pipeline references a connector + config + schedule + KB destination. File upload creates/uses a pipeline and registers a run; uploads always save to KB. - The chat agent gets read-only access to pipeline run history (pipelines + their recent runs/status) as context, so it can reason about what was fetched, when, and whether runs succeeded — even for data not saved to the KB. -- Deferred (post-MVP): platform scraper implementations, public pay-as-you-go API for Type-1 connectors, public MCP server exposing the KB. +- Deferred (post-MVP): platform scraper implementations (**Phase 8 — Platform actors**, public-data-only first; built in-house on the Phase-3 fetch core), public pay-as-you-go API for Type-1 connectors, public MCP server exposing the KB. Logged-in/account-based scraping deferred beyond that. ## Platform connector research list (deferred build, MVP = "coming soon") @@ -69,14 +69,18 @@ flowchart TD - Consolidate API to `/workspaces` and fix the `/searchspaces` vs `/search-spaces` inconsistency. - High-touch files: `routes/search_spaces_routes.py`, `routes/rbac_routes.py`, `utils/rbac.py` (`check_search_space_access`), `schemas/search_space.py`, plus `search_space_id` threading through agents/Redis keys/storage paths (`documents/{id}/...`). -### Phase 3 — WebURL Crawler & Crawl Billing (backend) [`subplans: 03a–03d`] +### Phase 3 — WebURL Crawler & Crawl Billing (backend) [`subplans: 03a–03f`] -The Universal WebURL Crawler is the flagship Type-1 data source (the moat). This phase hardens it on a single framework (Scrapling), generalizes proxy support, introduces pay-as-you-go crawl credits, and (deferred) adds opt-in captcha solving. It is broken into focused subplans: +The Universal WebURL Crawler is the flagship Type-1 data source (**the moat**). This phase hardens it into an **in-house, best-effort "undetectable, captcha-bypassing" crawler** on a single framework (Scrapling): it standardizes the fetch layer, generalizes proxy support, introduces pay-as-you-go crawl credits, adds **stealth hardening** (geoip coherence, persistent profiles, headed/Xvfb, fonts, humanization, a block classifier + per-domain strategy memory), and adds **opt-in captcha solving**. All tiers plug in behind a single `FetchStrategy` seam returning `CrawlOutcome`, so callers never depend on *how* a page was fetched — that seam is what lets the moat grow (and lets a **deferred paid-unblocker tier** drop in later by config). **Strategy decision (recorded in the log):** CloakBrowser is rejected on licensing and external unblocker APIs are deferred; we hold an in-house bypass moat for ~4–6 months, then move hostile targets to a paid tier if demand/maintenance justifies it. **Logged-in/account-based bypass is out of scope** (public data only this MVP). It is broken into focused subplans: + +> **Sequencing within Phase 3 (critical path vs hardening).** Only **`03a` + `03b` + `03c`** are on the **MVP critical path** — Phases 4–7 (connector taxonomy, pipelines, execution) consume the `03a` `CrawlOutcome`/`FetchStrategy` contract, the `03b` proxy provider, and the `03c` `WebCrawlCreditService` billing seam. **`03d` (captcha), `03e` (stealth hardening), and `03f` (test harness) are hardening/measurement that nothing downstream imports** — they tune the *same* seam behind the *same* contract, so they can land **in parallel with, or after, Phases 4–7** without blocking the pivot. Recommended build order: `03a → 03b → 03c` (then proceed to Phase 4), with `03e → 03d → 03f` slotted in whenever crawler robustness is prioritized. The dependency notes inside `03d`/`03e`/`03f` (each requires `03a`/`03b`) still hold; this only frees them from gating Phase 4+. - **`03a-crawler-core.md`** — Standardize the fetch layer on Scrapling. **Remove Firecrawl entirely** (no other frameworks). Define crisp per-URL success/empty/failure semantics, keep Trafilatura extraction, and expose a single billable "successful crawl" signal (one unit per URL that yields usable content, regardless of how many internal fallback tiers ran). - **`03b-proxy-expansion.md`** — Add a BYO `CustomProxyProvider` (the only new provider — **no branded vendors**) alongside `anonymous_proxies`, selectable via a **single, app-wide** `Config.PROXY_PROVIDER`. Add bounded client-side rotation+retry via Scrapling's `ProxyRotator`/`is_proxy_error` **only** when the active provider is pool-backed (`CUSTOM_PROXY_URLS`); single-endpoint providers (incl. `anonymous_proxies`) stay the default and no-op the retry. **No per-connector/per-crawl selection** (one provider app-wide); a per-pipeline override is left as a no-op seam for Phase 5/6. - **`03c-crawl-billing.md`** — Charge crawl credits at **$1 / 1000 successful requests = 1000 micro-USD per successful crawl**, drawn from the existing credit wallet (`credit_micros_balance`), gated by a new `WEB_CRAWL_CREDIT_BILLING_ENABLED` flag (off for self-hosted). Two surfaces: **connector/pipeline crawls** billed to the **workspace owner** via a dedicated `WebCrawlCreditService` (mirrors `EtlCreditService`'s gate → `check_credits` → `charge_credits`, **not** `billable_call`); **chat scrapes** fold their crawl cost into the chat turn's existing bill (turn accumulator). No DB migration (uses the existing free-form `web_crawl` usage_type). -- **`03d-captcha-solving.md`** *(DEFERRED — sequenced last, non-MVP-blocking)* — Covers the captcha types Scrapling does **not** (reCAPTCHA v2/v3, hCaptcha, image) via `captchatools`. `captchatools` is **itself** the provider registry (`new_harvester(solving_site=…)` across capmonster/2captcha/anticaptcha/capsolver/captchaai), so we do **not** rebuild a provider hierarchy — our layer is thin: config resolution + a StealthyFetcher `page_action` that detects the sitekey, harvests a token, and injects it. Scrapling already handles Cloudflare Turnstile (`03a`). Flags the **billing asymmetry** (solvers charge per *attempt*, `03c` bills per *success*) for resolution at build time. Requires a paid solver account. +- **`03e-stealth-hardening.md`** — The in-house "undetectable" layer on top of Scrapling's default patchright-Chromium stealth. **Geoip coherence** (match browser `locale`/`timezone_id` to the proxy's exit geo), **fingerprint flags** (`hide_canvas`/`block_webrtc`), **persistent per-domain profiles** (`user_data_dir`), **headed execution under Xvfb**, **real fonts** in the worker image, and **DIY behavioral humanization** via `page_action` (the Chromium engine has no built-in `humanize`). Adds a **block classifier** (label Cloudflare/DataDome/Kasada/captcha/empty from the response) + **per-domain strategy memory** (Redis, no migration) so the ladder learns the known-good tier per domain. Defines — but does **not** build — the **deferred paid-unblocker `FetchStrategy`** (ZenRows/ScrapFly/Bright Data) as the config-flagged escape hatch, with its own (later) billing. Honest ceiling: defeats Cloudflare + the moderate long tail, **not** top-tier behavioral fingerprinting (DataDome/Kasada/reCAPTCHA-Enterprise) — that's the deferred tier. +- **`03d-captcha-solving.md`** *(ACTIVE — sequenced last in Phase 3, after `03e`)* — Covers the captcha types Scrapling does **not** (reCAPTCHA v2/v3, hCaptcha, image) via `captchatools`, **opt-in + off by default**. `captchatools` is **itself** the provider registry (`new_harvester(solving_site=…)` across capmonster/2captcha/anticaptcha/capsolver/captchaai), so we do **not** rebuild a provider hierarchy — our layer is thin: config resolution + a StealthyFetcher `page_action` that detects the sitekey, harvests a token (egressing from the **same** proxy IP as the crawl), and injects it. Scrapling already handles Cloudflare Turnstile (`03a`), and `page_action` runs **after** `solve_cloudflare`, so the tiers compose. The **billing asymmetry** is now **resolved**: a **separate per-attempt** `web_crawl_captcha` unit (solvers charge per attempt regardless of crawl success), attached via a `WebCrawlCreditService` seam in `03c`. `ErrNoBalance` stops solving (no retry loop → avoids IP bans). Requires a paid solver account. +- **`03f-undetectability-testing.md`** *(MANUAL-only — no CI gating; sequenced last)* — A **manual scorecard harness** that drives the real Scrapling tiers against the industry-standard detection + sandbox sites (modeled on CloakBrowser's `bin/cloaktest` suite) to **quantify the free-stack ceiling** over time. Two labeled axes: **Suite S (stealth/anti-bot)** — browser-tier (`bot.sannysoft`, `bot.incolumitas`, CreepJS, `deviceandbrowserinfo`, FingerprintJS demo, reCAPTCHA-v3 score, `fingerprint-scan`/Castle.js, `browserscan`), HTTP/TLS-tier JA3/JA4 parity (`tls.peet.ws`, **informational** not a gate), and proxy/leak checks (`httpbin/ip`, WebRTC/DNS); **Suite E (extraction correctness)** — toscrape/scrapethissite sandboxes for the HTTP vs JS (DynamicFetcher) tiers. Reuses `03d`'s `page_action`+closure-cell for JS-object verdicts. Adopts CloakBrowser's bars as **aspirational** (sannysoft 0 fails, CreepJS ≤30%, reCAPTCHA ≥0.7) while recording **our actual numbers as the committed baseline**; the scorecard is the documented **trigger** for flipping `03e`'s deferred paid-unblocker tier. ### Phase 4 — Connector two-type restructure (backend) [`subplans: 04a–04b`] @@ -114,6 +118,13 @@ Split into two independently testable subplans: - A small migration adds a partial unique index `ON pipelines(workspace_id) WHERE connector_id IS NULL` (enforces the singleton + race-safe get-or-create). - KB-save-secondary: the opt-in `save_to_kb` + destination folder for connector pipelines already shipped in Phases 5–6; Phase 7 only records the inverse invariant (uploads always KB) and guards the connector default stays `False`. +### Phase 8 — Platform actors (FUTURE — post-MVP, public data only) [`subplan: TBD`] + +NOT planned in this umbrella; recorded so the Phase-3 architecture stays aimed at it. Once the hardened fetch core (Phases 3a–3e) is solid, **platform actors** layer **on top** of it: per-platform structured extractors (Google Maps/Local, LinkedIn public profiles/companies, Amazon products, etc. — see "Platform connector research list" above), built in-house "Apify-style" rather than via third-party paid actors. They reuse the existing machinery end-to-end: each actor is a **Type-1 data-source connector** (Phase 4 taxonomy), runs via **Pipelines** (Phases 5–6, manual/cron), saves to the **KB** (opt-in folder), and is **billed per run** (extending `03c`/`06`). They consume the `03a` `FetchStrategy` core (proxies + `03e` hardening + `03d` captcha) under their own extractors. + +- **Public data only** at first — discovery/extraction of publicly visible pages. **Logged-in/account-based bypass is explicitly deferred** beyond Phase 8's first cut; it needs sticky/static proxies + credential management (`03b` static-proxy hand-off) and is the higher-risk, later workstream. +- The **deferred paid-unblocker tier** (`03e §8`) is the fallback for any platform whose anti-bot exceeds the in-house ceiling. + ## Deferred — Frontend & client phases (separate umbrella, planned LATER) These are recorded for continuity but are NOT planned in this umbrella. They start once the backend phases above are working. @@ -140,10 +151,14 @@ These are recorded for continuity but are NOT planned in this umbrella. They sta - Phase 4 search APIs: all 5 enum values dropped (`SERPER_API`/`TAVILY_API`/`SEARXNG_API`/`LINKUP_API`/`BAIDU_SEARCH_API`) in 04b. Survivors (SearXNG/Linkup/Baidu) become PLATFORM providers keyed by env (Linkup/Baidu keys move from per-connector `config` to env — app-wide, not per-workspace). 04b carries a destructive migration deleting the 5 connector types' rows. - Phase 4 structure: split into 04a (taxonomy/gating/MCP-fix, no migration) and 04b (search repurposing + source-discovery endpoint, with migration); intended order 04a -> 04b (both orders safe). - Rename transition policy: HARD CUTOVER of the external API (paths + JSON field names) in Phase 2 — no backward-compat aliases. Rationale: the frontend is (re)built against the corrected backend later, so there is no old client to keep alive; backend correctness is verified via the test suite + OpenAPI rather than the existing UI. -- WebURL Crawler framework: STANDARDIZE on Scrapling; **remove Firecrawl entirely** (no other scraping frameworks now or planned). Scrapling's `StealthyFetcher` handles Cloudflare; captcha-tools (deferred) covers the rest. +- WebURL Crawler framework: STANDARDIZE on Scrapling; **remove Firecrawl entirely** (no other scraping frameworks now or planned). Scrapling's `StealthyFetcher` (patchright-Chromium as of 0.4.9 — **not** Camoufox) handles Cloudflare; `03e` stealth-hardening minimizes challenges; captcha-tools (`03d`) covers the rest. All fetch tiers sit behind a `FetchStrategy` seam returning `CrawlOutcome` (callers never depend on the tier). - Crawl billing: reuse the existing credit wallet (`credit_micros_balance`) with a new `web_crawl` usage_type. Price: **$1 / 1000 successful requests** (1000 micro-USD per success). Connector/pipeline crawls bill the **workspace owner**; chat scrapes fold their crawl cost into the already-billed chat turn. Gated by `WEB_CRAWL_CREDIT_BILLING_ENABLED` (off for self-hosted); no DB migration required. - Billable unit: one unit per URL that returns usable extracted content, regardless of how many internal fallback tiers were attempted (not per HTTP fetch, not per URL-processed). -- Captcha solving (captcha-tools): DEFERRED to the last Phase-3 subplan (`03d`); non-MVP-blocking. +- Captcha solving (captcha-tools): **ACTIVE** (no longer deferred) — sequenced last in Phase 3 (`03d`), **after** `03e` hardening, **opt-in + off by default**. Cloudflare stays in-framework (`03a`); reCAPTCHA/hCaptcha/image use `captchatools`. **Billing asymmetry RESOLVED → option (a): a separate per-attempt `web_crawl_captcha` unit** (`WEB_CRAWL_CAPTCHA_*` knobs on `WebCrawlCreditService`), since solvers charge per attempt regardless of crawl success. `ErrNoBalance` halts solving (no retry-loop IP bans). +- Crawler stealth strategy (the moat): **CloakBrowser REJECTED** (source-patched Chromium binary requires an OEM/SaaS license incompatible with our model). **External unblocker APIs DEFERRED** (ZenRows/ScrapFly/Bright Data) — pre-wired as a config-flagged `FetchStrategy` (`03e §8`) but not built. Plan: maintain an **in-house bypass moat for ~4–6 months** (Scrapling stealth + residential proxies + `03e` hardening + `03d` captcha), then move hostile/top-tier-fingerprinted targets (DataDome/Kasada/reCAPTCHA-Enterprise) to a paid tier if demand/maintenance justifies it. Realistic ceiling acknowledged: in-house beats Cloudflare + the moderate long tail, not top-tier behavioral fingerprinting. +- Authenticated/logged-in scraping: **OUT OF SCOPE this MVP** (public data only). Sticky/static proxies + credential management are deferred and paired with the future platform actors (`03b` static-proxy hand-off + Phase 8). +- Phase 3 stealth-hardening subplan `03e` ADDED: geoip locale/tz coherence, `hide_canvas`/`block_webrtc`, persistent per-domain profiles, headed+Xvfb, fonts, DIY humanization (`page_action`; Chromium engine has no built-in `humanize`), a block classifier, and per-domain strategy memory (Redis, no migration). +- Phase 3 test-harness subplan `03f` ADDED: **manual-only** (no CI/automated gating now) undetectability + extraction scorecard, modeled on CloakBrowser's `bin/cloaktest`. Two labeled axes (Suite S stealth + Suite E extraction) so they scale independently. Drives the **real** Scrapling tiers (browser + curl_cffi HTTP/TLS), reuses `03d`'s `page_action`+closure-cell for JS-object verdicts. **TLS JA3/JA4 parity = informational axis, not a hard gate.** Adopt CloakBrowser bars as aspirational; record our actual free-stack numbers as the committed baseline. The scorecard is the documented evidence/trigger for flipping `03e`'s deferred paid-unblocker tier. - Roadmap: WebURL Crawler & Crawl Billing inserted as the new Phase 3; connector two-type → Phase 4; pipelines → Phases 5/6/7. - Phase 5 pipelines data model: two new tables `pipelines` (mutable) + `pipeline_runs` (append-only), modeled on `automations`/`automation_runs`; ORM lives in `db.py` next to connectors/folders. `connector_id` nullable (NULL = Phase-7 Uploads), eligibility enforced at create via 04a's `is_pipeline_eligible`. Schedule = `schedule_cron` + `schedule_timezone` (default UTC) + `next_scheduled_at` (cron, matching automations). `pipeline_runs` pre-includes `charged_micros`/`crawls_*`/`result_blob_key` so Phase 6 needs no extra migration. Both tables published to Zero **full-row** (like folders/connectors). Routes reuse `CONNECTORS_*` permissions. Phase 5 ships the data model + API surface only; the `/run` endpoint enqueues a Phase-6 task stub. - Phase 7 uploads-as-pipeline: a **singleton "Uploads" pipeline** per workspace (`connector_id NULL`, `save_to_kb=true`), lazily get-or-created (race-safe via a partial unique index `ON pipelines(workspace_id) WHERE connector_id IS NULL`). Each `fileupload`/`folder-upload` request writes a **terminal audit `PipelineRun(trigger=upload, status=succeeded, documents_indexed=)`** — uploads are **route-recorded, not engine-executed** (Phase 6 fails NULL-connector runs by design; existing upload code stays the executor). Best-effort via an **inner** try/except (never 5xx the upload — the route's outer handler would otherwise 500 an already-committed upload). No crawl billing (uploads aren't crawls; `charged_micros` NULL). Per-file ETL truth stays on `Document.status`; accurate roll-up needs the deferred `documents.pipeline_run_id` provenance. Connector `save_to_kb` default stays `False` (opt-in for connectors, mandatory for uploads). Phase 7 also **guards Phase-5's generic CRUD** against the system Uploads pipeline: `POST /pipelines` rejects `connector_id=None` (supersedes Phase 5's permissive create), `/run` and schedule-`PUT` reject NULL-connector pipelines, and Phase 6's scheduler `_claim_due` filters `connector_id IS NOT NULL` as a backstop (so the Uploads pipeline can never be manually-run or scheduled into perpetually-failing runs). See `07-upload-pipeline-kb.md`. @@ -158,7 +173,9 @@ These are recorded for continuity but are NOT planned in this umbrella. They sta | 3 | `03a-crawler-core.md` | drafted | | 3 | `03b-proxy-expansion.md` | drafted | | 3 | `03c-crawl-billing.md` | drafted | -| 3 | `03d-captcha-solving.md` | drafted (deferred — last) | +| 3 | `03e-stealth-hardening.md` | drafted | +| 3 | `03d-captcha-solving.md` | drafted (active — sequenced after 03e) | +| 3 | `03f-undetectability-testing.md` | drafted (manual scorecard — sequenced last) | | 4 | `04a-connector-category.md` | drafted | | 4 | `04b-source-discovery.md` | drafted | | 5 | `05-pipelines-model.md` | drafted | diff --git a/plans/backend/03a-crawler-core.md b/plans/backend/03a-crawler-core.md index afa96e368..3c43c44e8 100644 --- a/plans/backend/03a-crawler-core.md +++ b/plans/backend/03a-crawler-core.md @@ -1,7 +1,7 @@ # Phase 3a — WebURL Crawler core (Scrapling-only) + success semantics > Part of **Phase 3 — WebURL Crawler & Crawl Billing**. See `00-umbrella-plan.md`. -> Sibling subplans: `03b-proxy-expansion.md`, `03c-crawl-billing.md`, `03d-captcha-solving.md` (deferred). +> Sibling subplans: `03b-proxy-expansion.md`, `03c-crawl-billing.md`, `03e-stealth-hardening.md`, `03d-captcha-solving.md`, `03f-undetectability-testing.md` (manual scorecard). > **Implementation note (applies to all Phase-3 plans).** Phase 3 lands **after** Phases 1–2, which rename `SearchSpace`→`Workspace` and `search_space_id`→`workspace_id` everywhere. Citations below use **today's** names so they stay greppable against current code; when implementing, the live code will already say `workspace_*` — map accordingly. Apply every edit by **symbol/grep** (e.g. `firecrawl_api_key`, `crawl_url`, `FIRECRAWL_API_KEY`), **not** by the absolute line numbers cited here — Phase 2's rename (and `03a`'s own Firecrawl removal) shift line numbers. @@ -11,7 +11,7 @@ Make the Universal WebURL Crawler a **single-framework (Scrapling) component** w Two hard requirements from the decisions log: -1. **Remove Firecrawl entirely.** No other scraping framework now or planned. Scrapling's `StealthyFetcher` can bypass Cloudflare Turnstile when invoked with `solve_cloudflare=True` (see tier design below); captcha-tools (`03d`, deferred) covers the rest. +1. **Remove Firecrawl entirely.** No other scraping framework now or planned. Scrapling's `StealthyFetcher` can bypass Cloudflare Turnstile when invoked with `solve_cloudflare=True` (see tier design below); captcha-tools (`03d`) covers reCAPTCHA/hCaptcha, and `03e` adds the broader stealth-hardening that makes the crawler "undetectable" as far as free tooling reaches. 2. **One billable unit = one URL that yields usable extracted content**, regardless of how many internal fallback tiers ran. `03a` must expose that signal; `03c` meters it. This subplan does NOT touch proxy rotation (→ `03b`), credit metering (→ `03c`), or captcha (→ `03d`). @@ -25,7 +25,11 @@ This subplan does NOT touch proxy rotation (→ `03b`), credit metering (→ `03 1. **Firecrawl** (premium, if `firecrawl_api_key` set) — `_crawl_with_firecrawl()` (lines 89–100, 223–272), imports `from firecrawl import AsyncFirecrawlApp` (line 22). 2. Scrapling `AsyncFetcher` (static HTTP, curl_cffi) — `_crawl_with_async_fetcher()` (lines 274–310). 3. Scrapling `DynamicFetcher` (browser, run in a thread) — `_crawl_with_dynamic()` (lines 312–339). -4. Scrapling `StealthyFetcher` (Camoufox anti-bot, run in a thread) — `_crawl_with_stealthy()` (lines 341–369). +4. Scrapling `StealthyFetcher` (patchright-Chromium anti-bot, run in a thread) — `_crawl_with_stealthy()` (lines 341–369). + +> **Engine note (do not say "Camoufox").** As of the pinned `scrapling[fetchers]>=0.4.9` (`pyproject.toml:91`), `StealthyFetcher` is **"completely stealthy built on top of Chromium"** (`references/Scrapling/scrapling/fetchers/stealth_chrome.py:8`) driven by **patchright** (`references/Scrapling/scrapling/engines/_browsers/_stealth.py:8–9`), **not** Camoufox — Scrapling removed Camoufox (zero matches in the 0.4.9 tree; `uv.lock` carries `patchright`, no `camoufox`). The default Playwright `channel` is `"chromium"` (patchright), or `"chrome"` when `real_chrome=True` (`_browsers/_base.py:469`). Stealth = the compiled-in flag set `DEFAULT_ARGS + STEALTH_ARGS` (`engines/constants.py:24–99`, incl. `--disable-blink-features=AutomationControlled` `:94`) + a persistent context by default (`_stealth.py:90–93`) — this is **runtime/config-level** stealth, which sets the realistic ceiling (see `03e`). +> +> **Scrub the in-code "Camoufox" mentions too.** The connector still carries stale Camoufox wording in its own docstrings — the module header tier list (`webcrawler_connector.py:10–12`) and `_crawl_with_stealthy`'s docstring (`:343` "StealthyFetcher (Camoufox)"). Fix these in the 03a refactor so the code matches reality (patchright-Chromium). Extraction is Trafilatura HTML→markdown in `_build_result()` (lines 371–469). Every Scrapling tier passes `proxy=get_proxy_url()` (lines 287, 329, 359). `crawl_url()` returns a tuple `(result_dict | None, error | None)`; `result_dict` has `content` / `metadata` / `crawler_type`. @@ -57,7 +61,7 @@ Firecrawl's API key is plumbed end-to-end and must be removed everywhere: ### Runtime/deps already in place -- `Dockerfile:112–115` runs `RUN scrapling install` to fetch patchright Chromium + Camoufox; the `scrapling[fetchers]` extra pulls playwright/patchright. **No new install step needed** once Firecrawl is gone. +- `Dockerfile:112–115` runs `RUN scrapling install`; the `scrapling[fetchers]` extra pulls playwright/patchright. **No new install step needed** once Firecrawl is gone. *(Accuracy fix: the Dockerfile comment says "patchright Chromium + Camoufox", but `scrapling install` in 0.4.9 only fetches Chromium — `references/Scrapling/scrapling/cli.py:122,131` runs `playwright install chromium` + `install-deps chromium`, no Camoufox. Drop the stale "+ Camoufox" wording from the comment when touching this file. `03e` may add `install-deps` extras for fonts/Xvfb.)* - Proxy is read via `app/utils/proxy/get_proxy_url()` (`__init__.py:13`), backed by the `PROXY_PROVIDER` registry (`config/__init__.py:983`). `03a` leaves this single-URL model untouched (rotation is `03b`). ## Target design @@ -66,9 +70,9 @@ Firecrawl's API key is plumbed end-to-end and must be removed everywhere: `crawl_url()` becomes a 3-tier ladder, preserving the existing thread-offload + `NotImplementedError` handling for the browser tiers (Windows `SelectorEventLoop` cannot spawn subprocesses — lines 134–141, 161–168): -1. `AsyncFetcher.get(...)` — fast static HTTP. +1. `AsyncFetcher.get(...)` — fast static HTTP. **TLS gap to close:** the current call passes `stealthy_headers=True` but **not** `impersonate` (`webcrawler_connector.py:284–289`), so its TLS ClientHello is curl_cffi's *default* JA3 — trivially bot-flagged and incoherent with the browser tiers' UA. Add an `impersonate="chrome"` profile here (Scrapling's static engine accepts it — `references/Scrapling/scrapling/engines/static.py:36–47`). Tracked as a lever in `03e §2b` and validated by `03f §S3`. 2. `DynamicFetcher.fetch(...)` — full browser (via `asyncio.to_thread`). -3. `StealthyFetcher.fetch(...)` — Camoufox anti-bot, last resort. Enable Cloudflare solving here by passing **`solve_cloudflare=True`** — a documented `StealthyFetcher.fetch` kwarg ("Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response", `references/Scrapling/scrapling/fetchers/stealth_chrome.py:38`; it's a `StealthSession` TypedDict key passed via `**kwargs`). The current stealthy call (connector lines 354–360) passes `headless`/`network_idle`/`block_ads`/`proxy` but **not** `solve_cloudflare`, so this is a real behavior add. (Note: `solve_cloudflare` runs the full browser challenge loop, so it's correctly scoped to the last-resort tier only.) +3. `StealthyFetcher.fetch(...)` — patchright-Chromium anti-bot, last resort. Enable Cloudflare solving here by passing **`solve_cloudflare=True`** — a documented `StealthyFetcher.fetch` kwarg ("Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response", `references/Scrapling/scrapling/fetchers/stealth_chrome.py:38`; it's a `StealthSession` TypedDict key passed via `**kwargs`). The current stealthy call (connector lines 354–360) passes `headless`/`network_idle`/`block_ads`/`proxy` but **not** `solve_cloudflare`, so this is a real behavior add. (Note: `solve_cloudflare` runs the full browser challenge loop, so it's correctly scoped to the last-resort tier only.) Trafilatura extraction (`_build_result`) and `format_to_structured_document()` are unchanged. @@ -83,7 +87,7 @@ class CrawlOutcomeStatus(str, Enum): FAILED = "failed" # invalid URL or every tier errored ``` -`crawl_url()` returns a small dataclass `CrawlOutcome(status, result, error, tier)` — **commit to the dataclass** (not a tuple): `03c` keys billing off `status == SUCCESS`, and Phase 6's fetch-only path (`06-pipelines-exec.md`) consumes `outcome.status` / `outcome.result` / `outcome.error` as attributes, so a tuple form would break that consumer. The **billable success predicate is single-sourced**: `status == CrawlOutcomeStatus.SUCCESS`. +`crawl_url()` returns a small dataclass `CrawlOutcome(status, result, error, tier)` — **commit to the dataclass** (not a tuple): `03c` keys billing off `status == SUCCESS`, and Phase 6's fetch-only path (`06-pipelines-exec.md`) consumes `outcome.status` / `outcome.result` / `outcome.error` as attributes, so a tuple form would break that consumer. The **billable success predicate is single-sourced**: `status == CrawlOutcomeStatus.SUCCESS`. Picking a dataclass (over a tuple) also leaves room for later subplans to **append fields without breaking callers** — `03d` adds `captcha_attempts` / `captcha_solved` for per-attempt billing, and `03e`'s block classifier can attach a `block_type`. (This is distinct from the indexer's positional return, which must stay 2-tuple — see the wrapper note below.) | Outcome | When | Billable (`03c`)? | Document status (indexer) | |---------|------|-------------------|---------------------------| @@ -93,6 +97,17 @@ class CrawlOutcomeStatus(str, Enum): **Billing-policy note for `03c`:** success is the *crawl* succeeding (we fetched + extracted), independent of downstream KB dedupe. The indexer currently marks unchanged content as `skipped` (`webcrawler_indexer.py:341–347`) and cross-connector duplicates as `failed` (`:350–369`) — those still represent a **successful crawl** and should bill. `03c` must count `CrawlOutcomeStatus.SUCCESS`, not `documents_indexed`. Flagging here; final call lives in `03c`. +### Extensibility seam (the tier ladder is a strategy chain) + +The 3-tier ladder is the first instance of a deliberate **`FetchStrategy` seam**: an *ordered list of strategies*, each `(url, ctx) -> CrawlOutcome`, tried in order until one returns `SUCCESS`. `crawl_url()` owns the chain; **every caller depends only on `CrawlOutcome`, never on which strategy produced it** (the indexer, the chat tool, and `03c` metering already do — keep that invariant sacred). This is what lets the moat grow without rework: + +- `03d` (captcha) attaches by escalating to the StealthyFetcher strategy with a `page_action` token-injector — a *parameterization* of the last tier, not a new caller contract. +- `03e` (stealth-hardening) tunes/adds strategies (humanize, headed/Xvfb, persistent profiles, fingerprint flags) **behind the same return type**. +- A future **paid-unblocker tier** (deferred — see `03e`) is just one more strategy appended last, flippable by config; no caller changes. +- Future **platform actors** (Phase 8) reuse the same fetch strategies under their own structured extractors. + +MVP scope here is only the 3 Scrapling tiers + the `CrawlOutcome` contract; the seam is a design constraint (keep tiers pluggable + callers outcome-only), **not** a call for a heavyweight Strategy framework now. + ### Success counter for the indexer Add an explicit `crawls_succeeded` counter in `index_crawled_urls()` incremented whenever `crawl_url` returns `SUCCESS` (right after line 297's call, before the dedupe/unchanged branches), and surface it in the task-success metadata (lines 455–466). `03c` meters against this **in-function** counter (it charges inside the indexer — the count does not need to escape via the return). @@ -113,7 +128,7 @@ Drop the Firecrawl-only `formats=["markdown"]` arg (markdown is already the Traf ## Risks / trade-offs -- **Loss of a managed fallback.** Firecrawl was a hosted last resort for hostile anti-bot sites. Mitigation: `StealthyFetcher` + Cloudflare solving now, `03b` proxy rotation, `03d` captcha solving. Acceptable per the decisions log (single-framework intent). +- **Loss of a managed fallback.** Firecrawl was a hosted last resort for hostile anti-bot sites. Mitigation: the in-house stack — `StealthyFetcher` + Cloudflare solving (this plan), `03b` proxy rotation, `03e` stealth-hardening (humanize/headed/profiles/fonts + block-classifier), `03d` captcha solving — plus a **deferred paid-unblocker tier** behind the seam for the hostile residual. Realistic ceiling: this defeats Cloudflare + the long tail of moderate anti-bot, but runtime-level (patchright) stealth does **not** reliably beat top-tier fingerprinting (DataDome/Kasada/reCAPTCHA-Enterprise) — that's the deferred paid tier's job (`03e`). Acceptable per the decisions log (single-framework intent + in-house moat). - **Browser tiers on dev/Windows.** `DynamicFetcher`/`StealthyFetcher` need subprocess support; the existing `to_thread` + `NotImplementedError` guards (lines 134–141, 161–168) are preserved so static-only crawling still works in `uvicorn --reload`. - **`uv.lock` churn.** Removing `firecrawl-py` requires a lockfile regen + image rebuild; no new runtime deps are added. @@ -121,5 +136,6 @@ Drop the Firecrawl-only `formats=["markdown"]` arg (markdown is already the Traf - Proxy provider expansion + rotation → `03b`. - Crawl credit metering on `CrawlOutcomeStatus.SUCCESS` → `03c`. -- reCAPTCHA/hCaptcha solving via captcha-tools → `03d` (deferred). Cloudflare Turnstile stays in-framework (Scrapling). +- reCAPTCHA/hCaptcha solving via captcha-tools → `03d` (**now active**, sequenced after `03e`). Cloudflare Turnstile stays in-framework (Scrapling). +- Stealth-hardening (humanize, headed/Xvfb, persistent profiles, fonts, geoip locale/tz, block-classifier + per-domain strategy memory) and the deferred paid-unblocker tier → `03e`. - Whether ad-hoc **chat** scrapes are billed (vs only pipeline crawls) → decided in `03c`. diff --git a/plans/backend/03b-proxy-expansion.md b/plans/backend/03b-proxy-expansion.md index abf20b547..35379cebb 100644 --- a/plans/backend/03b-proxy-expansion.md +++ b/plans/backend/03b-proxy-expansion.md @@ -1,7 +1,7 @@ # Phase 3b — Proxy provider expansion + rotation > Part of **Phase 3 — WebURL Crawler & Crawl Billing**. See `00-umbrella-plan.md`. -> Depends on `03a-crawler-core.md` (Scrapling-only crawler). Siblings: `03c-crawl-billing.md`, `03d-captcha-solving.md` (deferred). +> Depends on `03a-crawler-core.md` (Scrapling-only crawler). Siblings: `03c-crawl-billing.md`, `03e-stealth-hardening.md`, `03d-captcha-solving.md`. > **Implementation note.** Same convention as `03a`: citations use **today's** names (`search_space_id`/`SearchSpace`) — map to `workspace_id`/`Workspace` post Phases 1–2. Crucially, the `webcrawler_connector.py` line numbers below (e.g. the three tiers at 287/329/359) and the `scrape_webpage.py` lines **predate `03a`'s Firecrawl removal + `crawl_url` refactor**, so they will have moved by the time `03b` is implemented. Locate code by **symbol/grep**, not absolute lines. @@ -123,4 +123,6 @@ Because there is one global provider and rotation lives **inside** it, the crawl - Branded-vendor provider subclasses → not planned (use `CustomProxyProvider`). - **Static / sticky-session proxies (future).** A later capability will add **static proxy** support — sticky IPs held for the duration of a session — most likely paired with **authenticated/account-based scraping** to bypass logged-in platforms (the deferred platform connectors: LinkedIn, Instagram, etc.). This is a *different axis* from the rotating pool here: rotation maximizes IP diversity, whereas account bypass needs IP **stability** so a session/cookie stays bound to one IP. It is additive to this design — a new `ProxyProvider` (or a "sticky" mode/flag on `CustomProxyProvider`) registered under a new `PROXY_PROVIDER` key, with no change to the zero-arg getter contract — and stays consistent with the single-provider model (the active provider would be the static one when that workflow is selected). Build it alongside the platform connectors, not in Phase 3. - Crawl credit metering (proxy cost is absorbed into the flat `$1 / 1000 successful` price, **not** metered separately) → `03c`. -- Captcha solving → `03d` (deferred). +- Captcha solving → `03d`. +- **Geoip fingerprint coherence** (matching browser `locale`/`timezone_id` to the proxy's exit geo) → `03e`. 03b only owns *selecting* the proxy; `03e` consumes the chosen endpoint's geo. The provider's `RESIDENTIAL_PROXY_LOCATION` (`config/__init__.py`) is one input; resolving the actual exit IP's geo is `03e`'s job. +- **Surfacing the crawl's chosen endpoint into the strategy.** Both `03d` (IP-bound captcha solves) and `03e` (geoip + sticky reuse) need the **exact** endpoint a crawl used, *not* a fresh `get_proxy_url()` call (which rotates on a pool-backed `CustomProxyProvider`). The capture-once seam lives in `03a`'s `FetchStrategy` context; 03b only guarantees the rotating getter and notes the consumers. diff --git a/plans/backend/03c-crawl-billing.md b/plans/backend/03c-crawl-billing.md index 0ea6a01cd..9e30d8233 100644 --- a/plans/backend/03c-crawl-billing.md +++ b/plans/backend/03c-crawl-billing.md @@ -1,7 +1,7 @@ # Phase 3c — Crawl credit billing ($1 / 1000 successful requests) > Part of **Phase 3 — WebURL Crawler & Crawl Billing**. See `00-umbrella-plan.md`. -> Depends on `03a-crawler-core.md` (the `CrawlOutcomeStatus.SUCCESS` signal + `crawls_succeeded` counter). Sibling: `03b-proxy-expansion.md`. `03d` is deferred. +> Depends on `03a-crawler-core.md` (the `CrawlOutcomeStatus.SUCCESS` signal + `crawls_succeeded` counter). Siblings: `03b-proxy-expansion.md`, `03e-stealth-hardening.md`, `03d-captcha-solving.md` (now active). `03d` extends this service with a **separate per-attempt captcha unit** (see §"Captcha billing seam" + `03d §4`). ## Objective @@ -49,6 +49,8 @@ New `app/services/web_crawl_credit_service.py` — a dedicated service (independ A separate `CreditMeterService` generalization is deliberately deferred until a third per-unit biller appears — exposing the price/flag as statics already lets the chat surface share the math without coupling the two flows. +**Captcha billing seam (for `03d`).** `03d` (now active) adds a **separate per-attempt** captcha unit to this same service — `captcha_solves_to_micros(n) = n * config.WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE`, gated by `WEB_CRAWL_CAPTCHA_BILLING_ENABLED` (both new knobs, default FALSE), with `usage_type="web_crawl_captcha"`. It is **independent** of crawl SUCCESS because solvers charge per *attempt* regardless of crawl outcome (the asymmetry in the §"Risks" proxy/captcha note). Build the `WebCrawlCreditService` statics so `03d` can add one more `*_to_micros` static + one charge path without restructuring — no need to implement the captcha methods in `03c`, just leave the shape open. + ### 2. Wiring in `webcrawler_indexer.py` 1. **Resolve the owner.** Bill the **workspace owner** (the product concept), not the triggering `user_id`: `owner_user_id = (select SearchSpace.user_id where id == search_space_id)`. (Mirrors how `billable_calls._resolve_agent_billing_for_search_space` (func `:441`) reads `search_space.user_id` at `billable_calls.py:499` — but we only need the owner id, so a direct `select` is enough; no need to call that LLM-model-resolving helper.) @@ -117,7 +119,8 @@ One unit per `CrawlOutcomeStatus.SUCCESS` — a URL that yielded usable extracte - **Session coupling.** `charge_credits` commits the indexer session; placing it after the final doc commit avoids entangling a debit with mid-run document state. - **Gating asymmetry (intended).** The **indexer** path bills whenever `WEB_CRAWL_CREDIT_BILLING_ENABLED` is on and the owner has credit — no "premium tier" requirement (matches ETL, which gates only on its own flag). The **chat** path additionally requires the turn to be premium, because it piggybacks the turn's reserve/finalize machinery. Same price, two different gate sets — call this out in the credit-status UI copy (frontend phase). - **No cross-surface double charge.** A chat scrape calls `crawl_url` directly and never enters `webcrawler_indexer`, so a given crawl is billed by exactly one surface. -- **Proxy/captcha cost.** Absorbed into the flat $1/1000 (margin), **not** metered separately (see `03b`, and `03d` for captcha which has its own per-solve upstream cost — revisit there). +- **Proxy cost.** Absorbed into the flat $1/1000 (margin), **not** metered separately (see `03b`). +- **Captcha cost.** **Metered separately** as a per-attempt unit (`03d §4`, decided: option (a)) because the solver charges per attempt regardless of crawl success — it cannot ride the per-success flat rate. The `captcha_solves_to_micros` seam above is where it attaches. ## Resolved decisions (this pass) @@ -128,5 +131,5 @@ One unit per `CrawlOutcomeStatus.SUCCESS` — a URL that yielded usable extracte ## Out of scope (hand-offs) - Per-**pipeline** run accounting / `charged_micros` persistence for idempotency → Phases 5–7. Specifically, **Phase 6 moves pipeline-run billing out of the indexer to the run level**: it adds a `bill: bool = True` param to this indexer wiring and calls `index_crawled_urls(bill=False)` from the pipeline engine, which then does its own `check_credits` + `charge_credits(crawls_succeeded)` and stamps `PipelineRun.charged_micros` (idempotency). So this phase's in-indexer billing keeps serving the **connector `/index` + periodic** paths; the "pipeline crawls" half of the objective is refined by `06`. Structure the §2 wiring so it's easy to gate behind that flag. -- Captcha-solver upstream cost pass-through → `03d` (deferred). +- Captcha-solver upstream cost pass-through → `03d` (now active; separate per-attempt `web_crawl_captcha` unit via the seam above). - Surfacing crawl spend in the credit-status UI → frontend umbrella. diff --git a/plans/backend/03d-captcha-solving.md b/plans/backend/03d-captcha-solving.md index e2dc92773..7a95789fd 100644 --- a/plans/backend/03d-captcha-solving.md +++ b/plans/backend/03d-captcha-solving.md @@ -1,11 +1,16 @@ -# Phase 3d — Captcha solving (DEFERRED — sequenced last, non-MVP-blocking) +# Phase 3d — Captcha solving (reCAPTCHA / hCaptcha / image) > Part of **Phase 3 — WebURL Crawler & Crawl Billing**. See `00-umbrella-plan.md`. -> **Status: deferred.** Build only after `03a`/`03b`/`03c` ship and there's a real need. Depends on `03a` (the StealthyFetcher tier) and the app-wide proxy provider from `03b` (env-selected `get_active_provider()`, accessed via the zero-arg `get_proxy_url()`). Touches the crawl billing model from `03c`. +> **Status: ACTIVE (sequenced last in Phase 3).** No longer deferred — captcha bypass is now an explicit goal of the "undetectable, captcha-bypassing universal WebURL crawler." Build **after** `03a` (StealthyFetcher tier + `FetchStrategy` seam + `CrawlOutcome`), `03b` (app-wide proxy provider, accessed via the zero-arg `get_proxy_url()`), `03c` (crawl wallet), and `03e` (stealth-hardening — solving is the *last* resort once humanization + fingerprint coherence have minimized challenges). +> **Decision recap (this pass):** Cloudflare stays in-framework (free, `03a`); reCAPTCHA/hCaptcha/image use a **paid** solver via `captchatools`, **opt-in + off by default**, and are billed as a **separate per-attempt unit** (option (a) below — now decided, not deferred). Logged-in/account-gated captcha flows are **out of scope** (no authenticated scraping in this MVP — see umbrella Phase 8 / `03b` static-proxy hand-off). -## Why deferred +## Why this is the last tier, not the first -Scrapling already solves **Cloudflare Turnstile/Interstitial** via `solve_cloudflare=True` on the StealthyFetcher tier (`03a`; `references/Scrapling/scrapling/fetchers/stealth_chrome.py:38,90`). That covers the most common interstitial. The remaining captcha types (reCAPTCHA v2/v3, hCaptcha, image) need a **paid third-party solver**, add latency (10–60s/solve), have <100% success, and carry target-ToS/legal considerations. None of that is MVP-blocking, so it's sequenced last. +A solve is the **most expensive and least reliable** move available (paid per attempt, 10–60 s latency, <100% success, ToS-sensitive). So the crawler must exhaust the cheaper, in-house levers first; captcha solving only fires when everything else has already failed: + +1. `03e` humanization + fingerprint coherence (geoip locale/tz, persistent profile, headed/Xvfb) → **avoid** triggering a challenge at all. +2. `03a` `solve_cloudflare=True` on StealthyFetcher → free Cloudflare Turnstile/Interstitial. +3. **This subplan** → only the residual reCAPTCHA v2/v3 + hCaptcha + image challenges, and only when `CAPTCHA_SOLVING_ENABLED`. ## Scope boundary @@ -13,46 +18,89 @@ Scrapling already solves **Cloudflare Turnstile/Interstitial** via `solve_cloudf | --- | --- | --- | | Cloudflare Turnstile / Interstitial | Scrapling `solve_cloudflare=True` | `03a`, StealthyFetcher tier | | reCAPTCHA v2/v3, hCaptcha, image | **`captchatools`** (this subplan) | StealthyFetcher `page_action` | +| DataDome / Kasada / reCAPTCHA-**Enterprise** | **none in the free stack** | deferred paid-unblocker tier (`03e`) | ## Grounding (libraries verified) -- **`captchatools` is itself the provider registry.** `new_harvester(api_key, solving_site, sitekey, captcha_url, captcha_type="v2"|"v3"|"hcaptcha"|"image", ...)` → `.get_token(proxy=…, proxy_type=…, user_agent=…, b64_img=…)` returns a **token string** (`references/Captcha-Tools/README.md:28–47`). `solving_site` ∈ {`capmonster`,`2captcha`,`anticaptcha`,`capsolver`,`captchaai`} (`:113–127`). Errors: `ErrNoBalance`, `ErrWrongAPIKey`, `ErrWrongSitekey`, … via `captchatools.exceptions` (`:136–155`). **Implication:** we do **not** rebuild a multi-provider class hierarchy (unlike `03b`'s proxy registry) — `captchatools` already dispatches across the 5 services. Our layer is thin: config resolution + page detection/injection glue. +- **`captchatools` is itself the provider registry.** `new_harvester(api_key, solving_site, sitekey, captcha_url, captcha_type="v2"|"v3"|"hcaptcha"|"image", invisible_captcha=…, min_score=…, action=…)` → `.get_token(proxy=…, proxy_type=…, user_agent=…, b64_img=…)` returns a **token string** (`references/Captcha-Tools/README.md:28–47`). `solving_site` ∈ {`capmonster`,`2captcha`,`anticaptcha`,`capsolver`,`captchaai`} (`:113–127`). Errors: `ErrNoBalance`, `ErrWrongAPIKey`, `ErrWrongSitekey`, `ErrIncorrectCapType`, `ErrNoHarvester` via `captchatools.exceptions` (`:136–155`). **Implication:** we do **not** rebuild a multi-provider class hierarchy (unlike `03b`'s proxy registry) — `captchatools` already dispatches across the 5 services. Our layer is thin: config resolution + page detection/injection glue + billing. - **`captchatools` only harvests a token; it does not inject it.** The caller must drop the token into the page (`g-recaptcha-response` / `h-captcha-response` textarea, or invoke the JS callback) and submit. -- **Injection requires a browser page**, so this only works on the **StealthyFetcher** tier. Scrapling exposes `page_action`: "a function that takes the `page` object, runs after navigation, and does the automation you need" (`stealth_chrome.py:30,82`). AsyncFetcher (HTTP) and DynamicFetcher cannot solve interactive captchas — a captcha hit there must escalate to StealthyFetcher. +- **Injection requires a live browser page**, so this only works on the **StealthyFetcher** tier. Scrapling exposes `page_action`: "a function that takes the `page` object, runs after navigation, and does the automation you need" (`references/Scrapling/scrapling/fetchers/stealth_chrome.py:30,82`; engine docstrings `_browsers/_stealth.py:50,191,326,466`). The HTTP (`AsyncFetcher`) and `DynamicFetcher` tiers cannot solve interactive captchas — a captcha hit there must **escalate to StealthyFetcher** (`FetchStrategy` seam from `03a`; the ladder already ends there). +- **Ordering is already correct in Scrapling.** `solve_cloudflare` runs **before** `page_action` (sync `_stealth.py:253–254` then `:258–260`; async `:529–530` then `:534–536`). So Cloudflare is cleared first, *then* our injector runs — exactly the tier order we want. `page_action` exceptions are caught + logged by Scrapling (`:262` / `:538`), so our factory must also return a clear signal (it can't rely on raising to abort the fetch). -## Sketch (when built) +## Target design -1. **Config layer** (`app/utils/captcha/`, mirroring `app/utils/proxy/`'s config resolution from `03b`): - - `CaptchaConfig` = `(enabled, solving_site, api_key)`, resolved from **env only** — one app-wide config, mirroring `03b`'s env-only single-provider model (`get_active_provider()`); **no per-connector config** (that path was dropped in `03b`). - - Env: `CAPTCHA_SOLVING_ENABLED` (default FALSE), `CAPTCHA_SOLVER_PROVIDER`, `CAPTCHA_SOLVER_API_KEY`. Off by default → zero captcha attempts (and zero solver cost). -2. **Detection + injection `page_action` factory** — builds a callback passed to `StealthyFetcher.async_fetch(..., page_action=…)`: - - Detect sitekey in DOM (`.g-recaptcha[data-sitekey]`, `.h-captcha[data-sitekey]`, reCAPTCHA-v3 via `grecaptcha.execute`). - - Harvest: `new_harvester(solving_site, api_key, sitekey, captcha_url=page.url, captcha_type=…).get_token(proxy=, user_agent=)` — see the IP-binding caveat below. - - Inject token + dispatch events / invoke callback; submit; wait for navigation. -3. **Crawler escalation**: only the StealthyFetcher tier attempts solving; a captcha detected on a lower tier escalates to StealthyFetcher (the ladder already ends there per `03a`). -4. **Dependency**: add `captchatools` to `pyproject.toml` (build-time only). +### 1. Config layer (`app/utils/captcha/`, mirroring `app/utils/proxy/` from `03b`) -## The billing asymmetry (the hard part — decide at build time) +- `CaptchaConfig = (enabled, solving_site, api_key, captcha_type_default, min_score, max_attempts, timeout_s)`, resolved from **env only** — one app-wide config, mirroring `03b`'s env-only single-provider model (`get_active_provider()`); **no per-connector config** (that path was dropped in `03b`). +- Env knobs (next to the proxy + crawl-billing knobs in `config/__init__.py`): + - `CAPTCHA_SOLVING_ENABLED` (default `FALSE`) — off ⇒ zero solve attempts and zero solver cost. + - `CAPTCHA_SOLVER_PROVIDER` (e.g. `capsolver`), `CAPTCHA_SOLVER_API_KEY`. + - `CAPTCHA_MAX_ATTEMPTS_PER_URL` (default `1`), `CAPTCHA_SOLVE_TIMEOUT_S` (default `120`). +- A `captcha_enabled()` static (mirrors `WebCrawlCreditService.billing_enabled()`) so callers gate cheaply before constructing anything. -`03c` bills the workspace owner **per successful crawl** (`CrawlOutcomeStatus.SUCCESS`), and absorbs proxy cost into the flat $1/1000. **Captcha is different**: the solver charges **per attempt** (~$1–3 / 1000, type-dependent) **regardless of whether the crawl ultimately succeeds**. So a failed solve = real upstream cost with **no billable success** — proxy's absorb-it model doesn't transfer cleanly. +### 2. Detection + injection `page_action` factory -Options (resolve when this is actually built): -- **(a) Separate per-solve charge** — meter each solve attempt as its own unit (e.g. `web_crawl_captcha` usage_type, its own `*_MICROS_PER_*` knob), independent of crawl SUCCESS. Most cost-honest; bills even on failed solves (matching the upstream charge). -- **(b) Higher crawl price when captcha enabled** — absorb into a fatter flat rate; simplest UX, but cross-subsidizes failed solves and easy-vs-hard pages unevenly. -- **(c) Cost-plus pass-through** — meter the solver's reported cost × margin. +Builds the **sync** callable passed to `StealthyFetcher.fetch(..., page_action=…)` (03a runs StealthyFetcher via `asyncio.to_thread`, so the sync engine path `_stealth.py:258–262` applies — the factory returns a plain `def(page): ...`, not a coroutine): -Recommendation leaning **(a)** (separate per-attempt unit) because the upstream cost is per-attempt and significant, but defer the final call. +1. **Detect** the challenge + sitekey in the DOM: + - reCAPTCHA v2: `.g-recaptcha[data-sitekey]` (or iframe `src*="recaptcha"`). + - hCaptcha: `.h-captcha[data-sitekey]` (or iframe `src*="hcaptcha"`). + - reCAPTCHA v3: presence of `grecaptcha.execute` + a known `action` (config/site-mapped). + - If no sitekey found → **no-op return** (nothing to solve; let `wait_selector`/extraction proceed). +2. **Bind the proxy/UA** (the IP-coherence caveat below): harvest with the **exact endpoint used for this crawl tier** — + `new_harvester(solving_site, api_key, sitekey, captcha_url=page.url, captcha_type=…, min_score=…, action=…).get_token(proxy=, proxy_type="HTTP", user_agent=)` (`README.md:44–46,107–110`). +3. **Inject + submit**: set the response token into `g-recaptcha-response` / `h-captcha-response` (make textarea visible if needed), dispatch `input`/`change`, invoke the JS callback when present (`grecaptcha.getResponse` flows / `___grecaptcha_cfg` callbacks), then submit the form and `page.wait_for_load_state`. For v3 (no widget) the token is consumed by the site's own `execute()` flow. +4. **Signal outcome** via a mutable closure cell (since Scrapling swallows `page_action` exceptions at `:262` **and discards its return value** at `:260`/`:536`): record `solved: bool` + `attempts: int` into a captured `dict` the crawler reads after `fetch()` returns, so billing (§4) and the retry cap can act on it. **This `page_action`+closure-cell helper is shared with `03f`'s test harness** (which uses the same mechanism to read JS-object verdicts like CreepJS `window.Fingerprint`) — factor it once. + +### 3. Crawler escalation (uses the `03a` `FetchStrategy` seam) + +- Only the **StealthyFetcher** strategy attempts solving. A captcha detected on a lower tier (HTTP/DynamicFetcher) returns non-`SUCCESS`, and the ladder already escalates to StealthyFetcher (`03a`), which this time is constructed with the captcha `page_action` when `captcha_enabled()`. +- **Attempt cap:** at most `CAPTCHA_MAX_ATTEMPTS_PER_URL` solves per URL so one hostile page can't burn unbounded solver credit. The closure cell's `attempts` enforces it. +- No new caller *contract*: the strategy still returns `CrawlOutcome` (`03a` invariant). Captcha is a *parameterization* of the last tier, not a new return shape. +- **But billing needs the attempt count to escape the closure cell.** §4 charges per attempt on **both** the indexer and chat paths, so the count can't stay buried inside `page_action`'s closure — the crawler must copy it onto the outcome. Add **`captcha_attempts: int = 0`** (and `captcha_solved: bool = False`) as **dataclass fields on `CrawlOutcome`** (`03a`). This is safe: `03a`'s "don't widen the return" rule is about the *indexer's* positional `(total_processed, error)` tuple unpacked by length, **not** about `crawl_url`'s dataclass — adding fields to a dataclass breaks no consumer. The indexer sums `outcome.captcha_attempts`; the chat tool reads it off the same outcome. + +### 4. Billing — separate per-attempt unit (option (a), DECIDED) + +`03c` bills per **successful crawl** (`CrawlOutcomeStatus.SUCCESS`) and absorbs proxy cost into the flat $1/1000. **Captcha can't ride that model**: the solver charges **per attempt regardless of whether the crawl ultimately succeeds**, so a failed solve = real upstream cost with no billable crawl success. We therefore meter solves **independently**: + +- New static on `WebCrawlCreditService` (defined in `03c`, knobs added here): `captcha_solves_to_micros(n) = n * config.WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE`. +- Each **attempt** (not each success) is charged when `CAPTCHA_SOLVING_ENABLED` **and** `WEB_CRAWL_CAPTCHA_BILLING_ENABLED`: + - **Indexer/pipeline path:** after the crawl loop, debit `captcha_solves_to_micros(total_attempts)` — where `total_attempts = Σ outcome.captcha_attempts` over the batch — from the **workspace owner** (same owner resolution as `03c §2`) and `record_token_usage(usage_type="web_crawl_captcha", …, cost_micros=…, call_details={"attempts": n, "solved": k})`. Added before the charge commit, same one-transaction pattern as `03c`. + - **Chat-scrape path:** fold `captcha_solves_to_micros(outcome.captcha_attempts)` into the turn accumulator with `call_kind="web_crawl_captcha"` (same mechanism as `03c §3`), so it settles with the premium turn; non-premium/anonymous turns record-but-don't-debit. +- **No pre-block on solves** (unlike the crawl batch): attempts are bounded by `CAPTCHA_MAX_ATTEMPTS_PER_URL × len(urls)`, an upper bound the indexer can optionally pre-check against the wallet if we want symmetry with `03c §2` (recommended: pre-check the combined crawl + worst-case captcha estimate so a run can't strand mid-batch on solver insolvency). +- New env (next to `03c`'s knobs): `WEB_CRAWL_CAPTCHA_BILLING_ENABLED` (default `FALSE`), `WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE` (default e.g. `3000` = $3/1000 attempts; type-dependent, set with margin over the solver's per-attempt price). **No DB migration** — `web_crawl_captcha` is just another free-form `TokenUsage.usage_type` value (`db.py` `usage_type String(50)`). + +### 5. Dependency + Docker + +- Add `captchatools` to `pyproject.toml` (build-time only). It's a thin HTTP client for the solver services — no browser binaries. +- No `Dockerfile` change beyond the dependency (browser already installed by `03a`/`03e`). + +## Error handling (must not loop or silently burn credit) + +- `ErrNoBalance` / `ErrWrongAPIKey` (`README.md:134,136–143`) → **stop solving for the rest of the run** (set a process/run flag), surface a clear "captcha solver out of balance / misconfigured" message via `log_task_failure`, and fall through to a non-`SUCCESS` `CrawlOutcome`. The README explicitly warns balance-exhausted loops can get the **IP temporarily banned** (`:132–134`) — never retry on `ErrNoBalance`. +- `ErrWrongSitekey` / `ErrIncorrectCapType` → log + skip this URL's solve (likely detection bug, not transient); count the attempt for billing only if the solver actually charged. +- Timeout (`CAPTCHA_SOLVE_TIMEOUT_S`) → abort the solve, count one attempt, return non-`SUCCESS`. ## Risks / considerations - **Solver-tier only.** No captcha solving on the HTTP/DynamicFetcher tiers; must escalate to StealthyFetcher (slower, browser-backed). -- **Latency & flakiness.** Solves take 10–60s and aren't guaranteed; tune timeouts and a max-attempts cap so a single URL can't burn unbounded solver credit. -- **Solver-account balance.** `ErrNoBalance`/`ErrWrongAPIKey` (`README.md:136–155`) must surface clearly and disable solving rather than loop. -- **Proxy coherence (rotating-pool caveat).** Some captchas IP-bind the token, so the solver must egress from the **same IP** as the crawl. Under `03b`'s single app-wide provider this is automatic for single-endpoint providers (`anonymous_proxies`, single-URL `custom`). But a **pool-backed `CustomProxyProvider`** returns the *next* endpoint on each zero-arg `get_proxy_url()` call — so the `page_action` must **reuse the endpoint actually used for this crawl tier** (capture it once, pass it to `get_token(proxy=…)`), NOT call `get_proxy_url()` again (which would rotate to a different IP). This needs a small seam to surface the crawl's chosen endpoint into the `page_action` — note it when `03d` is built on top of `03b`'s rotation. -- **Policy / ToS.** Automated captcha solving may violate a target site's terms; gate behind the explicit `CAPTCHA_SOLVING_ENABLED` flag and treat as an opt-in, owner-acknowledged capability. +- **Latency & flakiness.** Solves take 10–60 s and aren't guaranteed; the `CAPTCHA_MAX_ATTEMPTS_PER_URL` + `CAPTCHA_SOLVE_TIMEOUT_S` caps keep a single URL from burning unbounded solver credit. +- **Proxy coherence (rotating-pool caveat).** Many captchas IP-bind the token, so the solver must egress from the **same IP** as the crawl. Under `03b`'s single app-wide provider this is automatic for single-endpoint providers (`anonymous_proxies`, single-URL `custom`). But a **pool-backed `CustomProxyProvider`** returns the *next* endpoint on each zero-arg `get_proxy_url()` call — so the `page_action` must **reuse the endpoint actually used for this crawl tier** (the crawler captures it once and passes it into the factory), NOT call `get_proxy_url()` again (which would rotate to a different IP). This is the same "surface the chosen endpoint" seam `03e`/`03d` both rely on — build it once in `03a`'s strategy context. +- **Enterprise captchas are out of reach.** reCAPTCHA **Enterprise**, DataDome, and Kasada use behavioral + device signals the token-injection model doesn't satisfy reliably; those route to the deferred paid-unblocker tier (`03e`), not here. +- **Policy / ToS.** Automated captcha solving may violate a target site's terms; gate behind the explicit `CAPTCHA_SOLVING_ENABLED` flag and treat as an opt-in, owner-acknowledged capability. Public data only (no logged-in bypass). + +## Work items + +1. **`app/utils/captcha/`**: `CaptchaConfig` + env resolution + `captcha_enabled()` static. +2. **`page_action` factory**: detection (v2/v3/hcaptcha/image sitekey), harvest via `captchatools` with bound proxy/UA, inject+submit, outcome closure cell. +3. **Crawler wiring**: construct StealthyFetcher with the factory when `captcha_enabled()`; enforce per-URL attempt cap; surface the crawl's chosen proxy endpoint into the factory. +4. **Billing**: `WebCrawlCreditService.captcha_solves_to_micros` + the two knobs; debit per-attempt on the indexer path (owner) and fold into the turn on the chat path; `record_token_usage(usage_type="web_crawl_captcha")`. +5. **Config + docs**: all env knobs in `Config` + `.env.example` (commented, hosted=TRUE / self-hosted=FALSE), plus the `captchatools` dependency. +6. **Tests**: solving disabled → zero attempts, zero solver cost (self-hosted); v2 detected → harvest+inject+submit path invoked with the **crawl's** proxy endpoint (not a re-rotated one); `ErrNoBalance` → stops solving + no retry loop; attempt cap honored; billing enabled → per-attempt debit even when the crawl ultimately fails; chat path folds captcha micros into a premium turn and record-only on anonymous; `web_crawl_captcha` `TokenUsage` rows written. ## Out of scope -- Anything in `03a`/`03b`/`03c`. +- Anything owned by `03a`/`03b`/`03c`/`03e`. - Cloudflare Turnstile (already handled by Scrapling in `03a`). -- Final captcha billing model — chosen at build time (see options above). +- Logged-in / account-gated captcha flows (no authenticated scraping this MVP → umbrella Phase 8 + `03b` static-proxy hand-off). +- Enterprise/behavioral anti-bot (DataDome/Kasada/reCAPTCHA-Enterprise) → deferred paid-unblocker tier (`03e`). diff --git a/plans/backend/03e-stealth-hardening.md b/plans/backend/03e-stealth-hardening.md new file mode 100644 index 000000000..05fb18343 --- /dev/null +++ b/plans/backend/03e-stealth-hardening.md @@ -0,0 +1,119 @@ +# Phase 3e — Stealth hardening (the in-house "undetectable" moat) + +> Part of **Phase 3 — WebURL Crawler & Crawl Billing**. See `00-umbrella-plan.md`. +> Depends on `03a` (Scrapling StealthyFetcher tier + `FetchStrategy` seam + `CrawlOutcome`) and `03b` (app-wide proxy provider). Precedes `03d` (captcha) in the escalation order — hardening **avoids** challenges; captcha solving is the paid last resort for the ones we can't avoid. Touches `03c` only via the deferred paid-unblocker tier (its own billing, decided if/when built). + +> **Implementation note.** Same convention as `03a`–`03d`: citations use **today's** names (`search_space_id`/`SearchSpace`) and predate `03a`'s `crawl_url` refactor; locate code by **symbol/grep**, not absolute lines. Scrapling references point at the on-disk (gitignored) `references/Scrapling/` checkout pinned to `scrapling[fetchers]>=0.4.9` (`pyproject.toml:91`). + +## Objective + +Push the Universal WebURL Crawler as far toward "undetectable" as the **free / in-house stack** allows, so we hold a scraping moat for the next ~4–6 months **without** a third-party unblocker (ZenRows/ScrapFly/Bright Data) or a source-patched browser (CloakBrowser — rejected on licensing). Everything here is **runtime/config-level** stealth layered on Scrapling's patchright-Chromium (`03a` engine note): geoip fingerprint coherence, persistent profiles, headed execution, real fonts, and behavioral humanization — plus a **block classifier** + **per-domain strategy memory** so the ladder learns and the cheap tiers get skipped once a domain's working strategy is known. The paid-unblocker tier is defined here as a **deferred** `FetchStrategy` (the explicit escape hatch for when in-house maintenance gets too costly), not built now. + +## The realistic ceiling (be honest with downstream devs) + +Scrapling already ships strong **runtime** stealth by default: `DEFAULT_ARGS + STEALTH_ARGS` (`references/Scrapling/scrapling/engines/constants.py:24–99`, incl. `--disable-blink-features=AutomationControlled` `:94`), a persistent context (`_browsers/_stealth.py:91–93,366–368`), and `navigator.webdriver` masking via patchright. With `03b` residential proxies + this plan's coherence/humanization on top, the crawler reliably handles **Cloudflare** (via `03a` `solve_cloudflare`) and the long tail of **moderate** anti-bot. + +What the free stack does **not** reliably beat: **DataDome, Kasada, reCAPTCHA-Enterprise**, and other behavioral/device-fingerprint systems. Those generally require C++ source-level browser patches (CloakBrowser's 58-patch approach — out on licensing) or a commercial unblocker. For **competitive-intelligence targets** these top-tier defenses are the exception, not the rule, so the in-house moat is a sound 4–6 month bet — with the deferred paid tier (§8) as the pre-wired escape hatch. **Do not promise "beats everything."** + +## Levers (all behind the `03a` `FetchStrategy` seam — callers stay outcome-only) + +### 1. Geoip fingerprint coherence (match the browser to the proxy exit IP) + +A residential IP in Berlin behind an `en-US`/`America/New_York` browser is an instant tell. Make the fingerprint cohere with the proxy's exit geo: + +- Resolve the proxy exit IP → country/locale/timezone (lightweight geoip; e.g. a bundled MaxmindLite/`geoip2` DB or the proxy provider's location knob `RESIDENTIAL_PROXY_LOCATION` from `03b`). +- Pass the matched values into StealthyFetcher: `locale=` (drives `navigator.language` + `Accept-Language` — `stealth_chrome.py:34–35`) and `timezone_id=` (`:36`). These flow into the Playwright context (`_browsers/_base.py:440–441`). +- This needs the crawl's **chosen proxy endpoint** surfaced into the strategy (the same seam `03d` needs for IP-bound solves) — capture it once in `03a`'s strategy context, don't re-call `get_proxy_url()` (which rotates on a pool-backed provider, `03b`). + +### 2. Fingerprint flags Scrapling already exposes (turn them on) + +- `hide_canvas=True` — random noise on canvas ops to defeat canvas fingerprinting (`stealth_chrome.py:40`). +- `block_webrtc=True` — forces WebRTC to respect the proxy, preventing the **real local IP leak** that unmasks proxied browsers (`:41`). +- `google_search=True` (default) — sets a Google referer so the first hit looks like organic arrival (`:45`); override per-need via `extra_headers` (`:46`). +- `additional_args=` — last-priority Playwright context overrides for anything not surfaced as a first-class param (`:51`). + +### 2b. HTTP-tier TLS fingerprint (the AsyncFetcher tier — `impersonate`) + +The cheap static tier (`03a` tier 1) is the **first** thing every crawl hits, yet it currently sets `stealthy_headers=True` but **no** `impersonate` (`webcrawler_connector.py:284–289`), so its TLS ClientHello is curl_cffi's default JA3 — a non-browser signature that fingerprinting WAFs flag before the body even loads. Pass an `impersonate="chrome"` profile (Scrapling's static engine selects a matching curl_cffi browser profile — `references/Scrapling/scrapling/engines/static.py:36–47`) so the HTTP tier's **JA3/JA4/HTTP-2** matches a real Chrome and coheres with the browser tiers' UA. Cheap, safe, default-on. `03f §S3` is the test that validates parity against `tls.peet.ws`. + +### 3. Persistent per-domain profiles (look like a returning human) + +Scrapling defaults to a **temporary** user-data dir (fresh = suspicious). Use `user_data_dir=` (`stealth_chrome.py:48`; `launch_persistent_context` `_stealth.py:93,366–368`) to keep a **persistent profile per domain** (or per domain+proxy-geo), so cookies/localStorage/site-trust carry across crawls and the browser presents as a returning visitor rather than a brand-new incognito session. Store profiles under a configured dir (`shared_tmp`-style volume so API + worker share them). + +### 4. Headed execution under Xvfb (defeat headless tells) + +`headless` defaults to hidden (`stealth_chrome.py:19/71`). Many WAFs flag headless Chromium. Run **headful** (`headless=False`) inside a virtual framebuffer (**Xvfb**) in the Docker worker so the browser is "visible" to itself but needs no real display. Gate behind a config flag (off by default for self-hosted, on for hosted workers that have Xvfb installed). + +### 5. Real fonts (canvas/emoji hash realism) + +A minimal container has almost no fonts, making canvas/emoji fingerprint hashes obviously synthetic. Install real font packages (DejaVu, Liberation, Noto incl. CJK + emoji) in the worker image so font-enumeration + canvas hashes resemble a real desktop. `Dockerfile` change only (`apt-get install fonts-*`). + +### 6. Behavioral humanization (DIY — Scrapling has no `humanize` for the Chromium engine) + +Unlike Camoufox, the patchright-Chromium StealthyFetcher exposes **no built-in `humanize`** (verified: zero matches in the StealthyFetcher param set). So humanization is custom, injected via the existing hooks: + +- `page_action=` (runs after navigation, `stealth_chrome.py:30`; engine `_stealth.py:258–262`) — randomized mouse moves/curves, scrolls, hover-before-click, and small think-time delays before extraction. This is the **same hook `03d` uses** for token injection, so the two compose (humanize → optional captcha solve). +- `init_script=` (JS executed on page creation, `:33`) — early shims for any residual JS tells not covered by patchright. +- Tunable `wait`/`network_idle` (`:29,27`) so dwell time isn't robotically constant. + +### 7. Block classifier + per-domain strategy memory (the "learning ladder") + +Two small in-house pieces make the ladder smart instead of brute-force: + +- **Block classifier** — inspect each `Response` (status + body/cookie markers) and label the outcome: `OK` / `CLOUDFLARE` / `CAPTCHA_RECAPTCHA` / `CAPTCHA_HCAPTCHA` / `DATADOME` / `KASADA` / `RATE_LIMITED` / `EMPTY`. (Markers: `"Just a moment"` + `cf-mitigated` → Cloudflare; `datadome` cookie/script → DataDome; `g-recaptcha`/`h-captcha` sitekeys → captcha; etc.) The label drives escalation (which tier/lever next), routes captcha types to `03d`, and feeds the memory below. It also makes `CrawlOutcome.status` decisions principled instead of "empty == fail". +- **Per-domain strategy memory** — cache the **strategy that last succeeded per domain** (e.g. Redis key `crawl:strategy:{domain}` with TTL, reusing the existing Redis the indexer already uses for `indexing_locks`). Next crawl of that domain starts at the known-good tier/lever set, skipping cheaper tiers that always fail there — fewer requests, lower latency, less proxy/captcha spend. No DB migration (Redis, best-effort, self-healing on miss). + +## 8. Deferred: paid-unblocker tier (the escape hatch, NOT built now) + +The moat strategy is explicit: **maintain in-house bypass for ~4–6 months, then move hostile targets to a paid unblocker if demand/maintenance justifies it.** That switch is **evidence-driven, not a guess** — `03f`'s manual scorecard quantifies the free-stack ceiling over time and is the documented trigger for flipping this tier. Pre-wire the seam so the switch is a config flip, not a refactor: + +- Define (but do not implement) a `PaidUnblockerStrategy` that satisfies the `03a` `FetchStrategy` contract `(url, ctx) -> CrawlOutcome`, appended **last** in the ladder and active only when an env flag + API key are set. +- It would call an external unblocker (ZenRows/ScrapFly/Bright Data Web Unlocker) for the residual `DATADOME`/`KASADA`/`reCAPTCHA-Enterprise` domains the block classifier flags as unreachable in-house. +- **Its own billing** (cost-plus pass-through, decided at build time) — separate from `03c`'s flat crawl unit and `03d`'s per-solve unit, because unblocker pricing is per-request and provider-specific. +- Until built, those domains simply return non-`SUCCESS` (free under `03c`). This keeps the umbrella's "WebURL Crawler is the moat" honest while bounding our maintenance risk. + +## Config / env changes + +Add (all default OFF / conservative; next to the `03b`/`03c` knobs in `config/__init__.py` + `.env.example`): + +- `CRAWL_GEOIP_MATCH_ENABLED` (default FALSE) + optional geoip DB path. +- `CRAWL_HIDE_CANVAS` / `CRAWL_BLOCK_WEBRTC` (default TRUE — cheap, safe). +- `CRAWL_PERSISTENT_PROFILES_DIR` (unset → Scrapling's temp default; set → per-domain profiles). +- `CRAWL_HEADED_XVFB_ENABLED` (default FALSE; requires Xvfb in the image). +- `CRAWL_HUMANIZE_ENABLED` (default TRUE) + dwell/jitter bounds. +- `CRAWL_STRATEGY_MEMORY_TTL_S` (default e.g. `86400`; 0 → disabled). +- `CRAWL_PAID_UNBLOCKER_ENABLED` (default FALSE) + provider/key (deferred tier). + +## Docker changes (`surfsense_backend/Dockerfile`) + +- Install **Xvfb** + a font set (`fonts-dejavu`, `fonts-liberation`, `fonts-noto`, `fonts-noto-cjk`, `fonts-noto-color-emoji`) in the worker image. (Note from `03a`: also drop the stale "+ Camoufox" comment near `:112`; `scrapling install` only fetches Chromium.) +- Headed runs need the browser launched under `xvfb-run` (or an Xvfb display in the worker entrypoint), gated by `CRAWL_HEADED_XVFB_ENABLED`. + +## Work items + +1. **Geoip coherence** — resolve proxy exit geo → `locale`/`timezone_id`; thread the crawl's chosen endpoint into the strategy context (shared seam with `03d`). +2. **Fingerprint flags** — wire `hide_canvas`/`block_webrtc`/`google_search`/`extra_headers`/`additional_args` from config into the StealthyFetcher tier. Also add `impersonate="chrome"` to the AsyncFetcher (HTTP) tier (§2b) so its TLS coheres. **Centralize this into a single per-tier kwargs builder** (one function that returns the StealthyFetcher / AsyncFetcher kwargs from config) — the crawler *and* `03f`'s harness both import it, so the scorecard grades the exact browser we ship (no test-vs-prod drift). +3. **Persistent profiles** — per-domain `user_data_dir` under `CRAWL_PERSISTENT_PROFILES_DIR` (shared volume). +4. **Headed + Xvfb** — `headless=False` path gated by flag; Xvfb + fonts in the worker image. +5. **Humanization** — a `page_action` humanizer (mouse/scroll/dwell) composing with `03d`'s injector; optional `init_script` shims. +6. **Block classifier** — `classify(response) -> BlockType`, used by the ladder + `CrawlOutcome.status` + `03d` routing. +7. **Per-domain strategy memory** — Redis read/write around the ladder; start at known-good tier; self-heal on miss. +8. **Paid-unblocker seam** — define the deferred `FetchStrategy` + config flag only (no provider integration). +9. **Instrumentation** — log `(domain, block_type, winning_strategy, attempts, latency)` for tuning the ladder. +10. **Config + Docker + tests**. + +## Risks / trade-offs + +- **Arms race / maintenance.** Fingerprint bypasses rot as WAFs update; this is exactly the cost the deferred paid tier (§8) hedges. Instrumentation (work item 9) plus **`03f`'s scorecard** are what tell us when in-house upkeep stops being worth it. +- **Headed/Xvfb cost.** Headful browsers use more CPU/RAM than headless; gate per-flag and only escalate to headed when the block classifier says cheaper tiers fail for a domain (per-domain memory keeps it from being the default). +- **Profile growth.** Persistent profiles accumulate disk; add a size/TTL cap and periodic prune. +- **Geoip accuracy.** A coarse country→locale map is fine; over-fitting per-city tz isn't worth it. Wrong-but-coherent beats default-mismatched. +- **No silver bullet.** Reiterate the ceiling (§"realistic ceiling") in any user-facing copy: the crawler is "best-effort undetectable," not guaranteed. + +## Out of scope (hand-offs) + +- Cloudflare solving (`03a`), reCAPTCHA/hCaptcha solving + its per-solve billing (`03d`), proxy rotation (`03b`), flat crawl billing (`03c`). +- **Logged-in / account-based bypass** (sticky/static proxies + credential management) — deferred to the platform-actor work (umbrella Phase 8 + `03b` static-proxy hand-off). Public data only this MVP. +- Building the paid-unblocker provider integration — deferred (§8 leaves only the seam + flag). +- **Measuring** undetectability (the scorecard that grades these levers) → `03f` (manual harness). This plan *builds* the levers; `03f` *tests* them. +- Platform-specific structured extractors (Google Maps, LinkedIn public, …) — these sit **on top** of this hardened fetch core as Phase-8 actors; this plan only delivers the core they depend on. diff --git a/plans/backend/03f-undetectability-testing.md b/plans/backend/03f-undetectability-testing.md new file mode 100644 index 000000000..93f02d1d6 --- /dev/null +++ b/plans/backend/03f-undetectability-testing.md @@ -0,0 +1,124 @@ +# Phase 3f — Undetectability & extraction test harness (manual scorecard) + +> Part of **Phase 3 — WebURL Crawler & Crawl Billing**. See `00-umbrella-plan.md`. +> **Status: ACTIVE, sequenced last in Phase 3** (after `03a`–`03e` ship — the harness measures what those built). **Manual-only** — no CI/automated gating for now (resolved decision); it's a dev-run scorecard, not a build gate. Depends on `03a` (the Scrapling tiers + `FetchStrategy`/`CrawlOutcome`), `03b` (proxy provider), `03e` (the stealth levers being tested), and reuses the `page_action` + closure-cell mechanism shared with `03d`. + +> **Convention note.** This is **dev/operator tooling**, not a product code path, so it's untouched by the Phase 1–2 rename (no `search_space_id`/`workspace_id` concern). Citations to the gitignored reference checkouts (`references/CloakBrowser/`, `references/Scrapling/`) are pinned to what's on disk; locate code by **symbol/grep** if lines drift. + +## Objective + +A **manual, repeatable scorecard** that answers one question: *how undetectable (and how correct) is our Universal WebURL Crawler right now?* It drives the real Scrapling tiers against the industry's standard detection + sandbox sites, parses each site's verdict, and prints a pass/fail + numeric scorecard. Its real job is to **quantify the free-stack ceiling** (`03e`) over time so we know — with evidence — when fingerprint maintenance stops being worth it and we should flip the deferred paid-unblocker tier. + +It is explicitly **two axes, two suites** (kept separate so each scales independently): + +- **Suite S — Stealth / anti-bot** (is the crawler detected?): browser-tier fingerprint/bot tests, HTTP/TLS-tier fingerprint, proxy/leak verification. +- **Suite E — Extraction correctness** (does the crawler get the content right?): scraping sandboxes for the HTTP vs JS (DynamicFetcher) tiers + Trafilatura quality. + +## How CloakBrowser tests (the pattern we copy) + +CloakBrowser's suite is a **driven-browser verdict-scraper**, not unit tests (`references/CloakBrowser/examples/stealth_test.py`, run via `bin/cloaktest`). The repeated shape: + +1. Launch a real page with **proxy + geoip**, toggleable **headed/headless** (`stealth_test.py:231` `launch(headless=…, proxy=…, geoip=True)`). +2. `page.goto(site, wait_until="networkidle")` then **sleep** — scores compute async (Castle.js ~20 s `fingerprint_scan_test.py:31`; CreepJS ~30 s `:96`; reCAPTCHA polls up to 30 s `stealth_test.py:156–164`). +3. `page.evaluate(js)` to **parse the verdict** from the rendered DOM (sannysoft table `stealth_test.py:32–45`) or from internal JS objects (`window.Fingerprint.headless` `fingerprint_scan_test.py:117–127`). +4. Apply an explicit **pass threshold** + screenshot (`stealth_test.py:169–217`). + +Their shipped bars (we adopt as **aspirational targets**, see Scorecard): sannysoft **0 fails**; `bot.incolumitas` ≤ `{WEBDRIVER, connectionRTT}` known-FPs; `browserscan` **0 Abnormal**; `deviceandbrowserinfo` `isBot=false`; FingerprintJS demo **not blocked**; reCAPTCHA v3 **≥0.7** (they hit 0.9); CreepJS **headless ≤30% / stealth ≤30%** (`fingerprint_scan_test.py:112–114,166–171`). + +## The bridge to our Scrapling crawler + +We are **not** a single browser; we're the `03a` tier ladder. The harness therefore tests **per tier**, and extracts verdicts two ways: + +- **DOM/JSON-rendered verdicts** → just `StealthyFetcher.fetch(url, …)` (or `Fetcher.get` for JSON endpoints) and parse the **returned post-JS page** with Scrapling's selector (`load_dom` is on by default — `references/Scrapling/scrapling/fetchers/stealth_chrome.py:43`). Covers sannysoft, incolumitas, deviceandbrowserinfo, the reCAPTCHA score text, and every JSON endpoint (`tls.peet.ws/api/all`, `httpbin/headers`). +- **Internal JS-object verdicts** (CreepJS `window.Fingerprint`, Castle.js score node) → a **`page_action`** that runs `page.evaluate()` and stashes the result into a **closure cell**, because Scrapling **discards `page_action`'s return value** (sync `_stealth.py:260`, async `:536`). **This is the exact same `page_action`+closure-cell plumbing as `03d`'s token injector** — building the harness de-risks `03d` (and vice-versa); factor it once. + +## Suite S — Stealth / anti-bot + +### S1. Browser tier (StealthyFetcher — the "undetectable" tier) + +| Site | Signal | Extraction | Aspirational bar | +|---|---|---|---| +| `bot.sannysoft.com` | webdriver/chrome/plugins/UA leaks | DOM table | 0 fails | +| `bot.incolumitas.com` | 30+ checks incl. behavioral | JSON-in-body | ≤ `{WEBDRIVER, connectionRTT}` | +| `browserscan.net/bot-detection` | WebDriver/CDP/Navigator | DOM text | 0 Abnormal | +| `deviceandbrowserinfo.com/are_you_a_bot` | fingerprint + behavioral | JSON `isBot` | `isBot=false` | +| `abrahamjuliot.github.io/creepjs` | fingerprint **consistency/lies**, headless% | `window.Fingerprint` (page_action) | headless ≤30%, stealth ≤30% | +| `fingerprint-scan.com` | Castle.js bot-risk + headless signals | DOM node + evaluate | low risk; headless signals 0 | +| `demo.fingerprint.com/web-scraping` | **behavioral block** (click→blocked vs flights) | page_action click + DOM | not blocked | +| `recaptcha-demo.appspot.com/...v3-request-scores.php` | Google server-verified human score | `wait_for_function` + regex (`recaptcha_score.py:22–28`) | score ≥0.7 | + +### S2. Per-property fingerprint detail (manual/debug — validates `03e` levers directly) + +- `browserleaks.com/canvas`, `/webgl`, `/fonts` — confirms `03e` `hide_canvas` + font packages. +- `browserleaks.com/webrtc` + DNS leak — confirms `03e` `block_webrtc` (no real-IP leak through the proxy). + +### S3. HTTP/TLS tier (AsyncFetcher / `Fetcher` — curl_cffi impersonation) + +Our HTTP tier is `curl_cffi`-based and *can* impersonate a real Chrome TLS stack (`references/Scrapling/scrapling/fetchers/requests.py:29`; `engines/static.py:6–9,36–47`) — **but only if `impersonate=` is set.** It currently is **not** (`webcrawler_connector.py:284–289` passes `stealthy_headers=True` only), so today this tier's JA3 is curl_cffi's default and **will fail parity**. This row is therefore the *driver* for the `03e §2b` `impersonate="chrome"` lever: run it before (red) and after (green) that fix. + +- `tls.peet.ws/api/all` (+ `/api/clean`) — JSON **JA3/JA4/Akamai-HTTP2/PeetPrint**; diff against a real-Chrome baseline. +- `httpbin.co/headers` (or httpbingo) — header set/order/UA sanity. + +**Recommendation (resolved): TLS parity is a first-class *axis* but an *informational threshold*, not a hard gate.** We record JA3/JA4 and flag drift from the Chrome baseline, but don't "fail" on it — curl_cffi impersonation is strong yet JA-hashes shift across versions, and this is a manual scorecard anyway. Treat a *mismatch* as a tuning signal (pick a closer `impersonate` profile), not a regression. + +### S4. Proxy / leak verification + +- `httpbin.org/ip` — exit IP == the proxy endpoint actually used (capture-once seam from `03b`/`03e`, not a re-rotated `get_proxy_url()`). +- WebRTC/DNS (S2) — no real-IP leak. + +## Suite E — Extraction correctness (separate axis) + +Purpose-built, ToS-safe sandboxes — validates the HTTP vs **DynamicFetcher (JS)** tiers and Trafilatura output, independent of stealth: + +- `books.toscrape.com` — static catalog + pagination (HTTP tier; baseline extraction). +- `quotes.toscrape.com` — has **`/js`** (JS-rendered), **`/js-delayed`** (`?delay=`), **scroll** (infinite), **login** (CSRF) variants → exercises the DynamicFetcher JS tier + `wait_selector`/`network_idle`. +- `scrapethissite.com` — mixed structures for extraction robustness. + +Assertion style: known expected values (e.g. first book title, quote count per page) so extraction regressions are caught deterministically. + +## Scorecard & thresholds (resolved) + +- **Adopt CloakBrowser's bars as aspirational targets** (above), but the harness's primary output is **our actual measured numbers recorded as the baseline** — a committed `scorecard.md`/JSON snapshot per run (date, tier, proxy on/off, headed/headless, per-site result). Subsequent runs diff against the last baseline so we see drift (ours improving, or a WAF tightening). +- Each row reports: site, tier, verdict, numeric (where applicable), PASS/FAIL vs aspirational bar, and screenshot path. +- A run is summarized as `passed/total` per suite (like `stealth_test.py:314–317`), **never blocking** anything. + +## Harness design + +- Lives under dev tooling (e.g. `surfsense_backend/scripts/crawler_testbench/` or `tests/manual/` — not collected by the normal pytest run, since it hits the live internet + needs proxies). A thin CLI mirrors `bin/cloaktest`: `python -m ... [--proxy URL] [--headed] [--headless] [--suite S|E|all] [--no-screenshots]`. +- **Reuse split (important — `crawl_url` is *not* a drop-in here):** + - **Suite E** drives the **real `crawl_url`** end-to-end — extraction correctness *is* the production path (auto-ladder + Trafilatura markdown is exactly what we want to assert). + - **Suite S** does **not** use `crawl_url`. Two reasons: (1) `crawl_url` auto-ladders and stops at the first `SUCCESS`, so a detection site might be answered by the cheap HTTP tier when we mean to grade the **StealthyFetcher** tier; (2) `crawl_url` returns Trafilatura **markdown** (`_build_result`), but verdict parsing needs the **raw post-JS DOM** (and `window.*` objects need a live page). So Suite S drives the **individual Scrapling fetchers directly**, per tier. + - **Avoid test-vs-prod drift:** Suite S must construct each fetcher from the **same centralized stealth-config builder** `03e` introduces (the single source of truth for `locale`/`timezone_id`/`hide_canvas`/`block_webrtc`/`impersonate`/profile/headed), **not** a hand-rolled kwargs set — otherwise the scorecard grades a browser we don't ship. (`03e` work item: expose that builder so both the crawler and this harness import it.) +- Outputs: console summary + screenshots + the `scorecard` snapshot. +- Runs with the app-wide proxy provider (`03b`) and `03e` levers on, so the scorecard reflects production fetch behavior — **except captcha solving, which Suite S forces OFF** (`CAPTCHA_SOLVING_ENABLED=false`): we measure the *unaided* stealth/score, and it avoids the `03d` injector firing paid solves against the reCAPTCHA-demo / FingerprintJS rows. Captcha solving is exercised separately (functional `03d` test), not in the stealth scorecard. + +## The ceiling-decision loop (why this matters) + +The scorecard is the evidence base for the moat strategy: re-run it (a) on a cadence and (b) whenever crawls start failing in the wild. When the **hard** rows (FingerprintJS demo, reCAPTCHA-v3 ≥0.7, and any DataDome/Kasada targets) degrade and `03e` levers can't recover them, that's the documented trigger to flip the **deferred paid-unblocker `FetchStrategy`** (`03e §8`). Without this harness, that decision is guesswork. + +## Config / dependencies + +- **No new production dependencies** — reuses Scrapling (already pinned) + the crawler. Optionally a tiny dev helper for the scorecard diff (or just stdlib `json`). +- Needs **residential proxy creds** (the `03b` env) to be meaningful; document a "no-proxy" mode that still runs but is expected to fail the harder rows (datacenter IP). +- A results dir (gitignored screenshots; committed `scorecard` snapshots). + +## Work items + +1. **Shared `page_action`+closure-cell helper** (factored with `03d`) for JS-object verdict extraction. +2. **Suite S runners** (S1 browser, S3 TLS, S4 proxy-leak) with per-site parse + aspirational thresholds; S2 as manual debug links. +3. **Suite E runners** against the toscrape/scrapethissite sandboxes with deterministic assertions. +4. **CLI + scorecard** (snapshot writer + diff vs last baseline) mirroring `bin/cloaktest` ergonomics. +5. **Docs**: a short runbook (how to run, read the scorecard, interpret the ceiling trigger). + +## Risks / trade-offs + +- **Flaky/rate-limited/changing sites.** Detection sites move DOM and tighten over time; the harness must tolerate parse misses (report `ERROR`, not crash — `stealth_test.py:298–300`) and is **manual** precisely so flakiness never blocks development. +- **Proxy required for realism.** Without residential egress the hard rows fail by design (datacenter IP); document this so a red scorecard isn't misread. +- **ToS.** These are public detection/sandbox sites intended for this purpose; keep volume low and don't hammer. +- **Not a guarantee.** Passing sannysoft/CreepJS ≠ beating DataDome/Kasada; the scorecard's value is *trend + ceiling visibility*, not a green checkmark. + +## Out of scope (hand-offs) + +- **CI automation** — deferred (resolved: manual-only now). If revisited, it needs tolerant thresholds + proxy creds + nightly cadence; the scorecard JSON is designed to make that easy later. +- The levers themselves (`03e`), captcha solving (`03d`), proxy provider (`03b`), billing (`03c`) — this plan only **measures** them. +- The deferred paid-unblocker integration (`03e §8`) — the harness defines its *trigger*, not its build.