From f4d197f7022b0ceec560925f38fe6c125a77b0fc Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 2 Apr 2026 13:18:20 +0200 Subject: [PATCH 01/33] feat: add native module support for desktop autocomplete --- surfsense_desktop/.npmrc | 1 + surfsense_desktop/electron-builder.yml | 13 +++++ surfsense_desktop/package.json | 8 +++- surfsense_desktop/pnpm-lock.yaml | 50 ++++++++++++++++++++ surfsense_desktop/scripts/build-electron.mjs | 2 +- 5 files changed, 71 insertions(+), 3 deletions(-) create mode 100644 surfsense_desktop/.npmrc diff --git a/surfsense_desktop/.npmrc b/surfsense_desktop/.npmrc new file mode 100644 index 000000000..d67f37488 --- /dev/null +++ b/surfsense_desktop/.npmrc @@ -0,0 +1 @@ +node-linker=hoisted diff --git a/surfsense_desktop/electron-builder.yml b/surfsense_desktop/electron-builder.yml index eaca0f19b..74c69d223 100644 --- a/surfsense_desktop/electron-builder.yml +++ b/surfsense_desktop/electron-builder.yml @@ -9,6 +9,16 @@ directories: files: - dist/**/* - "!node_modules" + - node_modules/uiohook-napi/**/* + - "!node_modules/uiohook-napi/build" + - "!node_modules/uiohook-napi/src" + - "!node_modules/uiohook-napi/libuiohook" + - "!node_modules/uiohook-napi/binding.gyp" + - node_modules/node-gyp-build/**/* + - node_modules/node-mac-permissions/**/* + - "!node_modules/node-mac-permissions/build" + - "!node_modules/node-mac-permissions/src" + - "!node_modules/node-mac-permissions/binding.gyp" - "!src" - "!scripts" - "!release" @@ -29,6 +39,9 @@ extraResources: filter: ["**/*"] asarUnpack: - "**/*.node" + - "node_modules/uiohook-napi/**/*" + - "node_modules/node-gyp-build/**/*" + - "node_modules/node-mac-permissions/**/*" mac: icon: assets/icon.icns category: public.app-category.productivity diff --git a/surfsense_desktop/package.json b/surfsense_desktop/package.json index bd0cc67ab..a2e452b7c 100644 --- a/surfsense_desktop/package.json +++ b/surfsense_desktop/package.json @@ -11,12 +11,14 @@ "dist:mac": "pnpm build && electron-builder --mac --config electron-builder.yml", "dist:win": "pnpm build && electron-builder --win --config electron-builder.yml", "dist:linux": "pnpm build && electron-builder --linux --config electron-builder.yml", - "typecheck": "tsc --noEmit" + "typecheck": "tsc --noEmit", + "postinstall": "electron-rebuild" }, "author": "MODSetter", "license": "MIT", "packageManager": "pnpm@10.24.0", "devDependencies": { + "@electron/rebuild": "^4.0.3", "@types/node": "^25.5.0", "concurrently": "^9.2.1", "dotenv": "^17.3.1", @@ -28,6 +30,8 @@ }, "dependencies": { "electron-updater": "^6.8.3", - "get-port-please": "^3.2.0" + "get-port-please": "^3.2.0", + "node-mac-permissions": "^2.5.0", + "uiohook-napi": "^1.5.5" } } diff --git a/surfsense_desktop/pnpm-lock.yaml b/surfsense_desktop/pnpm-lock.yaml index ea65be0bb..82bad9456 100644 --- a/surfsense_desktop/pnpm-lock.yaml +++ b/surfsense_desktop/pnpm-lock.yaml @@ -14,7 +14,16 @@ importers: get-port-please: specifier: ^3.2.0 version: 3.2.0 + node-mac-permissions: + specifier: ^2.5.0 + version: 2.5.0 + uiohook-napi: + specifier: ^1.5.5 + version: 1.5.5 devDependencies: + '@electron/rebuild': + specifier: ^4.0.3 + version: 4.0.3 '@types/node': specifier: ^25.5.0 version: 25.5.0 @@ -343,6 +352,7 @@ packages: '@xmldom/xmldom@0.8.11': resolution: {integrity: sha512-cQzWCtO6C8TQiYl1ruKNn2U6Ao4o4WBBcbL61yJl84x+j5sOWWFU9X7DpND8XZG3daDppSsigMdfAIl2upQBRw==} engines: {node: '>=10.0.0'} + deprecated: this version has critical issues, please update to the latest version abbrev@3.0.1: resolution: {integrity: sha512-AO2ac6pjRB3SJmGJo+v5/aK6Omggp6fsLrs6wN9bd35ulu4cCwaAU9+7ZhXjeqHVkaHThLuzH0nZr0YpCDhygg==} @@ -424,6 +434,9 @@ packages: base64-js@1.5.1: resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} + bindings@1.5.0: + resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==} + bl@4.1.0: resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} @@ -738,6 +751,9 @@ packages: picomatch: optional: true + file-uri-to-path@1.0.0: + resolution: {integrity: sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==} + filelist@1.0.6: resolution: {integrity: sha512-5giy2PkLYY1cP39p17Ech+2xlpTRL9HLspOfEgm0L6CwBXBTgsK5ou0JtzYuepxkaQ/tvhCFIJ5uXo0OrM2DxA==} @@ -1103,14 +1119,25 @@ packages: node-addon-api@1.7.2: resolution: {integrity: sha512-ibPK3iA+vaY1eEjESkQkM0BbCqFOaZMiXRTtdB0u7b4djtY6JnsjvPdUHVMg6xQt3B8fpTTWHI9A+ADjM9frzg==} + node-addon-api@7.1.1: + resolution: {integrity: sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==} + node-api-version@0.2.1: resolution: {integrity: sha512-2xP/IGGMmmSQpI1+O/k72jF/ykvZ89JeuKX3TLJAYPDVLUalrshrLHkeVcCCZqG/eEa635cr8IBYzgnDvM2O8Q==} + node-gyp-build@4.8.4: + resolution: {integrity: sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==} + hasBin: true + node-gyp@11.5.0: resolution: {integrity: sha512-ra7Kvlhxn5V9Slyus0ygMa2h+UqExPqUIkfk7Pc8QTLT956JLSy51uWFwHtIYy0vI8cB4BDhc/S03+880My/LQ==} engines: {node: ^18.17.0 || >=20.5.0} hasBin: true + node-mac-permissions@2.5.0: + resolution: {integrity: sha512-zR8SVCaN3WqV1xwWd04XVAdzm3UTdjbxciLrZtB0Cc7F2Kd34AJfhPD4hm1HU0YH3oGUZO4X9OBLY5ijSTHsGw==} + os: [darwin] + nopt@8.1.0: resolution: {integrity: sha512-ieGu42u/Qsa4TFktmaKEwM6MQH0pOWnaB3htzh0JRtx84+Mebc0cbZYN5bC+6WTZ4+77xrL9Pn5m7CV6VIkV7A==} engines: {node: ^18.17.0 || >=20.5.0} @@ -1424,6 +1451,10 @@ packages: engines: {node: '>=14.17'} hasBin: true + uiohook-napi@1.5.5: + resolution: {integrity: sha512-oSlTdnECw2GBfsJPTbBQBeE4v/EXP0EZmX6BJq5nzH/JgFaBE8JpFwEA/kLhiEP7HxQw28FViWiYgdIZzWuuJQ==} + engines: {node: '>= 16'} + undici-types@7.16.0: resolution: {integrity: sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==} @@ -1923,6 +1954,10 @@ snapshots: base64-js@1.5.1: {} + bindings@1.5.0: + dependencies: + file-uri-to-path: 1.0.0 + bl@4.1.0: dependencies: buffer: 5.7.1 @@ -2348,6 +2383,8 @@ snapshots: optionalDependencies: picomatch: 4.0.3 + file-uri-to-path@1.0.0: {} + filelist@1.0.6: dependencies: minimatch: 5.1.9 @@ -2739,10 +2776,14 @@ snapshots: node-addon-api@1.7.2: optional: true + node-addon-api@7.1.1: {} + node-api-version@0.2.1: dependencies: semver: 7.7.4 + node-gyp-build@4.8.4: {} + node-gyp@11.5.0: dependencies: env-paths: 2.2.1 @@ -2758,6 +2799,11 @@ snapshots: transitivePeerDependencies: - supports-color + node-mac-permissions@2.5.0: + dependencies: + bindings: 1.5.0 + node-addon-api: 7.1.1 + nopt@8.1.0: dependencies: abbrev: 3.0.1 @@ -3064,6 +3110,10 @@ snapshots: typescript@5.9.3: {} + uiohook-napi@1.5.5: + dependencies: + node-gyp-build: 4.8.4 + undici-types@7.16.0: {} undici-types@7.18.2: {} diff --git a/surfsense_desktop/scripts/build-electron.mjs b/surfsense_desktop/scripts/build-electron.mjs index 923830296..83d941dd2 100644 --- a/surfsense_desktop/scripts/build-electron.mjs +++ b/surfsense_desktop/scripts/build-electron.mjs @@ -104,7 +104,7 @@ async function buildElectron() { bundle: true, platform: 'node', target: 'node18', - external: ['electron'], + external: ['electron', 'uiohook-napi', 'node-mac-permissions'], sourcemap: true, minify: false, define: { From fbd033d0a4ed756a789879b1bba69c08bc71506d Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 2 Apr 2026 13:19:21 +0200 Subject: [PATCH 02/33] feat: add autocomplete streaming endpoint with KB context --- surfsense_backend/app/routes/__init__.py | 2 + .../app/routes/autocomplete_routes.py | 136 ++++++++++++++++++ 2 files changed, 138 insertions(+) create mode 100644 surfsense_backend/app/routes/autocomplete_routes.py diff --git a/surfsense_backend/app/routes/__init__.py b/surfsense_backend/app/routes/__init__.py index 1937f11cb..a063b5976 100644 --- a/surfsense_backend/app/routes/__init__.py +++ b/surfsense_backend/app/routes/__init__.py @@ -3,6 +3,7 @@ from fastapi import APIRouter from .airtable_add_connector_route import ( router as airtable_add_connector_router, ) +from .autocomplete_routes import router as autocomplete_router from .chat_comments_routes import router as chat_comments_router from .circleback_webhook_route import router as circleback_webhook_router from .clickup_add_connector_route import router as clickup_add_connector_router @@ -95,3 +96,4 @@ router.include_router(incentive_tasks_router) # Incentive tasks for earning fre router.include_router(stripe_router) # Stripe checkout for additional page packs router.include_router(youtube_router) # YouTube playlist resolution router.include_router(prompts_router) +router.include_router(autocomplete_router) # Lightweight autocomplete with KB context diff --git a/surfsense_backend/app/routes/autocomplete_routes.py b/surfsense_backend/app/routes/autocomplete_routes.py new file mode 100644 index 000000000..9a285a723 --- /dev/null +++ b/surfsense_backend/app/routes/autocomplete_routes.py @@ -0,0 +1,136 @@ +import logging +from typing import AsyncGenerator + +from fastapi import APIRouter, Depends, Query +from fastapi.responses import StreamingResponse +from langchain_core.messages import HumanMessage, SystemMessage +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db import User, get_async_session +from app.retriever.chunks_hybrid_search import ChucksHybridSearchRetriever +from app.services.llm_service import get_agent_llm +from app.services.new_streaming_service import VercelStreamingService +from app.users import current_active_user + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/autocomplete", tags=["autocomplete"]) + +AUTOCOMPLETE_SYSTEM_PROMPT = """You are an inline text autocomplete engine. Your job is to complete the user's text naturally. + +Rules: +- Output ONLY the continuation text. Do NOT repeat what the user already typed. +- Keep completions concise: 1-3 sentences maximum. +- Match the user's tone, style, and language. +- If knowledge base context is provided, use it to make the completion factually accurate and personalized. +- Do NOT add quotes, explanations, or meta-commentary. +- Do NOT start with a space unless grammatically required. +- If you cannot produce a useful completion, output nothing.""" + +KB_CONTEXT_TEMPLATE = """ +Relevant knowledge base context (use this to personalize the completion): +--- +{kb_context} +--- +""" + + +async def _stream_autocomplete( + text: str, + cursor_position: int, + search_space_id: int, + session: AsyncSession, +) -> AsyncGenerator[str, None]: + """Stream an autocomplete response with KB context.""" + streaming_service = VercelStreamingService() + + try: + # Text before cursor is what we're completing + text_before_cursor = text[:cursor_position] if cursor_position >= 0 else text + + if not text_before_cursor.strip(): + yield streaming_service.format_message_start() + yield streaming_service.format_finish() + yield streaming_service.format_done() + return + + # Fast KB lookup: vector-only search, top 3 chunks, no planner LLM + kb_context = "" + try: + retriever = ChucksHybridSearchRetriever(session) + chunks = await retriever.vector_search( + query_text=text_before_cursor[-200:], # last 200 chars for relevance + top_k=3, + search_space_id=search_space_id, + ) + if chunks: + kb_snippets = [] + for chunk in chunks: + content = getattr(chunk, "content", None) or getattr(chunk, "chunk_text", "") + if content: + kb_snippets.append(content[:300]) + if kb_snippets: + kb_context = KB_CONTEXT_TEMPLATE.format( + kb_context="\n\n".join(kb_snippets) + ) + except Exception as e: + logger.warning(f"KB search failed for autocomplete, proceeding without context: {e}") + + # Get the search space's configured LLM + llm = await get_agent_llm(session, search_space_id) + if not llm: + yield streaming_service.format_message_start() + error_msg = "No LLM configured for this search space" + yield streaming_service.format_error(error_msg) + yield streaming_service.format_done() + return + + system_prompt = AUTOCOMPLETE_SYSTEM_PROMPT + if kb_context: + system_prompt += kb_context + + messages = [ + SystemMessage(content=system_prompt), + HumanMessage(content=f"Complete this text:\n{text_before_cursor}"), + ] + + # Stream the response + yield streaming_service.format_message_start() + text_id = streaming_service.generate_text_id() + yield streaming_service.format_text_start(text_id) + + async for chunk in llm.astream(messages): + token = chunk.content if hasattr(chunk, "content") else str(chunk) + if token: + yield streaming_service.format_text_delta(text_id, token) + + yield streaming_service.format_text_end(text_id) + yield streaming_service.format_finish() + yield streaming_service.format_done() + + except Exception as e: + logger.error(f"Autocomplete streaming error: {e}") + yield streaming_service.format_error(str(e)) + yield streaming_service.format_done() + + +@router.post("/stream") +async def autocomplete_stream( + text: str = Query(..., description="Current text in the input field"), + cursor_position: int = Query(-1, description="Cursor position in the text (-1 for end)"), + search_space_id: int = Query(..., description="Search space ID for KB context and LLM config"), + user: User = Depends(current_active_user), + session: AsyncSession = Depends(get_async_session), +): + """Stream an autocomplete suggestion based on the current text and KB context.""" + if cursor_position < 0: + cursor_position = len(text) + + return StreamingResponse( + _stream_autocomplete(text, cursor_position, search_space_id, session), + media_type="text/event-stream", + headers={ + **VercelStreamingService.get_response_headers(), + "X-Accel-Buffering": "no", + }, + ) From bcc227a4ddc34fbd6af3fb1115e01af0997bac85 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 2 Apr 2026 13:19:59 +0200 Subject: [PATCH 03/33] feat: add suggestion tooltip UI and autocomplete API types --- surfsense_web/app/suggestion/layout.tsx | 13 ++ surfsense_web/app/suggestion/page.tsx | 160 ++++++++++++++++++++ surfsense_web/app/suggestion/suggestion.css | 96 ++++++++++++ surfsense_web/types/window.d.ts | 6 + 4 files changed, 275 insertions(+) create mode 100644 surfsense_web/app/suggestion/layout.tsx create mode 100644 surfsense_web/app/suggestion/page.tsx create mode 100644 surfsense_web/app/suggestion/suggestion.css diff --git a/surfsense_web/app/suggestion/layout.tsx b/surfsense_web/app/suggestion/layout.tsx new file mode 100644 index 000000000..36b7e037b --- /dev/null +++ b/surfsense_web/app/suggestion/layout.tsx @@ -0,0 +1,13 @@ +import "./suggestion.css"; + +export const metadata = { + title: "SurfSense Suggestion", +}; + +export default function SuggestionLayout({ + children, +}: { + children: React.ReactNode; +}) { + return
{children}
; +} diff --git a/surfsense_web/app/suggestion/page.tsx b/surfsense_web/app/suggestion/page.tsx new file mode 100644 index 000000000..14dfab3af --- /dev/null +++ b/surfsense_web/app/suggestion/page.tsx @@ -0,0 +1,160 @@ +"use client"; + +import { useCallback, useEffect, useRef, useState } from "react"; +import { getBearerToken } from "@/lib/auth-utils"; + +type SSEEvent = + | { type: "text-delta"; id: string; delta: string } + | { type: "text-start"; id: string } + | { type: "text-end"; id: string } + | { type: "start"; messageId: string } + | { type: "finish" } + | { type: "error"; errorText: string }; + +export default function SuggestionPage() { + const [suggestion, setSuggestion] = useState(""); + const [isLoading, setIsLoading] = useState(true); + const [error, setError] = useState(null); + const abortRef = useRef(null); + + const fetchSuggestion = useCallback( + async (text: string, cursorPosition: number, searchSpaceId: string) => { + abortRef.current?.abort(); + const controller = new AbortController(); + abortRef.current = controller; + + setIsLoading(true); + setSuggestion(""); + setError(null); + + const token = getBearerToken(); + if (!token) { + setError("Not authenticated"); + setIsLoading(false); + return; + } + + const backendUrl = + process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL || "http://localhost:8000"; + + const params = new URLSearchParams({ + text, + cursor_position: String(cursorPosition), + search_space_id: searchSpaceId, + }); + + try { + const response = await fetch( + `${backendUrl}/api/v1/autocomplete/stream?${params}`, + { + method: "POST", + headers: { + Authorization: `Bearer ${token}`, + "Content-Type": "application/json", + }, + signal: controller.signal, + }, + ); + + if (!response.ok) { + setError(`Error: ${response.status}`); + setIsLoading(false); + return; + } + + if (!response.body) { + setError("No response body"); + setIsLoading(false); + return; + } + + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ""; + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + buffer += decoder.decode(value, { stream: true }); + const events = buffer.split(/\r?\n\r?\n/); + buffer = events.pop() || ""; + + for (const event of events) { + const lines = event.split(/\r?\n/); + for (const line of lines) { + if (!line.startsWith("data: ")) continue; + const data = line.slice(6).trim(); + if (!data || data === "[DONE]") continue; + + try { + const parsed: SSEEvent = JSON.parse(data); + if (parsed.type === "text-delta") { + setSuggestion((prev) => { + const updated = prev + parsed.delta; + window.electronAPI?.updateSuggestionText?.(updated); + return updated; + }); + } else if (parsed.type === "error") { + setError(parsed.errorText); + } + } catch { + continue; + } + } + } + } + } catch (err) { + if (err instanceof DOMException && err.name === "AbortError") return; + setError("Failed to get suggestion"); + } finally { + setIsLoading(false); + } + }, + [], + ); + + useEffect(() => { + if (!window.electronAPI?.onAutocompleteContext) return; + + const cleanup = window.electronAPI.onAutocompleteContext((data) => { + const searchSpaceId = data.searchSpaceId || "1"; + fetchSuggestion(data.text, data.cursorPosition, searchSpaceId); + }); + + return cleanup; + }, [fetchSuggestion]); + + if (error) { + return ( +
+ {error} +
+ ); + } + + if (isLoading && !suggestion) { + return ( +
+
+ + + +
+
+ ); + } + + if (!suggestion) return null; + + return ( +
+

{suggestion}

+
+ Tab accept + · + Esc dismiss +
+
+ ); +} diff --git a/surfsense_web/app/suggestion/suggestion.css b/surfsense_web/app/suggestion/suggestion.css new file mode 100644 index 000000000..e9471e7f8 --- /dev/null +++ b/surfsense_web/app/suggestion/suggestion.css @@ -0,0 +1,96 @@ +.suggestion-body { + margin: 0; + padding: 0; + background: transparent; + overflow: hidden; + font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; + -webkit-font-smoothing: antialiased; + user-select: none; + -webkit-app-region: no-drag; +} + +.suggestion-tooltip { + background: rgba(30, 30, 30, 0.95); + backdrop-filter: blur(12px); + -webkit-backdrop-filter: blur(12px); + border: 1px solid rgba(255, 255, 255, 0.1); + border-radius: 10px; + padding: 10px 14px; + margin: 4px; + max-width: 400px; + box-shadow: 0 8px 32px rgba(0, 0, 0, 0.4), + 0 2px 8px rgba(0, 0, 0, 0.2); +} + +.suggestion-text { + color: rgba(255, 255, 255, 0.9); + font-size: 13px; + line-height: 1.5; + margin: 0 0 8px 0; + word-wrap: break-word; + white-space: pre-wrap; +} + +.suggestion-hint { + color: rgba(255, 255, 255, 0.4); + font-size: 11px; + display: flex; + align-items: center; + gap: 4px; +} + +.suggestion-key { + background: rgba(255, 255, 255, 0.1); + border: 1px solid rgba(255, 255, 255, 0.15); + border-radius: 3px; + padding: 1px 5px; + font-size: 10px; + font-weight: 500; + color: rgba(255, 255, 255, 0.6); +} + +.suggestion-separator { + margin: 0 2px; +} + +.suggestion-error { + border-color: rgba(255, 80, 80, 0.3); +} + +.suggestion-error-text { + color: rgba(255, 120, 120, 0.9); + font-size: 12px; +} + +.suggestion-loading { + display: flex; + gap: 4px; + padding: 4px 0; +} + +.suggestion-dot { + width: 5px; + height: 5px; + border-radius: 50%; + background: rgba(255, 255, 255, 0.4); + animation: suggestion-pulse 1.2s infinite ease-in-out; +} + +.suggestion-dot:nth-child(2) { + animation-delay: 0.15s; +} + +.suggestion-dot:nth-child(3) { + animation-delay: 0.3s; +} + +@keyframes suggestion-pulse { + 0%, 80%, 100% { + opacity: 0.3; + transform: scale(0.8); + } + 40% { + opacity: 1; + transform: scale(1); + } +} diff --git a/surfsense_web/types/window.d.ts b/surfsense_web/types/window.d.ts index 9cf1aa596..a30358527 100644 --- a/surfsense_web/types/window.d.ts +++ b/surfsense_web/types/window.d.ts @@ -14,6 +14,12 @@ interface ElectronAPI { setQuickAskMode: (mode: string) => Promise; getQuickAskMode: () => Promise; replaceText: (text: string) => Promise; + onAutocompleteContext: (callback: (data: { text: string; cursorPosition: number; searchSpaceId?: string }) => void) => () => void; + acceptSuggestion: (text: string) => Promise; + dismissSuggestion: () => Promise; + updateSuggestionText: (text: string) => Promise; + setAutocompleteEnabled: (enabled: boolean) => Promise; + getAutocompleteEnabled: () => Promise; } declare global { From ec2b7851b6393147b09a9f24273fe84d4314371f Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 2 Apr 2026 13:26:32 +0200 Subject: [PATCH 04/33] feat: add macOS permission infrastructure for autocomplete --- surfsense_desktop/src/ipc/channels.ts | 5 ++ surfsense_desktop/src/ipc/handlers.ts | 22 +++++++++ surfsense_desktop/src/modules/permissions.ts | 50 ++++++++++++++++++++ surfsense_desktop/src/preload.ts | 5 ++ surfsense_web/types/window.d.ts | 9 ++++ 5 files changed, 91 insertions(+) create mode 100644 surfsense_desktop/src/modules/permissions.ts diff --git a/surfsense_desktop/src/ipc/channels.ts b/surfsense_desktop/src/ipc/channels.ts index 25ec1bc0e..a5209dcf3 100644 --- a/surfsense_desktop/src/ipc/channels.ts +++ b/surfsense_desktop/src/ipc/channels.ts @@ -6,4 +6,9 @@ export const IPC_CHANNELS = { SET_QUICK_ASK_MODE: 'set-quick-ask-mode', GET_QUICK_ASK_MODE: 'get-quick-ask-mode', REPLACE_TEXT: 'replace-text', + // Permissions + GET_PERMISSIONS_STATUS: 'get-permissions-status', + REQUEST_ACCESSIBILITY: 'request-accessibility', + REQUEST_INPUT_MONITORING: 'request-input-monitoring', + RESTART_APP: 'restart-app', } as const; diff --git a/surfsense_desktop/src/ipc/handlers.ts b/surfsense_desktop/src/ipc/handlers.ts index 18e343719..fc31329f1 100644 --- a/surfsense_desktop/src/ipc/handlers.ts +++ b/surfsense_desktop/src/ipc/handlers.ts @@ -1,5 +1,11 @@ import { app, ipcMain, shell } from 'electron'; import { IPC_CHANNELS } from './channels'; +import { + getPermissionsStatus, + requestAccessibility, + requestInputMonitoring, + restartApp, +} from '../modules/permissions'; export function registerIpcHandlers(): void { ipcMain.on(IPC_CHANNELS.OPEN_EXTERNAL, (_event, url: string) => { @@ -16,4 +22,20 @@ export function registerIpcHandlers(): void { ipcMain.handle(IPC_CHANNELS.GET_APP_VERSION, () => { return app.getVersion(); }); + + ipcMain.handle(IPC_CHANNELS.GET_PERMISSIONS_STATUS, () => { + return getPermissionsStatus(); + }); + + ipcMain.handle(IPC_CHANNELS.REQUEST_ACCESSIBILITY, () => { + requestAccessibility(); + }); + + ipcMain.handle(IPC_CHANNELS.REQUEST_INPUT_MONITORING, () => { + requestInputMonitoring(); + }); + + ipcMain.handle(IPC_CHANNELS.RESTART_APP, () => { + restartApp(); + }); } diff --git a/surfsense_desktop/src/modules/permissions.ts b/surfsense_desktop/src/modules/permissions.ts new file mode 100644 index 000000000..9a6159c9a --- /dev/null +++ b/surfsense_desktop/src/modules/permissions.ts @@ -0,0 +1,50 @@ +import { app } from 'electron'; + +type PermissionStatus = 'authorized' | 'denied' | 'not determined' | 'restricted' | 'limited'; + +export interface PermissionsStatus { + accessibility: PermissionStatus; + inputMonitoring: PermissionStatus; +} + +function isMac(): boolean { + return process.platform === 'darwin'; +} + +function getNodeMacPermissions() { + return require('node-mac-permissions'); +} + +export function getPermissionsStatus(): PermissionsStatus { + if (!isMac()) { + return { accessibility: 'authorized', inputMonitoring: 'authorized' }; + } + + const perms = getNodeMacPermissions(); + return { + accessibility: perms.getAuthStatus('accessibility'), + inputMonitoring: perms.getAuthStatus('input-monitoring'), + }; +} + +export function allPermissionsGranted(): boolean { + const status = getPermissionsStatus(); + return status.accessibility === 'authorized' && status.inputMonitoring === 'authorized'; +} + +export function requestAccessibility(): void { + if (!isMac()) return; + const perms = getNodeMacPermissions(); + perms.askForAccessibilityAccess(); +} + +export async function requestInputMonitoring(): Promise { + if (!isMac()) return 'authorized'; + const perms = getNodeMacPermissions(); + return perms.askForInputMonitoringAccess('listen'); +} + +export function restartApp(): void { + app.relaunch(); + app.exit(0); +} diff --git a/surfsense_desktop/src/preload.ts b/surfsense_desktop/src/preload.ts index 264ec25b3..069276489 100644 --- a/surfsense_desktop/src/preload.ts +++ b/surfsense_desktop/src/preload.ts @@ -21,4 +21,9 @@ contextBridge.exposeInMainWorld('electronAPI', { setQuickAskMode: (mode: string) => ipcRenderer.invoke(IPC_CHANNELS.SET_QUICK_ASK_MODE, mode), getQuickAskMode: () => ipcRenderer.invoke(IPC_CHANNELS.GET_QUICK_ASK_MODE), replaceText: (text: string) => ipcRenderer.invoke(IPC_CHANNELS.REPLACE_TEXT, text), + // Permissions + getPermissionsStatus: () => ipcRenderer.invoke(IPC_CHANNELS.GET_PERMISSIONS_STATUS), + requestAccessibility: () => ipcRenderer.invoke(IPC_CHANNELS.REQUEST_ACCESSIBILITY), + requestInputMonitoring: () => ipcRenderer.invoke(IPC_CHANNELS.REQUEST_INPUT_MONITORING), + restartApp: () => ipcRenderer.invoke(IPC_CHANNELS.RESTART_APP), }); diff --git a/surfsense_web/types/window.d.ts b/surfsense_web/types/window.d.ts index a30358527..8cf331b42 100644 --- a/surfsense_web/types/window.d.ts +++ b/surfsense_web/types/window.d.ts @@ -14,6 +14,15 @@ interface ElectronAPI { setQuickAskMode: (mode: string) => Promise; getQuickAskMode: () => Promise; replaceText: (text: string) => Promise; + // Permissions + getPermissionsStatus: () => Promise<{ + accessibility: 'authorized' | 'denied' | 'not determined' | 'restricted' | 'limited'; + inputMonitoring: 'authorized' | 'denied' | 'not determined' | 'restricted' | 'limited'; + }>; + requestAccessibility: () => Promise; + requestInputMonitoring: () => Promise; + restartApp: () => Promise; + // Autocomplete onAutocompleteContext: (callback: (data: { text: string; cursorPosition: number; searchSpaceId?: string }) => void) => () => void; acceptSuggestion: (text: string) => Promise; dismissSuggestion: () => Promise; From eaabad38fcd2bfc8d6bef87f8ea60ea4d8192d78 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 2 Apr 2026 13:44:57 +0200 Subject: [PATCH 05/33] feat: add permission onboarding page and startup routing for macOS --- surfsense_desktop/src/ipc/handlers.ts | 4 +- surfsense_desktop/src/main.ts | 15 +- surfsense_desktop/src/modules/window.ts | 4 +- .../app/desktop/permissions/page.tsx | 212 ++++++++++++++++++ .../app/{ => desktop}/suggestion/layout.tsx | 0 .../app/{ => desktop}/suggestion/page.tsx | 0 .../{ => desktop}/suggestion/suggestion.css | 0 7 files changed, 228 insertions(+), 7 deletions(-) create mode 100644 surfsense_web/app/desktop/permissions/page.tsx rename surfsense_web/app/{ => desktop}/suggestion/layout.tsx (100%) rename surfsense_web/app/{ => desktop}/suggestion/page.tsx (100%) rename surfsense_web/app/{ => desktop}/suggestion/suggestion.css (100%) diff --git a/surfsense_desktop/src/ipc/handlers.ts b/surfsense_desktop/src/ipc/handlers.ts index fc31329f1..a6d82be4b 100644 --- a/surfsense_desktop/src/ipc/handlers.ts +++ b/surfsense_desktop/src/ipc/handlers.ts @@ -31,8 +31,8 @@ export function registerIpcHandlers(): void { requestAccessibility(); }); - ipcMain.handle(IPC_CHANNELS.REQUEST_INPUT_MONITORING, () => { - requestInputMonitoring(); + ipcMain.handle(IPC_CHANNELS.REQUEST_INPUT_MONITORING, async () => { + return await requestInputMonitoring(); }); ipcMain.handle(IPC_CHANNELS.RESTART_APP, () => { diff --git a/surfsense_desktop/src/main.ts b/surfsense_desktop/src/main.ts index 3ab41073b..bc164758b 100644 --- a/surfsense_desktop/src/main.ts +++ b/surfsense_desktop/src/main.ts @@ -7,6 +7,7 @@ import { setupAutoUpdater } from './modules/auto-updater'; import { setupMenu } from './modules/menu'; import { registerQuickAsk, unregisterQuickAsk } from './modules/quick-ask'; import { registerIpcHandlers } from './ipc/handlers'; +import { allPermissionsGranted } from './modules/permissions'; registerGlobalErrorHandlers(); @@ -16,7 +17,13 @@ if (!setupDeepLinks()) { registerIpcHandlers(); -// App lifecycle +function getInitialPath(): string { + if (process.platform === 'darwin' && !allPermissionsGranted()) { + return '/desktop/permissions'; + } + return '/dashboard'; +} + app.whenReady().then(async () => { setupMenu(); try { @@ -26,7 +33,9 @@ app.whenReady().then(async () => { setTimeout(() => app.quit(), 0); return; } - createMainWindow(); + + const initialPath = getInitialPath(); + createMainWindow(initialPath); registerQuickAsk(); setupAutoUpdater(); @@ -34,7 +43,7 @@ app.whenReady().then(async () => { app.on('activate', () => { if (BrowserWindow.getAllWindows().length === 0) { - createMainWindow(); + createMainWindow(getInitialPath()); } }); }); diff --git a/surfsense_desktop/src/modules/window.ts b/surfsense_desktop/src/modules/window.ts index 245814cad..7a77773d8 100644 --- a/surfsense_desktop/src/modules/window.ts +++ b/surfsense_desktop/src/modules/window.ts @@ -12,7 +12,7 @@ export function getMainWindow(): BrowserWindow | null { return mainWindow; } -export function createMainWindow(): BrowserWindow { +export function createMainWindow(initialPath = '/dashboard'): BrowserWindow { mainWindow = new BrowserWindow({ width: 1280, height: 800, @@ -33,7 +33,7 @@ export function createMainWindow(): BrowserWindow { mainWindow?.show(); }); - mainWindow.loadURL(`http://localhost:${getServerPort()}/dashboard`); + mainWindow.loadURL(`http://localhost:${getServerPort()}${initialPath}`); mainWindow.webContents.setWindowOpenHandler(({ url }) => { if (url.startsWith('http://localhost')) { diff --git a/surfsense_web/app/desktop/permissions/page.tsx b/surfsense_web/app/desktop/permissions/page.tsx new file mode 100644 index 000000000..2bcdc42df --- /dev/null +++ b/surfsense_web/app/desktop/permissions/page.tsx @@ -0,0 +1,212 @@ +"use client"; + +import { useEffect, useState } from "react"; +import { useRouter } from "next/navigation"; +import { Logo } from "@/components/Logo"; +import { Button } from "@/components/ui/button"; +import { Spinner } from "@/components/ui/spinner"; + +type PermissionStatus = "authorized" | "denied" | "not determined" | "restricted" | "limited"; + +interface PermissionsStatus { + accessibility: PermissionStatus; + inputMonitoring: PermissionStatus; +} + +const STEPS = [ + { + id: "input-monitoring", + title: "Input Monitoring", + description: "Helps you write faster by enriching your text with suggestions from your knowledge base.", + action: "requestInputMonitoring", + field: "inputMonitoring" as const, + }, + { + id: "accessibility", + title: "Accessibility", + description: "Lets you accept suggestions seamlessly, right where you're typing.", + action: "requestAccessibility", + field: "accessibility" as const, + }, +]; + +function StatusBadge({ status }: { status: PermissionStatus }) { + if (status === "authorized") { + return ( + + + Granted + + ); + } + if (status === "denied") { + return ( + + + Denied + + ); + } + return ( + + + Pending + + ); +} + +export default function DesktopPermissionsPage() { + const router = useRouter(); + const [permissions, setPermissions] = useState(null); + const [isElectron, setIsElectron] = useState(false); + + useEffect(() => { + if (!window.electronAPI) return; + setIsElectron(true); + + let interval: ReturnType | null = null; + + const isResolved = (s: string) => s === "authorized" || s === "restricted"; + + const poll = async () => { + const status = await window.electronAPI!.getPermissionsStatus(); + setPermissions(status); + + if (isResolved(status.accessibility) && isResolved(status.inputMonitoring)) { + if (interval) clearInterval(interval); + } + }; + + poll(); + interval = setInterval(poll, 2000); + return () => { if (interval) clearInterval(interval); }; + }, []); + + if (!isElectron) { + return ( +
+

This page is only available in the desktop app.

+
+ ); + } + + if (!permissions) { + return ( +
+ +
+ ); + } + + const allGranted = permissions.accessibility === "authorized" && permissions.inputMonitoring === "authorized"; + + const handleRequest = async (action: string) => { + if (action === "requestInputMonitoring") { + await window.electronAPI!.requestInputMonitoring(); + } else if (action === "requestAccessibility") { + await window.electronAPI!.requestAccessibility(); + } + }; + + const handleContinue = () => { + if (allGranted) { + window.electronAPI!.restartApp(); + } + }; + + const handleSkip = () => { + router.push("/dashboard"); + }; + + return ( +
+
+ {/* Header */} +
+ +
+

System Permissions

+

+ SurfSense needs two macOS permissions to provide system-wide autocomplete. +

+
+
+ + {/* Steps */} +
+ {STEPS.map((step, index) => { + const status = permissions[step.field]; + const isGranted = status === "authorized"; + + return ( +
+
+
+ + {isGranted ? "✓" : index + 1} + +
+

{step.title}

+

{step.description}

+
+
+ +
+ {!isGranted && ( +
+ + {status === "denied" && ( +

+ Toggle SurfSense on in System Settings to continue. +

+ )} +
+ )} +
+ ); + })} +
+ + {/* Footer */} +
+ {allGranted ? ( + <> + +

+ A restart is needed for permissions to take effect. +

+ + ) : ( + <> + + + + )} +
+
+
+ ); +} diff --git a/surfsense_web/app/suggestion/layout.tsx b/surfsense_web/app/desktop/suggestion/layout.tsx similarity index 100% rename from surfsense_web/app/suggestion/layout.tsx rename to surfsense_web/app/desktop/suggestion/layout.tsx diff --git a/surfsense_web/app/suggestion/page.tsx b/surfsense_web/app/desktop/suggestion/page.tsx similarity index 100% rename from surfsense_web/app/suggestion/page.tsx rename to surfsense_web/app/desktop/suggestion/page.tsx diff --git a/surfsense_web/app/suggestion/suggestion.css b/surfsense_web/app/desktop/suggestion/suggestion.css similarity index 100% rename from surfsense_web/app/suggestion/suggestion.css rename to surfsense_web/app/desktop/suggestion/suggestion.css From b2706b00a1bf793e8d1aa63235a0c53d5cc6766c Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 2 Apr 2026 14:29:12 +0200 Subject: [PATCH 06/33] feat: add autocomplete module with keystroke monitoring and IPC wiring --- surfsense_desktop/src/ipc/channels.ts | 7 + surfsense_desktop/src/main.ts | 3 + surfsense_desktop/src/modules/autocomplete.ts | 267 ++++++++++++++++++ surfsense_desktop/src/modules/platform.ts | 40 +++ surfsense_desktop/src/preload.ts | 13 + 5 files changed, 330 insertions(+) create mode 100644 surfsense_desktop/src/modules/autocomplete.ts diff --git a/surfsense_desktop/src/ipc/channels.ts b/surfsense_desktop/src/ipc/channels.ts index a5209dcf3..2965f516f 100644 --- a/surfsense_desktop/src/ipc/channels.ts +++ b/surfsense_desktop/src/ipc/channels.ts @@ -11,4 +11,11 @@ export const IPC_CHANNELS = { REQUEST_ACCESSIBILITY: 'request-accessibility', REQUEST_INPUT_MONITORING: 'request-input-monitoring', RESTART_APP: 'restart-app', + // Autocomplete + AUTOCOMPLETE_CONTEXT: 'autocomplete-context', + ACCEPT_SUGGESTION: 'accept-suggestion', + DISMISS_SUGGESTION: 'dismiss-suggestion', + UPDATE_SUGGESTION_TEXT: 'update-suggestion-text', + SET_AUTOCOMPLETE_ENABLED: 'set-autocomplete-enabled', + GET_AUTOCOMPLETE_ENABLED: 'get-autocomplete-enabled', } as const; diff --git a/surfsense_desktop/src/main.ts b/surfsense_desktop/src/main.ts index bc164758b..9623be82e 100644 --- a/surfsense_desktop/src/main.ts +++ b/surfsense_desktop/src/main.ts @@ -6,6 +6,7 @@ import { setupDeepLinks, handlePendingDeepLink } from './modules/deep-links'; import { setupAutoUpdater } from './modules/auto-updater'; import { setupMenu } from './modules/menu'; import { registerQuickAsk, unregisterQuickAsk } from './modules/quick-ask'; +import { registerAutocomplete, unregisterAutocomplete } from './modules/autocomplete'; import { registerIpcHandlers } from './ipc/handlers'; import { allPermissionsGranted } from './modules/permissions'; @@ -37,6 +38,7 @@ app.whenReady().then(async () => { const initialPath = getInitialPath(); createMainWindow(initialPath); registerQuickAsk(); + registerAutocomplete(); setupAutoUpdater(); handlePendingDeepLink(); @@ -56,4 +58,5 @@ app.on('window-all-closed', () => { app.on('will-quit', () => { unregisterQuickAsk(); + unregisterAutocomplete(); }); diff --git a/surfsense_desktop/src/modules/autocomplete.ts b/surfsense_desktop/src/modules/autocomplete.ts new file mode 100644 index 000000000..2b877723f --- /dev/null +++ b/surfsense_desktop/src/modules/autocomplete.ts @@ -0,0 +1,267 @@ +import { BrowserWindow, clipboard, ipcMain, screen, shell } from 'electron'; +import path from 'path'; +import { IPC_CHANNELS } from '../ipc/channels'; +import { allPermissionsGranted } from './permissions'; +import { getFieldContent, getFrontmostApp, hasAccessibilityPermission, simulatePaste } from './platform'; +import { getServerPort } from './server'; +import { getMainWindow } from './window'; + +const DEBOUNCE_MS = 600; +const TOOLTIP_WIDTH = 420; +const TOOLTIP_HEIGHT = 140; + +let uIOhook: any = null; +let UiohookKey: any = {}; +let IGNORED_KEYCODES: Set = new Set(); + +let suggestionWindow: BrowserWindow | null = null; +let debounceTimer: ReturnType | null = null; +let hookStarted = false; +let autocompleteEnabled = true; +let savedClipboard = ''; +let sourceApp = ''; +let pendingSuggestionText = ''; + +function loadUiohook(): boolean { + if (uIOhook) return true; + try { + const mod = require('uiohook-napi'); + uIOhook = mod.uIOhook; + UiohookKey = mod.UiohookKey; + IGNORED_KEYCODES = new Set([ + UiohookKey.Shift, UiohookKey.ShiftRight, + UiohookKey.Ctrl, UiohookKey.CtrlRight, + UiohookKey.Alt, UiohookKey.AltRight, + UiohookKey.Meta, UiohookKey.MetaRight, + UiohookKey.CapsLock, UiohookKey.NumLock, UiohookKey.ScrollLock, + UiohookKey.F1, UiohookKey.F2, UiohookKey.F3, UiohookKey.F4, + UiohookKey.F5, UiohookKey.F6, UiohookKey.F7, UiohookKey.F8, + UiohookKey.F9, UiohookKey.F10, UiohookKey.F11, UiohookKey.F12, + UiohookKey.PrintScreen, + UiohookKey.Insert, UiohookKey.Delete, + UiohookKey.Home, UiohookKey.End, + UiohookKey.PageUp, UiohookKey.PageDown, + UiohookKey.ArrowUp, UiohookKey.ArrowDown, + UiohookKey.ArrowLeft, UiohookKey.ArrowRight, + ]); + console.log('[autocomplete] uiohook-napi loaded'); + return true; + } catch (err) { + console.error('[autocomplete] Failed to load uiohook-napi:', err); + return false; + } +} + +function destroySuggestion(): void { + if (suggestionWindow && !suggestionWindow.isDestroyed()) { + suggestionWindow.close(); + } + suggestionWindow = null; +} + +function clampToScreen(x: number, y: number, w: number, h: number): { x: number; y: number } { + const display = screen.getDisplayNearestPoint({ x, y }); + const { x: dx, y: dy, width: dw, height: dh } = display.workArea; + return { + x: Math.max(dx, Math.min(x, dx + dw - w)), + y: Math.max(dy, Math.min(y, dy + dh - h)), + }; +} + +function createSuggestionWindow(x: number, y: number): BrowserWindow { + destroySuggestion(); + + const pos = clampToScreen(x, y + 20, TOOLTIP_WIDTH, TOOLTIP_HEIGHT); + + suggestionWindow = new BrowserWindow({ + width: TOOLTIP_WIDTH, + height: TOOLTIP_HEIGHT, + x: pos.x, + y: pos.y, + frame: false, + transparent: true, + focusable: false, + alwaysOnTop: true, + skipTaskbar: true, + resizable: false, + hasShadow: true, + type: 'panel', + webPreferences: { + preload: path.join(__dirname, 'preload.js'), + contextIsolation: true, + nodeIntegration: false, + sandbox: true, + }, + show: false, + }); + + suggestionWindow.loadURL(`http://localhost:${getServerPort()}/desktop/suggestion?t=${Date.now()}`); + + suggestionWindow.once('ready-to-show', () => { + suggestionWindow?.showInactive(); + }); + + suggestionWindow.webContents.setWindowOpenHandler(({ url }) => { + if (url.startsWith('http://localhost')) { + return { action: 'allow' }; + } + shell.openExternal(url); + return { action: 'deny' }; + }); + + suggestionWindow.on('closed', () => { + suggestionWindow = null; + }); + + return suggestionWindow; +} + +function clearDebounce(): void { + if (debounceTimer) { + clearTimeout(debounceTimer); + debounceTimer = null; + } +} + +function isSurfSenseWindow(): boolean { + const app = getFrontmostApp(); + return app === 'Electron' || app === 'SurfSense' || app === 'surfsense-desktop'; +} + +function onKeyDown(event: { keycode: number; ctrlKey?: boolean; metaKey?: boolean; altKey?: boolean }): void { + if (!autocompleteEnabled) return; + + if (event.keycode === UiohookKey.Tab && suggestionWindow && !suggestionWindow.isDestroyed()) { + if (pendingSuggestionText) { + acceptAndInject(pendingSuggestionText); + } + return; + } + + if (event.keycode === UiohookKey.Escape) { + if (suggestionWindow && !suggestionWindow.isDestroyed()) { + destroySuggestion(); + pendingSuggestionText = ''; + } + clearDebounce(); + return; + } + + if (IGNORED_KEYCODES.has(event.keycode)) return; + if (event.ctrlKey || event.metaKey || event.altKey) return; + if (isSurfSenseWindow()) return; + + if (suggestionWindow && !suggestionWindow.isDestroyed()) { + destroySuggestion(); + } + + clearDebounce(); + debounceTimer = setTimeout(() => { + triggerAutocomplete(); + }, DEBOUNCE_MS); +} + +async function triggerAutocomplete(): Promise { + if (!hasAccessibilityPermission()) return; + if (isSurfSenseWindow()) return; + + const fieldContent = getFieldContent(); + if (!fieldContent || !fieldContent.text.trim()) return; + if (fieldContent.text.trim().length < 5) return; + + sourceApp = getFrontmostApp(); + savedClipboard = clipboard.readText(); + + const cursor = screen.getCursorScreenPoint(); + const win = createSuggestionWindow(cursor.x, cursor.y); + + let searchSpaceId = '1'; + const mainWin = getMainWindow(); + if (mainWin && !mainWin.isDestroyed()) { + const mainUrl = mainWin.webContents.getURL(); + const match = mainUrl.match(/\/dashboard\/(\d+)/); + if (match) { + searchSpaceId = match[1]; + } + } + + win.webContents.once('did-finish-load', () => { + if (suggestionWindow && !suggestionWindow.isDestroyed()) { + suggestionWindow.webContents.send(IPC_CHANNELS.AUTOCOMPLETE_CONTEXT, { + text: fieldContent.text, + cursorPosition: fieldContent.cursorPosition, + searchSpaceId, + }); + } + }); +} + +async function acceptAndInject(text: string): Promise { + if (!sourceApp) return; + if (!hasAccessibilityPermission()) return; + + clipboard.writeText(text); + destroySuggestion(); + pendingSuggestionText = ''; + + try { + await new Promise((r) => setTimeout(r, 50)); + simulatePaste(); + await new Promise((r) => setTimeout(r, 100)); + clipboard.writeText(savedClipboard); + } catch { + clipboard.writeText(savedClipboard); + } +} + +function registerIpcHandlers(): void { + ipcMain.handle(IPC_CHANNELS.ACCEPT_SUGGESTION, async (_event, text: string) => { + await acceptAndInject(text); + }); + ipcMain.handle(IPC_CHANNELS.DISMISS_SUGGESTION, () => { + destroySuggestion(); + pendingSuggestionText = ''; + }); + ipcMain.handle(IPC_CHANNELS.UPDATE_SUGGESTION_TEXT, (_event, text: string) => { + pendingSuggestionText = text; + }); + ipcMain.handle(IPC_CHANNELS.SET_AUTOCOMPLETE_ENABLED, (_event, enabled: boolean) => { + autocompleteEnabled = enabled; + if (!enabled) { + clearDebounce(); + destroySuggestion(); + } + }); + ipcMain.handle(IPC_CHANNELS.GET_AUTOCOMPLETE_ENABLED, () => autocompleteEnabled); +} + +export function registerAutocomplete(): void { + registerIpcHandlers(); + + if (!allPermissionsGranted()) { + console.log('[autocomplete] Permissions not granted — hook not started'); + return; + } + + if (!loadUiohook()) { + console.error('[autocomplete] Cannot start: uiohook-napi failed to load'); + return; + } + + uIOhook.on('keydown', onKeyDown); + try { + uIOhook.start(); + hookStarted = true; + console.log('[autocomplete] uIOhook started'); + } catch (err) { + console.error('[autocomplete] uIOhook.start() failed:', err); + } +} + +export function unregisterAutocomplete(): void { + clearDebounce(); + destroySuggestion(); + if (uIOhook && hookStarted) { + try { uIOhook.stop(); } catch { /* already stopped */ } + } +} diff --git a/surfsense_desktop/src/modules/platform.ts b/surfsense_desktop/src/modules/platform.ts index 37e126799..262866d07 100644 --- a/surfsense_desktop/src/modules/platform.ts +++ b/surfsense_desktop/src/modules/platform.ts @@ -53,3 +53,43 @@ export function checkAccessibilityPermission(): boolean { if (process.platform !== 'darwin') return true; return systemPreferences.isTrustedAccessibilityClient(true); } + +export function hasAccessibilityPermission(): boolean { + if (process.platform !== 'darwin') return true; + return systemPreferences.isTrustedAccessibilityClient(false); +} + +export interface FieldContent { + text: string; + cursorPosition: number; +} + +export function getFieldContent(): FieldContent | null { + if (process.platform !== 'darwin') return null; + + try { + const text = execSync( + 'osascript -e \'tell application "System Events" to get value of attribute "AXValue" of focused UI element of first application process whose frontmost is true\'', + { timeout: 500 } + ).toString().trim(); + + let cursorPosition = text.length; + try { + const rangeStr = execSync( + 'osascript -e \'tell application "System Events" to get value of attribute "AXSelectedTextRange" of focused UI element of first application process whose frontmost is true\'', + { timeout: 500 } + ).toString().trim(); + + const locationMatch = rangeStr.match(/location[:\s]*(\d+)/i); + if (locationMatch) { + cursorPosition = parseInt(locationMatch[1], 10); + } + } catch { + // Fall back to end of text + } + + return { text, cursorPosition }; + } catch { + return null; + } +} diff --git a/surfsense_desktop/src/preload.ts b/surfsense_desktop/src/preload.ts index 069276489..956afcc46 100644 --- a/surfsense_desktop/src/preload.ts +++ b/surfsense_desktop/src/preload.ts @@ -26,4 +26,17 @@ contextBridge.exposeInMainWorld('electronAPI', { requestAccessibility: () => ipcRenderer.invoke(IPC_CHANNELS.REQUEST_ACCESSIBILITY), requestInputMonitoring: () => ipcRenderer.invoke(IPC_CHANNELS.REQUEST_INPUT_MONITORING), restartApp: () => ipcRenderer.invoke(IPC_CHANNELS.RESTART_APP), + // Autocomplete + onAutocompleteContext: (callback: (data: { text: string; cursorPosition: number; searchSpaceId?: string }) => void) => { + const listener = (_event: unknown, data: { text: string; cursorPosition: number; searchSpaceId?: string }) => callback(data); + ipcRenderer.on(IPC_CHANNELS.AUTOCOMPLETE_CONTEXT, listener); + return () => { + ipcRenderer.removeListener(IPC_CHANNELS.AUTOCOMPLETE_CONTEXT, listener); + }; + }, + acceptSuggestion: (text: string) => ipcRenderer.invoke(IPC_CHANNELS.ACCEPT_SUGGESTION, text), + dismissSuggestion: () => ipcRenderer.invoke(IPC_CHANNELS.DISMISS_SUGGESTION), + updateSuggestionText: (text: string) => ipcRenderer.invoke(IPC_CHANNELS.UPDATE_SUGGESTION_TEXT, text), + setAutocompleteEnabled: (enabled: boolean) => ipcRenderer.invoke(IPC_CHANNELS.SET_AUTOCOMPLETE_ENABLED, enabled), + getAutocompleteEnabled: () => ipcRenderer.invoke(IPC_CHANNELS.GET_AUTOCOMPLETE_ENABLED), }); From 6899134a20605b619e372564666a25eba5bb76fa Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 2 Apr 2026 14:37:26 +0200 Subject: [PATCH 07/33] feat: add autocomplete toggle in desktop settings --- .../components/DesktopContent.tsx | 79 +++++++++++++++++++ .../settings/user-settings-dialog.tsx | 7 +- 2 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 surfsense_web/app/dashboard/[search_space_id]/user-settings/components/DesktopContent.tsx diff --git a/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/DesktopContent.tsx b/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/DesktopContent.tsx new file mode 100644 index 000000000..1522e153f --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/DesktopContent.tsx @@ -0,0 +1,79 @@ +"use client"; + +import { useEffect, useState } from "react"; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; +import { Label } from "@/components/ui/label"; +import { Switch } from "@/components/ui/switch"; +import { Spinner } from "@/components/ui/spinner"; + +export function DesktopContent() { + const [isElectron, setIsElectron] = useState(false); + const [loading, setLoading] = useState(true); + const [enabled, setEnabled] = useState(true); + + useEffect(() => { + if (!window.electronAPI) { + setLoading(false); + return; + } + setIsElectron(true); + + window.electronAPI.getAutocompleteEnabled().then((val) => { + setEnabled(val); + setLoading(false); + }); + }, []); + + if (!isElectron) { + return ( +
+

+ Desktop settings are only available in the SurfSense desktop app. +

+
+ ); + } + + if (loading) { + return ( +
+ +
+ ); + } + + const handleToggle = async (checked: boolean) => { + setEnabled(checked); + await window.electronAPI!.setAutocompleteEnabled(checked); + }; + + return ( +
+ + + Autocomplete + + Get inline writing suggestions powered by your knowledge base as you type in any app. + + + +
+
+ +

+ Show suggestions while typing in other applications. +

+
+ +
+ + +
+ ); +} diff --git a/surfsense_web/components/settings/user-settings-dialog.tsx b/surfsense_web/components/settings/user-settings-dialog.tsx index 389ebc5fd..b74ff973b 100644 --- a/surfsense_web/components/settings/user-settings-dialog.tsx +++ b/surfsense_web/components/settings/user-settings-dialog.tsx @@ -1,13 +1,14 @@ "use client"; import { useAtom } from "jotai"; -import { Globe, KeyRound, Receipt, Sparkles, User } from "lucide-react"; +import { Globe, KeyRound, Monitor, Receipt, Sparkles, User } from "lucide-react"; import { useTranslations } from "next-intl"; import { ApiKeyContent } from "@/app/dashboard/[search_space_id]/user-settings/components/ApiKeyContent"; import { CommunityPromptsContent } from "@/app/dashboard/[search_space_id]/user-settings/components/CommunityPromptsContent"; import { ProfileContent } from "@/app/dashboard/[search_space_id]/user-settings/components/ProfileContent"; import { PromptsContent } from "@/app/dashboard/[search_space_id]/user-settings/components/PromptsContent"; import { PurchaseHistoryContent } from "@/app/dashboard/[search_space_id]/user-settings/components/PurchaseHistoryContent"; +import { DesktopContent } from "@/app/dashboard/[search_space_id]/user-settings/components/DesktopContent"; import { userSettingsDialogAtom } from "@/atoms/settings/settings-dialog.atoms"; import { SettingsDialog } from "@/components/settings/settings-dialog"; @@ -37,6 +38,9 @@ export function UserSettingsDialog() { label: "Purchase History", icon: , }, + ...(typeof window !== "undefined" && window.electronAPI + ? [{ value: "desktop", label: "Desktop", icon: }] + : []), ]; return ( @@ -54,6 +58,7 @@ export function UserSettingsDialog() { {state.initialTab === "prompts" && } {state.initialTab === "community-prompts" && } {state.initialTab === "purchases" && } + {state.initialTab === "desktop" && } ); From 9c1d9357c4e3b0fe5eb25f737069d6494cea2188 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 2 Apr 2026 20:19:16 +0200 Subject: [PATCH 08/33] refactor: fix dynamic tooltip resizing and split autocomplete into SPR modules --- surfsense_desktop/electron-builder.yml | 12 +- surfsense_desktop/package.json | 1 + surfsense_desktop/pnpm-lock.yaml | 3 + surfsense_desktop/scripts/build-electron.mjs | 2 +- surfsense_desktop/src/main.ts | 3 +- .../index.ts} | 181 ++++++++---------- .../modules/autocomplete/keystroke-buffer.ts | 76 ++++++++ .../modules/autocomplete/suggestion-window.ts | 103 ++++++++++ surfsense_desktop/src/modules/platform.ts | 49 ----- .../app/desktop/permissions/page.tsx | 13 +- surfsense_web/app/desktop/suggestion/page.tsx | 6 +- .../app/desktop/suggestion/suggestion.css | 70 ++++--- 12 files changed, 326 insertions(+), 193 deletions(-) rename surfsense_desktop/src/modules/{autocomplete.ts => autocomplete/index.ts} (55%) create mode 100644 surfsense_desktop/src/modules/autocomplete/keystroke-buffer.ts create mode 100644 surfsense_desktop/src/modules/autocomplete/suggestion-window.ts diff --git a/surfsense_desktop/electron-builder.yml b/surfsense_desktop/electron-builder.yml index 74c69d223..115b69c8e 100644 --- a/surfsense_desktop/electron-builder.yml +++ b/surfsense_desktop/electron-builder.yml @@ -10,13 +10,13 @@ files: - dist/**/* - "!node_modules" - node_modules/uiohook-napi/**/* - - "!node_modules/uiohook-napi/build" - "!node_modules/uiohook-napi/src" - "!node_modules/uiohook-napi/libuiohook" - "!node_modules/uiohook-napi/binding.gyp" - node_modules/node-gyp-build/**/* + - node_modules/bindings/**/* + - node_modules/file-uri-to-path/**/* - node_modules/node-mac-permissions/**/* - - "!node_modules/node-mac-permissions/build" - "!node_modules/node-mac-permissions/src" - "!node_modules/node-mac-permissions/binding.gyp" - "!src" @@ -41,13 +41,19 @@ asarUnpack: - "**/*.node" - "node_modules/uiohook-napi/**/*" - "node_modules/node-gyp-build/**/*" + - "node_modules/bindings/**/*" + - "node_modules/file-uri-to-path/**/*" - "node_modules/node-mac-permissions/**/*" mac: icon: assets/icon.icns category: public.app-category.productivity artifactName: "${productName}-${version}-${arch}.${ext}" - hardenedRuntime: true + hardenedRuntime: false gatekeeperAssess: false + extendInfo: + NSInputMonitoringUsageDescription: "SurfSense uses input monitoring to provide system-wide autocomplete suggestions as you type." + NSAccessibilityUsageDescription: "SurfSense uses accessibility features to read text fields and insert suggestions." + NSAppleEventsUsageDescription: "SurfSense uses Apple Events to read text from the active application and insert autocomplete suggestions." target: - target: dmg arch: [x64, arm64] diff --git a/surfsense_desktop/package.json b/surfsense_desktop/package.json index a2e452b7c..01a63b265 100644 --- a/surfsense_desktop/package.json +++ b/surfsense_desktop/package.json @@ -29,6 +29,7 @@ "wait-on": "^9.0.4" }, "dependencies": { + "bindings": "^1.5.0", "electron-updater": "^6.8.3", "get-port-please": "^3.2.0", "node-mac-permissions": "^2.5.0", diff --git a/surfsense_desktop/pnpm-lock.yaml b/surfsense_desktop/pnpm-lock.yaml index 82bad9456..d0b453d31 100644 --- a/surfsense_desktop/pnpm-lock.yaml +++ b/surfsense_desktop/pnpm-lock.yaml @@ -8,6 +8,9 @@ importers: .: dependencies: + bindings: + specifier: ^1.5.0 + version: 1.5.0 electron-updater: specifier: ^6.8.3 version: 6.8.3 diff --git a/surfsense_desktop/scripts/build-electron.mjs b/surfsense_desktop/scripts/build-electron.mjs index 83d941dd2..c2869ec46 100644 --- a/surfsense_desktop/scripts/build-electron.mjs +++ b/surfsense_desktop/scripts/build-electron.mjs @@ -104,7 +104,7 @@ async function buildElectron() { bundle: true, platform: 'node', target: 'node18', - external: ['electron', 'uiohook-napi', 'node-mac-permissions'], + external: ['electron', 'uiohook-napi', 'node-mac-permissions', 'bindings', 'file-uri-to-path'], sourcemap: true, minify: false, define: { diff --git a/surfsense_desktop/src/main.ts b/surfsense_desktop/src/main.ts index 9623be82e..c96453c6d 100644 --- a/surfsense_desktop/src/main.ts +++ b/surfsense_desktop/src/main.ts @@ -19,7 +19,8 @@ if (!setupDeepLinks()) { registerIpcHandlers(); function getInitialPath(): string { - if (process.platform === 'darwin' && !allPermissionsGranted()) { + const granted = allPermissionsGranted(); + if (process.platform === 'darwin' && !granted) { return '/desktop/permissions'; } return '/dashboard'; diff --git a/surfsense_desktop/src/modules/autocomplete.ts b/surfsense_desktop/src/modules/autocomplete/index.ts similarity index 55% rename from surfsense_desktop/src/modules/autocomplete.ts rename to surfsense_desktop/src/modules/autocomplete/index.ts index 2b877723f..2ea37d051 100644 --- a/surfsense_desktop/src/modules/autocomplete.ts +++ b/surfsense_desktop/src/modules/autocomplete/index.ts @@ -1,20 +1,19 @@ -import { BrowserWindow, clipboard, ipcMain, screen, shell } from 'electron'; -import path from 'path'; -import { IPC_CHANNELS } from '../ipc/channels'; -import { allPermissionsGranted } from './permissions'; -import { getFieldContent, getFrontmostApp, hasAccessibilityPermission, simulatePaste } from './platform'; -import { getServerPort } from './server'; -import { getMainWindow } from './window'; +import { clipboard, ipcMain, screen } from 'electron'; +import { IPC_CHANNELS } from '../../ipc/channels'; +import { getFrontmostApp, hasAccessibilityPermission, simulatePaste } from '../platform'; +import { getMainWindow } from '../window'; +import { + appendToBuffer, buildKeycodeMap, getBuffer, getBufferTrimmed, + getLastTrackedApp, removeLastChar, resetBuffer, resolveChar, setLastTrackedApp, +} from './keystroke-buffer'; +import { createSuggestionWindow, destroySuggestion, getSuggestionWindow } from './suggestion-window'; const DEBOUNCE_MS = 600; -const TOOLTIP_WIDTH = 420; -const TOOLTIP_HEIGHT = 140; let uIOhook: any = null; let UiohookKey: any = {}; let IGNORED_KEYCODES: Set = new Set(); -let suggestionWindow: BrowserWindow | null = null; let debounceTimer: ReturnType | null = null; let hookStarted = false; let autocompleteEnabled = true; @@ -38,12 +37,8 @@ function loadUiohook(): boolean { UiohookKey.F5, UiohookKey.F6, UiohookKey.F7, UiohookKey.F8, UiohookKey.F9, UiohookKey.F10, UiohookKey.F11, UiohookKey.F12, UiohookKey.PrintScreen, - UiohookKey.Insert, UiohookKey.Delete, - UiohookKey.Home, UiohookKey.End, - UiohookKey.PageUp, UiohookKey.PageDown, - UiohookKey.ArrowUp, UiohookKey.ArrowDown, - UiohookKey.ArrowLeft, UiohookKey.ArrowRight, ]); + buildKeycodeMap(); console.log('[autocomplete] uiohook-napi loaded'); return true; } catch (err) { @@ -52,70 +47,6 @@ function loadUiohook(): boolean { } } -function destroySuggestion(): void { - if (suggestionWindow && !suggestionWindow.isDestroyed()) { - suggestionWindow.close(); - } - suggestionWindow = null; -} - -function clampToScreen(x: number, y: number, w: number, h: number): { x: number; y: number } { - const display = screen.getDisplayNearestPoint({ x, y }); - const { x: dx, y: dy, width: dw, height: dh } = display.workArea; - return { - x: Math.max(dx, Math.min(x, dx + dw - w)), - y: Math.max(dy, Math.min(y, dy + dh - h)), - }; -} - -function createSuggestionWindow(x: number, y: number): BrowserWindow { - destroySuggestion(); - - const pos = clampToScreen(x, y + 20, TOOLTIP_WIDTH, TOOLTIP_HEIGHT); - - suggestionWindow = new BrowserWindow({ - width: TOOLTIP_WIDTH, - height: TOOLTIP_HEIGHT, - x: pos.x, - y: pos.y, - frame: false, - transparent: true, - focusable: false, - alwaysOnTop: true, - skipTaskbar: true, - resizable: false, - hasShadow: true, - type: 'panel', - webPreferences: { - preload: path.join(__dirname, 'preload.js'), - contextIsolation: true, - nodeIntegration: false, - sandbox: true, - }, - show: false, - }); - - suggestionWindow.loadURL(`http://localhost:${getServerPort()}/desktop/suggestion?t=${Date.now()}`); - - suggestionWindow.once('ready-to-show', () => { - suggestionWindow?.showInactive(); - }); - - suggestionWindow.webContents.setWindowOpenHandler(({ url }) => { - if (url.startsWith('http://localhost')) { - return { action: 'allow' }; - } - shell.openExternal(url); - return { action: 'deny' }; - }); - - suggestionWindow.on('closed', () => { - suggestionWindow = null; - }); - - return suggestionWindow; -} - function clearDebounce(): void { if (debounceTimer) { clearTimeout(debounceTimer); @@ -128,10 +59,24 @@ function isSurfSenseWindow(): boolean { return app === 'Electron' || app === 'SurfSense' || app === 'surfsense-desktop'; } -function onKeyDown(event: { keycode: number; ctrlKey?: boolean; metaKey?: boolean; altKey?: boolean }): void { +function onKeyDown(event: { + keycode: number; + shiftKey?: boolean; + ctrlKey?: boolean; + metaKey?: boolean; + altKey?: boolean; +}): void { if (!autocompleteEnabled) return; - if (event.keycode === UiohookKey.Tab && suggestionWindow && !suggestionWindow.isDestroyed()) { + const currentApp = getFrontmostApp(); + if (currentApp !== getLastTrackedApp()) { + resetBuffer(); + setLastTrackedApp(currentApp); + } + + const win = getSuggestionWindow(); + + if (event.keycode === UiohookKey.Tab && win && !win.isDestroyed()) { if (pendingSuggestionText) { acceptAndInject(pendingSuggestionText); } @@ -139,7 +84,7 @@ function onKeyDown(event: { keycode: number; ctrlKey?: boolean; metaKey?: boolea } if (event.keycode === UiohookKey.Escape) { - if (suggestionWindow && !suggestionWindow.isDestroyed()) { + if (win && !win.isDestroyed()) { destroySuggestion(); pendingSuggestionText = ''; } @@ -147,11 +92,41 @@ function onKeyDown(event: { keycode: number; ctrlKey?: boolean; metaKey?: boolea return; } - if (IGNORED_KEYCODES.has(event.keycode)) return; - if (event.ctrlKey || event.metaKey || event.altKey) return; - if (isSurfSenseWindow()) return; + if (currentApp === 'Electron' || currentApp === 'SurfSense' || currentApp === 'surfsense-desktop') { + return; + } - if (suggestionWindow && !suggestionWindow.isDestroyed()) { + if (event.ctrlKey || event.metaKey || event.altKey) { + resetBuffer(); + clearDebounce(); + return; + } + + if (event.keycode === UiohookKey.Backspace) { + removeLastChar(); + } else if (event.keycode === UiohookKey.Delete) { + // forward delete doesn't affect our trailing buffer + } else if (event.keycode === UiohookKey.Enter) { + appendToBuffer('\n'); + } else if (event.keycode === UiohookKey.Space) { + appendToBuffer(' '); + } else if ( + event.keycode === UiohookKey.ArrowLeft || event.keycode === UiohookKey.ArrowRight || + event.keycode === UiohookKey.ArrowUp || event.keycode === UiohookKey.ArrowDown || + event.keycode === UiohookKey.Home || event.keycode === UiohookKey.End || + event.keycode === UiohookKey.PageUp || event.keycode === UiohookKey.PageDown + ) { + resetBuffer(); + clearDebounce(); + return; + } else if (IGNORED_KEYCODES.has(event.keycode)) { + return; + } else { + const ch = resolveChar(event.keycode, !!event.shiftKey); + if (ch) appendToBuffer(ch); + } + + if (win && !win.isDestroyed()) { destroySuggestion(); } @@ -161,13 +136,16 @@ function onKeyDown(event: { keycode: number; ctrlKey?: boolean; metaKey?: boolea }, DEBOUNCE_MS); } +function onMouseClick(): void { + resetBuffer(); +} + async function triggerAutocomplete(): Promise { if (!hasAccessibilityPermission()) return; if (isSurfSenseWindow()) return; - const fieldContent = getFieldContent(); - if (!fieldContent || !fieldContent.text.trim()) return; - if (fieldContent.text.trim().length < 5) return; + const text = getBufferTrimmed(); + if (!text || text.length < 5) return; sourceApp = getFrontmostApp(); savedClipboard = clipboard.readText(); @@ -186,13 +164,16 @@ async function triggerAutocomplete(): Promise { } win.webContents.once('did-finish-load', () => { - if (suggestionWindow && !suggestionWindow.isDestroyed()) { - suggestionWindow.webContents.send(IPC_CHANNELS.AUTOCOMPLETE_CONTEXT, { - text: fieldContent.text, - cursorPosition: fieldContent.cursorPosition, - searchSpaceId, - }); - } + const sw = getSuggestionWindow(); + setTimeout(() => { + if (sw && !sw.isDestroyed()) { + sw.webContents.send(IPC_CHANNELS.AUTOCOMPLETE_CONTEXT, { + text: getBuffer(), + cursorPosition: getBuffer().length, + searchSpaceId, + }); + } + }, 300); }); } @@ -209,6 +190,7 @@ async function acceptAndInject(text: string): Promise { simulatePaste(); await new Promise((r) => setTimeout(r, 100)); clipboard.writeText(savedClipboard); + appendToBuffer(text); } catch { clipboard.writeText(savedClipboard); } @@ -238,21 +220,16 @@ function registerIpcHandlers(): void { export function registerAutocomplete(): void { registerIpcHandlers(); - if (!allPermissionsGranted()) { - console.log('[autocomplete] Permissions not granted — hook not started'); - return; - } - if (!loadUiohook()) { console.error('[autocomplete] Cannot start: uiohook-napi failed to load'); return; } uIOhook.on('keydown', onKeyDown); + uIOhook.on('click', onMouseClick); try { uIOhook.start(); hookStarted = true; - console.log('[autocomplete] uIOhook started'); } catch (err) { console.error('[autocomplete] uIOhook.start() failed:', err); } diff --git a/surfsense_desktop/src/modules/autocomplete/keystroke-buffer.ts b/surfsense_desktop/src/modules/autocomplete/keystroke-buffer.ts new file mode 100644 index 000000000..ca232d307 --- /dev/null +++ b/surfsense_desktop/src/modules/autocomplete/keystroke-buffer.ts @@ -0,0 +1,76 @@ +const MAX_BUFFER_LENGTH = 4000; +const KEYCODE_TO_CHAR: Record = {}; + +let keystrokeBuffer = ''; +let lastTrackedApp = ''; + +export function buildKeycodeMap(): void { + const letters: [string, number][] = [ + ['q', 16], ['w', 17], ['e', 18], ['r', 19], ['t', 20], + ['y', 21], ['u', 22], ['i', 23], ['o', 24], ['p', 25], + ['a', 30], ['s', 31], ['d', 32], ['f', 33], ['g', 34], + ['h', 35], ['j', 36], ['k', 37], ['l', 38], + ['z', 44], ['x', 45], ['c', 46], ['v', 47], + ['b', 48], ['n', 49], ['m', 50], + ]; + for (const [ch, code] of letters) { + KEYCODE_TO_CHAR[code] = [ch, ch.toUpperCase()]; + } + + const digits: [string, string, number][] = [ + ['1', '!', 2], ['2', '@', 3], ['3', '#', 4], ['4', '$', 5], + ['5', '%', 6], ['6', '^', 7], ['7', '&', 8], ['8', '*', 9], + ['9', '(', 10], ['0', ')', 11], + ]; + for (const [norm, shifted, code] of digits) { + KEYCODE_TO_CHAR[code] = [norm, shifted]; + } + + const punctuation: [string, string, number][] = [ + [';', ':', 39], ['=', '+', 13], [',', '<', 51], ['-', '_', 12], + ['.', '>', 52], ['/', '?', 53], ['`', '~', 41], ['[', '{', 26], + ['\\', '|', 43], [']', '}', 27], ["'", '"', 40], + ]; + for (const [norm, shifted, code] of punctuation) { + KEYCODE_TO_CHAR[code] = [norm, shifted]; + } +} + +export function resetBuffer(): void { + keystrokeBuffer = ''; +} + +export function appendToBuffer(char: string): void { + keystrokeBuffer += char; + if (keystrokeBuffer.length > MAX_BUFFER_LENGTH) { + keystrokeBuffer = keystrokeBuffer.slice(-MAX_BUFFER_LENGTH); + } +} + +export function removeLastChar(): void { + if (keystrokeBuffer.length > 0) { + keystrokeBuffer = keystrokeBuffer.slice(0, -1); + } +} + +export function getBuffer(): string { + return keystrokeBuffer; +} + +export function getBufferTrimmed(): string { + return keystrokeBuffer.trim(); +} + +export function getLastTrackedApp(): string { + return lastTrackedApp; +} + +export function setLastTrackedApp(app: string): void { + lastTrackedApp = app; +} + +export function resolveChar(keycode: number, shift: boolean): string | null { + const mapping = KEYCODE_TO_CHAR[keycode]; + if (!mapping) return null; + return shift ? mapping[1] : mapping[0]; +} diff --git a/surfsense_desktop/src/modules/autocomplete/suggestion-window.ts b/surfsense_desktop/src/modules/autocomplete/suggestion-window.ts new file mode 100644 index 000000000..f03930cf6 --- /dev/null +++ b/surfsense_desktop/src/modules/autocomplete/suggestion-window.ts @@ -0,0 +1,103 @@ +import { BrowserWindow, screen, shell } from 'electron'; +import path from 'path'; +import { getServerPort } from '../server'; + +const TOOLTIP_WIDTH = 420; +const TOOLTIP_HEIGHT = 38; +const MAX_HEIGHT = 400; + +let suggestionWindow: BrowserWindow | null = null; +let resizeTimer: ReturnType | null = null; + +function clampToScreen(x: number, y: number, w: number, h: number): { x: number; y: number } { + const display = screen.getDisplayNearestPoint({ x, y }); + const { x: dx, y: dy, width: dw, height: dh } = display.workArea; + return { + x: Math.max(dx, Math.min(x, dx + dw - w)), + y: Math.max(dy, Math.min(y, dy + dh - h)), + }; +} + +function stopResizePolling(): void { + if (resizeTimer) { clearInterval(resizeTimer); resizeTimer = null; } +} + +function startResizePolling(win: BrowserWindow): void { + stopResizePolling(); + let lastH = 0; + resizeTimer = setInterval(async () => { + if (!win || win.isDestroyed()) { stopResizePolling(); return; } + try { + const h: number = await win.webContents.executeJavaScript( + `document.body.scrollHeight` + ); + if (h > 0 && h !== lastH) { + lastH = h; + const clamped = Math.min(h, MAX_HEIGHT); + const bounds = win.getBounds(); + win.setBounds({ x: bounds.x, y: bounds.y, width: TOOLTIP_WIDTH, height: clamped }); + } + } catch {} + }, 150); +} + +export function getSuggestionWindow(): BrowserWindow | null { + return suggestionWindow; +} + +export function destroySuggestion(): void { + stopResizePolling(); + if (suggestionWindow && !suggestionWindow.isDestroyed()) { + suggestionWindow.close(); + } + suggestionWindow = null; +} + +export function createSuggestionWindow(x: number, y: number): BrowserWindow { + destroySuggestion(); + + const pos = clampToScreen(x, y + 20, TOOLTIP_WIDTH, TOOLTIP_HEIGHT); + + suggestionWindow = new BrowserWindow({ + width: TOOLTIP_WIDTH, + height: TOOLTIP_HEIGHT, + x: pos.x, + y: pos.y, + frame: false, + transparent: true, + focusable: false, + alwaysOnTop: true, + skipTaskbar: true, + hasShadow: true, + type: 'panel', + webPreferences: { + preload: path.join(__dirname, '..', 'preload.js'), + contextIsolation: true, + nodeIntegration: false, + sandbox: true, + }, + show: false, + }); + + suggestionWindow.loadURL(`http://localhost:${getServerPort()}/desktop/suggestion?t=${Date.now()}`); + + suggestionWindow.once('ready-to-show', () => { + suggestionWindow?.showInactive(); + if (suggestionWindow) startResizePolling(suggestionWindow); + }); + + suggestionWindow.webContents.setWindowOpenHandler(({ url }) => { + if (url.startsWith('http://localhost')) { + return { action: 'allow' }; + } + shell.openExternal(url); + return { action: 'deny' }; + }); + + suggestionWindow.on('closed', () => { + stopResizePolling(); + suggestionWindow = null; + }); + + return suggestionWindow; +} diff --git a/surfsense_desktop/src/modules/platform.ts b/surfsense_desktop/src/modules/platform.ts index 262866d07..1ab0c38fb 100644 --- a/surfsense_desktop/src/modules/platform.ts +++ b/surfsense_desktop/src/modules/platform.ts @@ -19,20 +19,6 @@ export function getFrontmostApp(): string { return ''; } -export function getSelectedText(): string { - try { - if (process.platform === 'darwin') { - return execSync( - 'osascript -e \'tell application "System Events" to get value of attribute "AXSelectedText" of focused UI element of first application process whose frontmost is true\'' - ).toString().trim(); - } - // Windows: no reliable accessibility API for selected text across apps - } catch { - return ''; - } - return ''; -} - export function simulateCopy(): void { if (process.platform === 'darwin') { execSync('osascript -e \'tell application "System Events" to keystroke "c" using command down\''); @@ -58,38 +44,3 @@ export function hasAccessibilityPermission(): boolean { if (process.platform !== 'darwin') return true; return systemPreferences.isTrustedAccessibilityClient(false); } - -export interface FieldContent { - text: string; - cursorPosition: number; -} - -export function getFieldContent(): FieldContent | null { - if (process.platform !== 'darwin') return null; - - try { - const text = execSync( - 'osascript -e \'tell application "System Events" to get value of attribute "AXValue" of focused UI element of first application process whose frontmost is true\'', - { timeout: 500 } - ).toString().trim(); - - let cursorPosition = text.length; - try { - const rangeStr = execSync( - 'osascript -e \'tell application "System Events" to get value of attribute "AXSelectedTextRange" of focused UI element of first application process whose frontmost is true\'', - { timeout: 500 } - ).toString().trim(); - - const locationMatch = rangeStr.match(/location[:\s]*(\d+)/i); - if (locationMatch) { - cursorPosition = parseInt(locationMatch[1], 10); - } - } catch { - // Fall back to end of text - } - - return { text, cursorPosition }; - } catch { - return null; - } -} diff --git a/surfsense_web/app/desktop/permissions/page.tsx b/surfsense_web/app/desktop/permissions/page.tsx index 2bcdc42df..8bde63357 100644 --- a/surfsense_web/app/desktop/permissions/page.tsx +++ b/surfsense_web/app/desktop/permissions/page.tsx @@ -169,11 +169,14 @@ export default function DesktopPermissionsPage() { > Open System Settings - {status === "denied" && ( -

- Toggle SurfSense on in System Settings to continue. -

- )} + {status === "denied" && ( +

+ Toggle SurfSense on in System Settings to continue. +

+ )} +

+ If SurfSense doesn't appear in the list, click + and select it from Applications. +

)} diff --git a/surfsense_web/app/desktop/suggestion/page.tsx b/surfsense_web/app/desktop/suggestion/page.tsx index 14dfab3af..69a19e3f1 100644 --- a/surfsense_web/app/desktop/suggestion/page.tsx +++ b/surfsense_web/app/desktop/suggestion/page.tsx @@ -151,9 +151,9 @@ export default function SuggestionPage() {

{suggestion}

- Tab accept - · - Esc dismiss + Tab accept + + Esc dismiss
); diff --git a/surfsense_web/app/desktop/suggestion/suggestion.css b/surfsense_web/app/desktop/suggestion/suggestion.css index e9471e7f8..0d3332103 100644 --- a/surfsense_web/app/desktop/suggestion/suggestion.css +++ b/surfsense_web/app/desktop/suggestion/suggestion.css @@ -1,8 +1,16 @@ +html, body { + margin: 0 !important; + padding: 0 !important; + background: transparent !important; + overflow: hidden !important; + height: auto !important; + width: 100% !important; +} + .suggestion-body { margin: 0; padding: 0; background: transparent; - overflow: hidden; font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; -webkit-font-smoothing: antialiased; user-select: none; @@ -10,69 +18,73 @@ } .suggestion-tooltip { - background: rgba(30, 30, 30, 0.95); - backdrop-filter: blur(12px); - -webkit-backdrop-filter: blur(12px); - border: 1px solid rgba(255, 255, 255, 0.1); - border-radius: 10px; - padding: 10px 14px; + background: #1e1e1e; + border: 1px solid #3c3c3c; + border-radius: 8px; + padding: 8px 12px; margin: 4px; max-width: 400px; - box-shadow: 0 8px 32px rgba(0, 0, 0, 0.4), - 0 2px 8px rgba(0, 0, 0, 0.2); + box-shadow: 0 4px 16px rgba(0, 0, 0, 0.5); } .suggestion-text { - color: rgba(255, 255, 255, 0.9); + color: #d4d4d4; font-size: 13px; - line-height: 1.5; - margin: 0 0 8px 0; + line-height: 1.45; + margin: 0 0 6px 0; word-wrap: break-word; white-space: pre-wrap; } .suggestion-hint { - color: rgba(255, 255, 255, 0.4); + color: #666; font-size: 11px; display: flex; align-items: center; - gap: 4px; + gap: 6px; + border-top: 1px solid #2a2a2a; + padding-top: 6px; } -.suggestion-key { - background: rgba(255, 255, 255, 0.1); - border: 1px solid rgba(255, 255, 255, 0.15); +.suggestion-hint kbd { + background: #2a2a2a; + border: 1px solid #3c3c3c; border-radius: 3px; - padding: 1px 5px; + padding: 0 4px; + font-family: inherit; font-size: 10px; - font-weight: 500; - color: rgba(255, 255, 255, 0.6); + font-weight: 600; + color: #999; + line-height: 18px; } .suggestion-separator { - margin: 0 2px; + width: 1px; + height: 10px; + background: #333; } .suggestion-error { - border-color: rgba(255, 80, 80, 0.3); + border-color: #5c2626; } .suggestion-error-text { - color: rgba(255, 120, 120, 0.9); + color: #f48771; font-size: 12px; } .suggestion-loading { display: flex; - gap: 4px; - padding: 4px 0; + gap: 5px; + padding: 2px 0; + justify-content: center; } .suggestion-dot { - width: 5px; - height: 5px; + width: 4px; + height: 4px; border-radius: 50%; - background: rgba(255, 255, 255, 0.4); + background: #666; animation: suggestion-pulse 1.2s infinite ease-in-out; } @@ -91,6 +103,6 @@ } 40% { opacity: 1; - transform: scale(1); + transform: scale(1.1); } } From 3e68d4aa3ed04c87f155a4da08a6610251755f74 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 2 Apr 2026 20:38:09 +0200 Subject: [PATCH 09/33] refactor: extract autocomplete service and fix tooltip screen-edge positioning --- .../app/routes/autocomplete_routes.py | 109 +---------------- .../app/services/autocomplete_service.py | 110 ++++++++++++++++++ .../modules/autocomplete/suggestion-window.ts | 27 +++-- 3 files changed, 130 insertions(+), 116 deletions(-) create mode 100644 surfsense_backend/app/services/autocomplete_service.py diff --git a/surfsense_backend/app/routes/autocomplete_routes.py b/surfsense_backend/app/routes/autocomplete_routes.py index 9a285a723..68c56d0e0 100644 --- a/surfsense_backend/app/routes/autocomplete_routes.py +++ b/surfsense_backend/app/routes/autocomplete_routes.py @@ -1,118 +1,14 @@ -import logging -from typing import AsyncGenerator - from fastapi import APIRouter, Depends, Query from fastapi.responses import StreamingResponse -from langchain_core.messages import HumanMessage, SystemMessage from sqlalchemy.ext.asyncio import AsyncSession from app.db import User, get_async_session -from app.retriever.chunks_hybrid_search import ChucksHybridSearchRetriever -from app.services.llm_service import get_agent_llm +from app.services.autocomplete_service import stream_autocomplete from app.services.new_streaming_service import VercelStreamingService from app.users import current_active_user -logger = logging.getLogger(__name__) - router = APIRouter(prefix="/autocomplete", tags=["autocomplete"]) -AUTOCOMPLETE_SYSTEM_PROMPT = """You are an inline text autocomplete engine. Your job is to complete the user's text naturally. - -Rules: -- Output ONLY the continuation text. Do NOT repeat what the user already typed. -- Keep completions concise: 1-3 sentences maximum. -- Match the user's tone, style, and language. -- If knowledge base context is provided, use it to make the completion factually accurate and personalized. -- Do NOT add quotes, explanations, or meta-commentary. -- Do NOT start with a space unless grammatically required. -- If you cannot produce a useful completion, output nothing.""" - -KB_CONTEXT_TEMPLATE = """ -Relevant knowledge base context (use this to personalize the completion): ---- -{kb_context} ---- -""" - - -async def _stream_autocomplete( - text: str, - cursor_position: int, - search_space_id: int, - session: AsyncSession, -) -> AsyncGenerator[str, None]: - """Stream an autocomplete response with KB context.""" - streaming_service = VercelStreamingService() - - try: - # Text before cursor is what we're completing - text_before_cursor = text[:cursor_position] if cursor_position >= 0 else text - - if not text_before_cursor.strip(): - yield streaming_service.format_message_start() - yield streaming_service.format_finish() - yield streaming_service.format_done() - return - - # Fast KB lookup: vector-only search, top 3 chunks, no planner LLM - kb_context = "" - try: - retriever = ChucksHybridSearchRetriever(session) - chunks = await retriever.vector_search( - query_text=text_before_cursor[-200:], # last 200 chars for relevance - top_k=3, - search_space_id=search_space_id, - ) - if chunks: - kb_snippets = [] - for chunk in chunks: - content = getattr(chunk, "content", None) or getattr(chunk, "chunk_text", "") - if content: - kb_snippets.append(content[:300]) - if kb_snippets: - kb_context = KB_CONTEXT_TEMPLATE.format( - kb_context="\n\n".join(kb_snippets) - ) - except Exception as e: - logger.warning(f"KB search failed for autocomplete, proceeding without context: {e}") - - # Get the search space's configured LLM - llm = await get_agent_llm(session, search_space_id) - if not llm: - yield streaming_service.format_message_start() - error_msg = "No LLM configured for this search space" - yield streaming_service.format_error(error_msg) - yield streaming_service.format_done() - return - - system_prompt = AUTOCOMPLETE_SYSTEM_PROMPT - if kb_context: - system_prompt += kb_context - - messages = [ - SystemMessage(content=system_prompt), - HumanMessage(content=f"Complete this text:\n{text_before_cursor}"), - ] - - # Stream the response - yield streaming_service.format_message_start() - text_id = streaming_service.generate_text_id() - yield streaming_service.format_text_start(text_id) - - async for chunk in llm.astream(messages): - token = chunk.content if hasattr(chunk, "content") else str(chunk) - if token: - yield streaming_service.format_text_delta(text_id, token) - - yield streaming_service.format_text_end(text_id) - yield streaming_service.format_finish() - yield streaming_service.format_done() - - except Exception as e: - logger.error(f"Autocomplete streaming error: {e}") - yield streaming_service.format_error(str(e)) - yield streaming_service.format_done() - @router.post("/stream") async def autocomplete_stream( @@ -122,12 +18,11 @@ async def autocomplete_stream( user: User = Depends(current_active_user), session: AsyncSession = Depends(get_async_session), ): - """Stream an autocomplete suggestion based on the current text and KB context.""" if cursor_position < 0: cursor_position = len(text) return StreamingResponse( - _stream_autocomplete(text, cursor_position, search_space_id, session), + stream_autocomplete(text, cursor_position, search_space_id, session), media_type="text/event-stream", headers={ **VercelStreamingService.get_response_headers(), diff --git a/surfsense_backend/app/services/autocomplete_service.py b/surfsense_backend/app/services/autocomplete_service.py new file mode 100644 index 000000000..7c172275d --- /dev/null +++ b/surfsense_backend/app/services/autocomplete_service.py @@ -0,0 +1,110 @@ +import logging +from typing import AsyncGenerator + +from langchain_core.messages import HumanMessage, SystemMessage +from sqlalchemy.ext.asyncio import AsyncSession + +from app.retriever.chunks_hybrid_search import ChucksHybridSearchRetriever +from app.services.llm_service import get_agent_llm +from app.services.new_streaming_service import VercelStreamingService + +logger = logging.getLogger(__name__) + +SYSTEM_PROMPT = """You are an inline text autocomplete engine. Your job is to complete the user's text naturally. + +Rules: +- Output ONLY the continuation text. Do NOT repeat what the user already typed. +- Keep completions concise: 1-3 sentences maximum. +- Match the user's tone, style, and language. +- If knowledge base context is provided, use it to make the completion factually accurate and personalized. +- Do NOT add quotes, explanations, or meta-commentary. +- Do NOT start with a space unless grammatically required. +- If you cannot produce a useful completion, output nothing.""" + +KB_CONTEXT_TEMPLATE = """ +Relevant knowledge base context (use this to personalize the completion): +--- +{kb_context} +--- +""" + + +async def _retrieve_kb_context( + session: AsyncSession, + text: str, + search_space_id: int, +) -> str: + try: + retriever = ChucksHybridSearchRetriever(session) + chunks = await retriever.vector_search( + query_text=text[-200:], + top_k=3, + search_space_id=search_space_id, + ) + if not chunks: + return "" + snippets = [] + for chunk in chunks: + content = getattr(chunk, "content", None) or getattr(chunk, "chunk_text", "") + if content: + snippets.append(content[:300]) + if not snippets: + return "" + return KB_CONTEXT_TEMPLATE.format(kb_context="\n\n".join(snippets)) + except Exception as e: + logger.warning(f"KB search failed for autocomplete, proceeding without context: {e}") + return "" + + +async def stream_autocomplete( + text: str, + cursor_position: int, + search_space_id: int, + session: AsyncSession, +) -> AsyncGenerator[str, None]: + """Build context, call the LLM, and yield SSE-formatted tokens.""" + streaming = VercelStreamingService() + text_before_cursor = text[:cursor_position] if cursor_position >= 0 else text + + if not text_before_cursor.strip(): + yield streaming.format_message_start() + yield streaming.format_finish() + yield streaming.format_done() + return + + kb_context = await _retrieve_kb_context(session, text_before_cursor, search_space_id) + + llm = await get_agent_llm(session, search_space_id) + if not llm: + yield streaming.format_message_start() + yield streaming.format_error("No LLM configured for this search space") + yield streaming.format_done() + return + + system_prompt = SYSTEM_PROMPT + if kb_context: + system_prompt += kb_context + + messages = [ + SystemMessage(content=system_prompt), + HumanMessage(content=f"Complete this text:\n{text_before_cursor}"), + ] + + try: + yield streaming.format_message_start() + text_id = streaming.generate_text_id() + yield streaming.format_text_start(text_id) + + async for chunk in llm.astream(messages): + token = chunk.content if hasattr(chunk, "content") else str(chunk) + if token: + yield streaming.format_text_delta(text_id, token) + + yield streaming.format_text_end(text_id) + yield streaming.format_finish() + yield streaming.format_done() + + except Exception as e: + logger.error(f"Autocomplete streaming error: {e}") + yield streaming.format_error(str(e)) + yield streaming.format_done() diff --git a/surfsense_desktop/src/modules/autocomplete/suggestion-window.ts b/surfsense_desktop/src/modules/autocomplete/suggestion-window.ts index f03930cf6..e8a2f3a91 100644 --- a/surfsense_desktop/src/modules/autocomplete/suggestion-window.ts +++ b/surfsense_desktop/src/modules/autocomplete/suggestion-window.ts @@ -8,14 +8,22 @@ const MAX_HEIGHT = 400; let suggestionWindow: BrowserWindow | null = null; let resizeTimer: ReturnType | null = null; +let cursorOrigin = { x: 0, y: 0 }; -function clampToScreen(x: number, y: number, w: number, h: number): { x: number; y: number } { - const display = screen.getDisplayNearestPoint({ x, y }); +const CURSOR_GAP = 20; + +function positionOnScreen(cursorX: number, cursorY: number, w: number, h: number): { x: number; y: number } { + const display = screen.getDisplayNearestPoint({ x: cursorX, y: cursorY }); const { x: dx, y: dy, width: dw, height: dh } = display.workArea; - return { - x: Math.max(dx, Math.min(x, dx + dw - w)), - y: Math.max(dy, Math.min(y, dy + dh - h)), - }; + + const x = Math.max(dx, Math.min(cursorX, dx + dw - w)); + + const spaceBelow = (dy + dh) - (cursorY + CURSOR_GAP); + const y = spaceBelow >= h + ? cursorY + CURSOR_GAP + : cursorY - h - CURSOR_GAP; + + return { x, y: Math.max(dy, y) }; } function stopResizePolling(): void { @@ -34,8 +42,8 @@ function startResizePolling(win: BrowserWindow): void { if (h > 0 && h !== lastH) { lastH = h; const clamped = Math.min(h, MAX_HEIGHT); - const bounds = win.getBounds(); - win.setBounds({ x: bounds.x, y: bounds.y, width: TOOLTIP_WIDTH, height: clamped }); + const pos = positionOnScreen(cursorOrigin.x, cursorOrigin.y, TOOLTIP_WIDTH, clamped); + win.setBounds({ x: pos.x, y: pos.y, width: TOOLTIP_WIDTH, height: clamped }); } } catch {} }, 150); @@ -55,8 +63,9 @@ export function destroySuggestion(): void { export function createSuggestionWindow(x: number, y: number): BrowserWindow { destroySuggestion(); + cursorOrigin = { x, y }; - const pos = clampToScreen(x, y + 20, TOOLTIP_WIDTH, TOOLTIP_HEIGHT); + const pos = positionOnScreen(x, y, TOOLTIP_WIDTH, TOOLTIP_HEIGHT); suggestionWindow = new BrowserWindow({ width: TOOLTIP_WIDTH, From a99d999a3658c09ce133940cc0dd15a3353d6cd7 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 2 Apr 2026 21:29:05 +0200 Subject: [PATCH 10/33] fix: correct preload.js path after autocomplete module restructure --- surfsense_desktop/src/modules/autocomplete/suggestion-window.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/surfsense_desktop/src/modules/autocomplete/suggestion-window.ts b/surfsense_desktop/src/modules/autocomplete/suggestion-window.ts index e8a2f3a91..8f61b2901 100644 --- a/surfsense_desktop/src/modules/autocomplete/suggestion-window.ts +++ b/surfsense_desktop/src/modules/autocomplete/suggestion-window.ts @@ -80,7 +80,7 @@ export function createSuggestionWindow(x: number, y: number): BrowserWindow { hasShadow: true, type: 'panel', webPreferences: { - preload: path.join(__dirname, '..', 'preload.js'), + preload: path.join(__dirname, 'preload.js'), contextIsolation: true, nodeIntegration: false, sandbox: true, From fc84dcffb05f794ca19641253ad8cab68b85d43f Mon Sep 17 00:00:00 2001 From: okxint Date: Fri, 3 Apr 2026 13:59:14 +0530 Subject: [PATCH 11/33] fix: memoize formatRelativeTime in thread list to prevent unnecessary re-renders --- surfsense_web/components/assistant-ui/thread-list.tsx | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/surfsense_web/components/assistant-ui/thread-list.tsx b/surfsense_web/components/assistant-ui/thread-list.tsx index f1d10ca16..e8b8db6fe 100644 --- a/surfsense_web/components/assistant-ui/thread-list.tsx +++ b/surfsense_web/components/assistant-ui/thread-list.tsx @@ -9,7 +9,7 @@ import { TrashIcon, } from "lucide-react"; import { useRouter } from "next/navigation"; -import { memo, useCallback, useEffect, useState } from "react"; +import { memo, useCallback, useEffect, useMemo, useState } from "react"; import { Button } from "@/components/ui/button"; import { DropdownMenu, @@ -224,6 +224,11 @@ const ThreadListItemComponent = memo(function ThreadListItemComponent({ onUnarchive, onDelete, }: ThreadListItemComponentProps) { + const relativeTime = useMemo( + () => formatRelativeTime(new Date(thread.updatedAt)), + [thread.updatedAt] + ); + return ( - {status === "denied" && ( -

- Toggle SurfSense on in System Settings to continue. -

- )} +
+
+
+ + {allGranted ? "\u2713" : "1"} + +
+

Accessibility

- If SurfSense doesn't appear in the list, click + and select it from Applications. + Lets SurfSense insert suggestions seamlessly, right where you're typing.

-
- )} +
- ); - })} + +
+ {!allGranted && ( +
+ + {permissions.accessibility === "denied" && ( +

+ Toggle SurfSense on in System Settings to continue. +

+ )} +

+ If SurfSense doesn't appear in the list, click + and select it from Applications. +

+
+ )} + {/* Footer */} @@ -198,7 +168,7 @@ export default function DesktopPermissionsPage() { ) : ( <> - {permissions.accessibility === "denied" && ( -

- Toggle SurfSense on in System Settings to continue. -

+ {!isGranted && ( +
+ + {status === "denied" && ( +

+ Toggle SurfSense on in System Settings to continue. +

+ )} +

+ If SurfSense doesn't appear in the list, click + and select it from Applications. +

+
)} -

- If SurfSense doesn't appear in the list, click + and select it from Applications. -

- )} - + ); + })} {/* Footer */} @@ -168,7 +198,7 @@ export default function DesktopPermissionsPage() { ) : ( <> + ); diff --git a/surfsense_web/app/desktop/suggestion/suggestion.css b/surfsense_web/app/desktop/suggestion/suggestion.css index 0d3332103..712d12618 100644 --- a/surfsense_web/app/desktop/suggestion/suggestion.css +++ b/surfsense_web/app/desktop/suggestion/suggestion.css @@ -36,32 +36,44 @@ html, body { white-space: pre-wrap; } -.suggestion-hint { - color: #666; - font-size: 11px; +.suggestion-actions { display: flex; - align-items: center; - gap: 6px; + justify-content: flex-end; + gap: 4px; border-top: 1px solid #2a2a2a; padding-top: 6px; } -.suggestion-hint kbd { - background: #2a2a2a; - border: 1px solid #3c3c3c; +.suggestion-btn { + padding: 2px 8px; border-radius: 3px; - padding: 0 4px; + border: 1px solid #3c3c3c; font-family: inherit; font-size: 10px; - font-weight: 600; - color: #999; - line-height: 18px; + font-weight: 500; + cursor: pointer; + line-height: 16px; + transition: background 0.15s, border-color 0.15s; } -.suggestion-separator { - width: 1px; - height: 10px; +.suggestion-btn-accept { + background: #2563eb; + border-color: #3b82f6; + color: #fff; +} + +.suggestion-btn-accept:hover { + background: #1d4ed8; +} + +.suggestion-btn-dismiss { + background: #2a2a2a; + color: #999; +} + +.suggestion-btn-dismiss:hover { background: #333; + color: #ccc; } .suggestion-error { diff --git a/surfsense_web/types/window.d.ts b/surfsense_web/types/window.d.ts index a5b8566f9..dc3a6465e 100644 --- a/surfsense_web/types/window.d.ts +++ b/surfsense_web/types/window.d.ts @@ -17,8 +17,10 @@ interface ElectronAPI { // Permissions getPermissionsStatus: () => Promise<{ accessibility: 'authorized' | 'denied' | 'not determined' | 'restricted' | 'limited'; + screenRecording: 'authorized' | 'denied' | 'not determined' | 'restricted' | 'limited'; }>; requestAccessibility: () => Promise; + requestScreenRecording: () => Promise; restartApp: () => Promise; // Autocomplete onAutocompleteContext: (callback: (data: { screenshot: string; searchSpaceId?: string }) => void) => () => void; From 080acf5e0a18b62953cffcf19806633cc9122c98 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 3 Apr 2026 20:16:23 +0200 Subject: [PATCH 21/33] remove dead code: allPermissionsGranted, simulateCopy, pendingSuggestionText, updateSuggestionText --- surfsense_desktop/src/ipc/channels.ts | 1 - surfsense_desktop/src/modules/autocomplete/index.ts | 7 ------- surfsense_desktop/src/modules/permissions.ts | 5 ----- surfsense_desktop/src/modules/platform.ts | 8 -------- surfsense_desktop/src/preload.ts | 1 - surfsense_web/app/desktop/suggestion/page.tsx | 6 +----- surfsense_web/types/window.d.ts | 1 - 7 files changed, 1 insertion(+), 28 deletions(-) diff --git a/surfsense_desktop/src/ipc/channels.ts b/surfsense_desktop/src/ipc/channels.ts index e41355eaf..d0e2b9ab4 100644 --- a/surfsense_desktop/src/ipc/channels.ts +++ b/surfsense_desktop/src/ipc/channels.ts @@ -15,7 +15,6 @@ export const IPC_CHANNELS = { AUTOCOMPLETE_CONTEXT: 'autocomplete-context', ACCEPT_SUGGESTION: 'accept-suggestion', DISMISS_SUGGESTION: 'dismiss-suggestion', - UPDATE_SUGGESTION_TEXT: 'update-suggestion-text', SET_AUTOCOMPLETE_ENABLED: 'set-autocomplete-enabled', GET_AUTOCOMPLETE_ENABLED: 'get-autocomplete-enabled', } as const; diff --git a/surfsense_desktop/src/modules/autocomplete/index.ts b/surfsense_desktop/src/modules/autocomplete/index.ts index 958886b63..3ed9c4a00 100644 --- a/surfsense_desktop/src/modules/autocomplete/index.ts +++ b/surfsense_desktop/src/modules/autocomplete/index.ts @@ -11,7 +11,6 @@ const SHORTCUT = 'CommandOrControl+Shift+Space'; let autocompleteEnabled = true; let savedClipboard = ''; let sourceApp = ''; -let pendingSuggestionText = ''; function isSurfSenseWindow(): boolean { const app = getFrontmostApp(); @@ -72,7 +71,6 @@ async function acceptAndInject(text: string): Promise { clipboard.writeText(text); destroySuggestion(); - pendingSuggestionText = ''; try { await new Promise((r) => setTimeout(r, 50)); @@ -90,10 +88,6 @@ function registerIpcHandlers(): void { }); ipcMain.handle(IPC_CHANNELS.DISMISS_SUGGESTION, () => { destroySuggestion(); - pendingSuggestionText = ''; - }); - ipcMain.handle(IPC_CHANNELS.UPDATE_SUGGESTION_TEXT, (_event, text: string) => { - pendingSuggestionText = text; }); ipcMain.handle(IPC_CHANNELS.SET_AUTOCOMPLETE_ENABLED, (_event, enabled: boolean) => { autocompleteEnabled = enabled; @@ -111,7 +105,6 @@ export function registerAutocomplete(): void { const sw = getSuggestionWindow(); if (sw && !sw.isDestroyed()) { destroySuggestion(); - pendingSuggestionText = ''; return; } triggerAutocomplete(); diff --git a/surfsense_desktop/src/modules/permissions.ts b/surfsense_desktop/src/modules/permissions.ts index a2f057795..02786113e 100644 --- a/surfsense_desktop/src/modules/permissions.ts +++ b/surfsense_desktop/src/modules/permissions.ts @@ -27,11 +27,6 @@ export function getPermissionsStatus(): PermissionsStatus { }; } -export function allPermissionsGranted(): boolean { - const status = getPermissionsStatus(); - return status.accessibility === 'authorized' && status.screenRecording === 'authorized'; -} - export function requestAccessibility(): void { if (!isMac()) return; const perms = getNodeMacPermissions(); diff --git a/surfsense_desktop/src/modules/platform.ts b/surfsense_desktop/src/modules/platform.ts index 1ab0c38fb..1e6ac74e4 100644 --- a/surfsense_desktop/src/modules/platform.ts +++ b/surfsense_desktop/src/modules/platform.ts @@ -19,14 +19,6 @@ export function getFrontmostApp(): string { return ''; } -export function simulateCopy(): void { - if (process.platform === 'darwin') { - execSync('osascript -e \'tell application "System Events" to keystroke "c" using command down\''); - } else if (process.platform === 'win32') { - execSync('powershell -command "Add-Type -AssemblyName System.Windows.Forms; [System.Windows.Forms.SendKeys]::SendWait(\'^c\')"'); - } -} - export function simulatePaste(): void { if (process.platform === 'darwin') { execSync('osascript -e \'tell application "System Events" to keystroke "v" using command down\''); diff --git a/surfsense_desktop/src/preload.ts b/surfsense_desktop/src/preload.ts index 5c8b64f6f..31c5ca865 100644 --- a/surfsense_desktop/src/preload.ts +++ b/surfsense_desktop/src/preload.ts @@ -36,7 +36,6 @@ contextBridge.exposeInMainWorld('electronAPI', { }, acceptSuggestion: (text: string) => ipcRenderer.invoke(IPC_CHANNELS.ACCEPT_SUGGESTION, text), dismissSuggestion: () => ipcRenderer.invoke(IPC_CHANNELS.DISMISS_SUGGESTION), - updateSuggestionText: (text: string) => ipcRenderer.invoke(IPC_CHANNELS.UPDATE_SUGGESTION_TEXT, text), setAutocompleteEnabled: (enabled: boolean) => ipcRenderer.invoke(IPC_CHANNELS.SET_AUTOCOMPLETE_ENABLED, enabled), getAutocompleteEnabled: () => ipcRenderer.invoke(IPC_CHANNELS.GET_AUTOCOMPLETE_ENABLED), }); diff --git a/surfsense_web/app/desktop/suggestion/page.tsx b/surfsense_web/app/desktop/suggestion/page.tsx index 4de90e03c..b68fe450d 100644 --- a/surfsense_web/app/desktop/suggestion/page.tsx +++ b/surfsense_web/app/desktop/suggestion/page.tsx @@ -116,11 +116,7 @@ export default function SuggestionPage() { try { const parsed: SSEEvent = JSON.parse(data); if (parsed.type === "text-delta") { - setSuggestion((prev) => { - const updated = prev + parsed.delta; - window.electronAPI?.updateSuggestionText?.(updated); - return updated; - }); + setSuggestion((prev) => prev + parsed.delta); } else if (parsed.type === "error") { setError(friendlyError(parsed.errorText)); } diff --git a/surfsense_web/types/window.d.ts b/surfsense_web/types/window.d.ts index dc3a6465e..2fc550306 100644 --- a/surfsense_web/types/window.d.ts +++ b/surfsense_web/types/window.d.ts @@ -26,7 +26,6 @@ interface ElectronAPI { onAutocompleteContext: (callback: (data: { screenshot: string; searchSpaceId?: string }) => void) => () => void; acceptSuggestion: (text: string) => Promise; dismissSuggestion: () => Promise; - updateSuggestionText: (text: string) => Promise; setAutocompleteEnabled: (enabled: boolean) => Promise; getAutocompleteEnabled: () => Promise; } From 960b8fc01273795407398fe8b108ebbd6009ef58 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 3 Apr 2026 21:34:01 +0200 Subject: [PATCH 22/33] add app context and KB grounding to autocomplete suggestions --- .../app/routes/autocomplete_routes.py | 7 +- .../services/vision_autocomplete_service.py | 117 +++++++++++++++++- .../src/modules/autocomplete/index.ts | 5 +- surfsense_desktop/src/modules/platform.ts | 18 +++ surfsense_desktop/src/preload.ts | 4 +- surfsense_web/app/desktop/suggestion/page.tsx | 6 +- surfsense_web/types/window.d.ts | 2 +- 7 files changed, 150 insertions(+), 9 deletions(-) diff --git a/surfsense_backend/app/routes/autocomplete_routes.py b/surfsense_backend/app/routes/autocomplete_routes.py index 329476ca1..a6f66f408 100644 --- a/surfsense_backend/app/routes/autocomplete_routes.py +++ b/surfsense_backend/app/routes/autocomplete_routes.py @@ -14,6 +14,8 @@ router = APIRouter(prefix="/autocomplete", tags=["autocomplete"]) class VisionAutocompleteRequest(BaseModel): screenshot: str search_space_id: int + app_name: str = "" + window_title: str = "" @router.post("/vision/stream") @@ -23,7 +25,10 @@ async def vision_autocomplete_stream( session: AsyncSession = Depends(get_async_session), ): return StreamingResponse( - stream_vision_autocomplete(body.screenshot, body.search_space_id, session), + stream_vision_autocomplete( + body.screenshot, body.search_space_id, session, + app_name=body.app_name, window_title=body.window_title, + ), media_type="text/event-stream", headers={ **VercelStreamingService.get_response_headers(), diff --git a/surfsense_backend/app/services/vision_autocomplete_service.py b/surfsense_backend/app/services/vision_autocomplete_service.py index 526b0d35c..0804df7fb 100644 --- a/surfsense_backend/app/services/vision_autocomplete_service.py +++ b/surfsense_backend/app/services/vision_autocomplete_service.py @@ -4,11 +4,21 @@ from typing import AsyncGenerator from langchain_core.messages import HumanMessage, SystemMessage from sqlalchemy.ext.asyncio import AsyncSession +from app.retriever.chunks_hybrid_search import ChucksHybridSearchRetriever from app.services.llm_service import get_vision_llm from app.services.new_streaming_service import VercelStreamingService logger = logging.getLogger(__name__) +KB_TOP_K = 5 +KB_MAX_CHARS = 4000 + +EXTRACT_QUERY_PROMPT = """Look at this screenshot and describe in 1-2 short sentences what the user is working on and what topic they need to write about. Be specific about the subject matter. Output ONLY the description, nothing else.""" + +EXTRACT_QUERY_PROMPT_WITH_APP = """The user is currently in the application "{app_name}" with the window titled "{window_title}". + +Look at this screenshot and describe in 1-2 short sentences what the user is working on and what topic they need to write about. Be specific about the subject matter. Output ONLY the description, nothing else.""" + VISION_SYSTEM_PROMPT = """You are a smart writing assistant that analyzes the user's screen to draft or complete text. You will receive a screenshot of the user's screen. Your job: @@ -28,13 +38,107 @@ Rules: - Do NOT describe the screenshot or explain your reasoning. - If you cannot determine what to write, output nothing.""" +APP_CONTEXT_BLOCK = """ + +The user is currently working in "{app_name}" (window: "{window_title}"). Use this to understand the type of application and adapt your tone and format accordingly.""" + +KB_CONTEXT_BLOCK = """ + +You also have access to the user's knowledge base documents below. Use them to write more accurate, informed, and contextually relevant text. Do NOT cite or reference the documents explicitly — just let the knowledge inform your writing naturally. + + +{kb_context} +""" + + +def _build_system_prompt(app_name: str, window_title: str, kb_context: str) -> str: + """Assemble the system prompt from optional context blocks.""" + prompt = VISION_SYSTEM_PROMPT + if app_name: + prompt += APP_CONTEXT_BLOCK.format(app_name=app_name, window_title=window_title) + if kb_context: + prompt += KB_CONTEXT_BLOCK.format(kb_context=kb_context) + return prompt + + +async def _extract_query_from_screenshot( + llm, screenshot_data_url: str, + app_name: str = "", window_title: str = "", +) -> str | None: + """Ask the Vision LLM to describe what the user is working on.""" + if app_name: + prompt_text = EXTRACT_QUERY_PROMPT_WITH_APP.format( + app_name=app_name, window_title=window_title, + ) + else: + prompt_text = EXTRACT_QUERY_PROMPT + + try: + response = await llm.ainvoke([ + HumanMessage(content=[ + {"type": "text", "text": prompt_text}, + {"type": "image_url", "image_url": {"url": screenshot_data_url}}, + ]), + ]) + query = response.content.strip() if hasattr(response, "content") else "" + return query if query else None + except Exception as e: + logger.warning(f"Failed to extract query from screenshot: {e}") + return None + + +async def _search_knowledge_base( + session: AsyncSession, search_space_id: int, query: str +) -> str: + """Search the KB and return formatted context string.""" + try: + retriever = ChucksHybridSearchRetriever(session) + results = await retriever.hybrid_search( + query_text=query, + top_k=KB_TOP_K, + search_space_id=search_space_id, + ) + + if not results: + return "" + + parts: list[str] = [] + char_count = 0 + for doc in results: + title = doc.get("document", {}).get("title", "Untitled") + for chunk in doc.get("chunks", []): + content = chunk.get("content", "").strip() + if not content: + continue + entry = f"[{title}]\n{content}" + if char_count + len(entry) > KB_MAX_CHARS: + break + parts.append(entry) + char_count += len(entry) + if char_count >= KB_MAX_CHARS: + break + + return "\n\n---\n\n".join(parts) + except Exception as e: + logger.warning(f"KB search failed, proceeding without context: {e}") + return "" + async def stream_vision_autocomplete( screenshot_data_url: str, search_space_id: int, session: AsyncSession, + *, + app_name: str = "", + window_title: str = "", ) -> AsyncGenerator[str, None]: - """Analyze a screenshot with the vision LLM and stream a text completion.""" + """Analyze a screenshot with the vision LLM and stream a text completion. + + Pipeline: + 1. Extract a search query from the screenshot (non-streaming) + 2. Search the knowledge base for relevant context + 3. Stream the final completion with screenshot + KB + app context + """ streaming = VercelStreamingService() llm = await get_vision_llm(session, search_space_id) @@ -44,8 +148,17 @@ async def stream_vision_autocomplete( yield streaming.format_done() return + kb_context = "" + query = await _extract_query_from_screenshot( + llm, screenshot_data_url, app_name=app_name, window_title=window_title, + ) + if query: + kb_context = await _search_knowledge_base(session, search_space_id, query) + + system_prompt = _build_system_prompt(app_name, window_title, kb_context) + messages = [ - SystemMessage(content=VISION_SYSTEM_PROMPT), + SystemMessage(content=system_prompt), HumanMessage(content=[ { "type": "text", diff --git a/surfsense_desktop/src/modules/autocomplete/index.ts b/surfsense_desktop/src/modules/autocomplete/index.ts index 3ed9c4a00..0d5073de4 100644 --- a/surfsense_desktop/src/modules/autocomplete/index.ts +++ b/surfsense_desktop/src/modules/autocomplete/index.ts @@ -1,6 +1,6 @@ import { clipboard, globalShortcut, ipcMain, screen } from 'electron'; import { IPC_CHANNELS } from '../../ipc/channels'; -import { getFrontmostApp, hasAccessibilityPermission, simulatePaste } from '../platform'; +import { getFrontmostApp, getWindowTitle, hasAccessibilityPermission, simulatePaste } from '../platform'; import { hasScreenRecordingPermission, requestAccessibility, requestScreenRecording } from '../permissions'; import { getMainWindow } from '../window'; import { captureScreen } from './screenshot'; @@ -27,6 +27,7 @@ async function triggerAutocomplete(): Promise { } sourceApp = getFrontmostApp(); + const windowTitle = getWindowTitle(); savedClipboard = clipboard.readText(); const screenshot = await captureScreen(); @@ -55,6 +56,8 @@ async function triggerAutocomplete(): Promise { sw.webContents.send(IPC_CHANNELS.AUTOCOMPLETE_CONTEXT, { screenshot, searchSpaceId, + appName: sourceApp, + windowTitle, }); } }, 300); diff --git a/surfsense_desktop/src/modules/platform.ts b/surfsense_desktop/src/modules/platform.ts index 1e6ac74e4..122e2efed 100644 --- a/surfsense_desktop/src/modules/platform.ts +++ b/surfsense_desktop/src/modules/platform.ts @@ -32,6 +32,24 @@ export function checkAccessibilityPermission(): boolean { return systemPreferences.isTrustedAccessibilityClient(true); } +export function getWindowTitle(): string { + try { + if (process.platform === 'darwin') { + return execSync( + 'osascript -e \'tell application "System Events" to get title of front window of first application process whose frontmost is true\'' + ).toString().trim(); + } + if (process.platform === 'win32') { + return execSync( + 'powershell -command "(Get-Process | Where-Object { $_.MainWindowHandle -eq (Add-Type -MemberDefinition \'[DllImport(\\\"user32.dll\\\")] public static extern IntPtr GetForegroundWindow();\' -Name W -PassThru)::GetForegroundWindow() }).MainWindowTitle"' + ).toString().trim(); + } + } catch { + return ''; + } + return ''; +} + export function hasAccessibilityPermission(): boolean { if (process.platform !== 'darwin') return true; return systemPreferences.isTrustedAccessibilityClient(false); diff --git a/surfsense_desktop/src/preload.ts b/surfsense_desktop/src/preload.ts index 31c5ca865..2bd09f13c 100644 --- a/surfsense_desktop/src/preload.ts +++ b/surfsense_desktop/src/preload.ts @@ -27,8 +27,8 @@ contextBridge.exposeInMainWorld('electronAPI', { requestScreenRecording: () => ipcRenderer.invoke(IPC_CHANNELS.REQUEST_SCREEN_RECORDING), restartApp: () => ipcRenderer.invoke(IPC_CHANNELS.RESTART_APP), // Autocomplete - onAutocompleteContext: (callback: (data: { screenshot: string; searchSpaceId?: string }) => void) => { - const listener = (_event: unknown, data: { screenshot: string; searchSpaceId?: string }) => callback(data); + onAutocompleteContext: (callback: (data: { screenshot: string; searchSpaceId?: string; appName?: string; windowTitle?: string }) => void) => { + const listener = (_event: unknown, data: { screenshot: string; searchSpaceId?: string; appName?: string; windowTitle?: string }) => callback(data); ipcRenderer.on(IPC_CHANNELS.AUTOCOMPLETE_CONTEXT, listener); return () => { ipcRenderer.removeListener(IPC_CHANNELS.AUTOCOMPLETE_CONTEXT, listener); diff --git a/surfsense_web/app/desktop/suggestion/page.tsx b/surfsense_web/app/desktop/suggestion/page.tsx index b68fe450d..b7d9b97bd 100644 --- a/surfsense_web/app/desktop/suggestion/page.tsx +++ b/surfsense_web/app/desktop/suggestion/page.tsx @@ -46,7 +46,7 @@ export default function SuggestionPage() { }, [error]); const fetchSuggestion = useCallback( - async (screenshot: string, searchSpaceId: string) => { + async (screenshot: string, searchSpaceId: string, appName?: string, windowTitle?: string) => { abortRef.current?.abort(); const controller = new AbortController(); abortRef.current = controller; @@ -77,6 +77,8 @@ export default function SuggestionPage() { body: JSON.stringify({ screenshot, search_space_id: parseInt(searchSpaceId, 10), + app_name: appName || "", + window_title: windowTitle || "", }), signal: controller.signal, }, @@ -142,7 +144,7 @@ export default function SuggestionPage() { const cleanup = window.electronAPI.onAutocompleteContext((data) => { const searchSpaceId = data.searchSpaceId || "1"; if (data.screenshot) { - fetchSuggestion(data.screenshot, searchSpaceId); + fetchSuggestion(data.screenshot, searchSpaceId, data.appName, data.windowTitle); } }); diff --git a/surfsense_web/types/window.d.ts b/surfsense_web/types/window.d.ts index 2fc550306..85b6bdf51 100644 --- a/surfsense_web/types/window.d.ts +++ b/surfsense_web/types/window.d.ts @@ -23,7 +23,7 @@ interface ElectronAPI { requestScreenRecording: () => Promise; restartApp: () => Promise; // Autocomplete - onAutocompleteContext: (callback: (data: { screenshot: string; searchSpaceId?: string }) => void) => () => void; + onAutocompleteContext: (callback: (data: { screenshot: string; searchSpaceId?: string; appName?: string; windowTitle?: string }) => void) => () => void; acceptSuggestion: (text: string) => Promise; dismissSuggestion: () => Promise; setAutocompleteEnabled: (enabled: boolean) => Promise; From 0cd997f67325be3870022ec5917e28d3fa9712a9 Mon Sep 17 00:00:00 2001 From: SohamBhattacharjee2003 <125297948+SohamBhattacharjee2003@users.noreply.github.com> Date: Sat, 4 Apr 2026 01:28:45 +0530 Subject: [PATCH 23/33] fix: resolve setTimeout memory leak on unmounted component --- .../components/report-panel/report-panel.tsx | 10 +++++++++- surfsense_web/hooks/use-api-key.ts | 12 ++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/surfsense_web/components/report-panel/report-panel.tsx b/surfsense_web/components/report-panel/report-panel.tsx index 3d48fb8c5..c602a3f04 100644 --- a/surfsense_web/components/report-panel/report-panel.tsx +++ b/surfsense_web/components/report-panel/report-panel.tsx @@ -123,6 +123,13 @@ export function ReportPanelContent({ const [copied, setCopied] = useState(false); const [exporting, setExporting] = useState(null); const [saving, setSaving] = useState(false); + const copyTimerRef = useRef | undefined>(undefined); + + useEffect(() => { + return () => { + if (copyTimerRef.current) clearTimeout(copyTimerRef.current); + }; + }, []); // Editor state — tracks the latest markdown from the Plate editor const [editedMarkdown, setEditedMarkdown] = useState(null); @@ -197,7 +204,8 @@ export function ReportPanelContent({ try { await navigator.clipboard.writeText(currentMarkdown); setCopied(true); - setTimeout(() => setCopied(false), 2000); + if (copyTimerRef.current) clearTimeout(copyTimerRef.current); + copyTimerRef.current = setTimeout(() => setCopied(false), 2000); } catch (err) { console.error("Failed to copy:", err); } diff --git a/surfsense_web/hooks/use-api-key.ts b/surfsense_web/hooks/use-api-key.ts index 0c595b420..b50dd65f1 100644 --- a/surfsense_web/hooks/use-api-key.ts +++ b/surfsense_web/hooks/use-api-key.ts @@ -1,4 +1,4 @@ -import { useCallback, useEffect, useState } from "react"; +import { useCallback, useEffect, useRef, useState } from "react"; import { toast } from "sonner"; import { getBearerToken } from "@/lib/auth-utils"; import { copyToClipboard as copyToClipboardUtil } from "@/lib/utils"; @@ -14,6 +14,13 @@ export function useApiKey(): UseApiKeyReturn { const [apiKey, setApiKey] = useState(null); const [copied, setCopied] = useState(false); const [isLoading, setIsLoading] = useState(true); + const copyTimerRef = useRef | undefined>(undefined); + + useEffect(() => { + return () => { + if (copyTimerRef.current) clearTimeout(copyTimerRef.current); + }; + }, []); useEffect(() => { // Load API key from localStorage @@ -41,7 +48,8 @@ export function useApiKey(): UseApiKeyReturn { if (success) { setCopied(true); toast.success("API key copied to clipboard"); - setTimeout(() => { + if (copyTimerRef.current) clearTimeout(copyTimerRef.current); + copyTimerRef.current = setTimeout(() => { setCopied(false); }, 2000); } else { From ce40da80ea9f2a2afd02bd44aeb5390ac565fcff Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 4 Apr 2026 02:51:28 +0530 Subject: [PATCH 24/33] feat: implement page limit estimation and enforcement in file based connector indexers - Added a static method `estimate_pages_from_metadata` to `PageLimitService` for estimating page counts based on file metadata. - Integrated page limit checks in Google Drive, Dropbox, and OneDrive indexers to prevent exceeding user quotas during file indexing. - Updated relevant indexing methods to utilize the new page estimation logic and enforce limits accordingly. - Enhanced tests for page limit functionality, ensuring accurate estimation and enforcement across different file types. --- .../app/services/page_limit_service.py | 239 +++---- .../app/tasks/connector_indexers/base.py | 1 - .../connector_indexers/dropbox_indexer.py | 52 ++ .../google_drive_indexer.py | 88 +++ .../connector_indexers/onedrive_indexer.py | 80 +++ .../integration/document_upload/conftest.py | 63 ++ .../test_google_drive_parallel.py | 27 +- .../connector_indexers/test_page_limits.py | 648 ++++++++++++++++++ 8 files changed, 1041 insertions(+), 157 deletions(-) create mode 100644 surfsense_backend/tests/unit/connector_indexers/test_page_limits.py diff --git a/surfsense_backend/app/services/page_limit_service.py b/surfsense_backend/app/services/page_limit_service.py index 080d05b5d..ea22067be 100644 --- a/surfsense_backend/app/services/page_limit_service.py +++ b/surfsense_backend/app/services/page_limit_service.py @@ -3,7 +3,7 @@ Service for managing user page limits for ETL services. """ import os -from pathlib import Path +from pathlib import Path, PurePosixPath from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession @@ -223,10 +223,91 @@ class PageLimitService: # Estimate ~2000 characters per page return max(1, content_length // 2000) + @staticmethod + def estimate_pages_from_metadata( + file_name_or_ext: str, file_size: int | str | None = None + ) -> int: + """Size-based page estimation from file name/extension and byte size. + + Pure function — no file I/O, no database access. Used by cloud + connectors (which only have API metadata) and as the internal + fallback for :meth:`estimate_pages_before_processing`. + + ``file_name_or_ext`` can be a full filename (``"report.pdf"``) or + a bare extension (``".pdf"``). ``file_size`` may be an int, a + stringified int from a cloud API, or *None*. + """ + if file_size is not None: + try: + file_size = int(file_size) + except (ValueError, TypeError): + file_size = 0 + else: + file_size = 0 + + if file_size <= 0: + return 1 + + ext = PurePosixPath(file_name_or_ext).suffix.lower() if file_name_or_ext else "" + if not ext and file_name_or_ext.startswith("."): + ext = file_name_or_ext.lower() + file_ext = ext + + if file_ext == ".pdf": + return max(1, file_size // (100 * 1024)) + + if file_ext in { + ".doc", ".docx", ".docm", ".dot", ".dotm", + ".odt", ".ott", ".sxw", ".stw", ".uot", + ".rtf", ".pages", ".wpd", ".wps", + ".abw", ".zabw", ".cwk", ".hwp", ".lwp", + ".mcw", ".mw", ".sdw", ".vor", + }: + return max(1, file_size // (50 * 1024)) + + if file_ext in { + ".ppt", ".pptx", ".pptm", ".pot", ".potx", + ".odp", ".otp", ".sxi", ".sti", ".uop", + ".key", ".sda", ".sdd", ".sdp", + }: + return max(1, file_size // (200 * 1024)) + + if file_ext in { + ".xls", ".xlsx", ".xlsm", ".xlsb", ".xlw", ".xlr", + ".ods", ".ots", ".fods", ".numbers", + ".123", ".wk1", ".wk2", ".wk3", ".wk4", ".wks", + ".wb1", ".wb2", ".wb3", ".wq1", ".wq2", + ".csv", ".tsv", ".slk", ".sylk", ".dif", ".dbf", + ".prn", ".qpw", ".602", ".et", ".eth", + }: + return max(1, file_size // (100 * 1024)) + + if file_ext in {".epub"}: + return max(1, file_size // (50 * 1024)) + + if file_ext in {".txt", ".log", ".md", ".markdown", ".htm", ".html", ".xml"}: + return max(1, file_size // 3000) + + if file_ext in { + ".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", + ".webp", ".svg", ".cgm", ".odg", ".pbd", + }: + return 1 + + if file_ext in {".mp3", ".m4a", ".wav", ".mpga"}: + return max(1, file_size // (1024 * 1024)) + + if file_ext in {".mp4", ".mpeg", ".webm"}: + return max(1, file_size // (5 * 1024 * 1024)) + + return max(1, file_size // (80 * 1024)) + def estimate_pages_before_processing(self, file_path: str) -> int: """ - Estimate page count from file before processing (to avoid unnecessary API calls). - This is called BEFORE sending to ETL services to prevent cost on rejected files. + Estimate page count from a local file before processing. + + For PDFs, attempts to read the actual page count via pypdf. + For everything else, delegates to :meth:`estimate_pages_from_metadata`. Args: file_path: Path to the file @@ -240,7 +321,6 @@ class PageLimitService: file_ext = Path(file_path).suffix.lower() file_size = os.path.getsize(file_path) - # PDF files - try to get actual page count if file_ext == ".pdf": try: import pypdf @@ -249,153 +329,6 @@ class PageLimitService: pdf_reader = pypdf.PdfReader(f) return len(pdf_reader.pages) except Exception: - # If PDF reading fails, fall back to size estimation - # Typical PDF: ~100KB per page (conservative estimate) - return max(1, file_size // (100 * 1024)) + pass # fall through to size-based estimation - # Word Processing Documents - # Microsoft Word, LibreOffice Writer, WordPerfect, Pages, etc. - elif file_ext in [ - ".doc", - ".docx", - ".docm", - ".dot", - ".dotm", # Microsoft Word - ".odt", - ".ott", - ".sxw", - ".stw", - ".uot", # OpenDocument/StarOffice Writer - ".rtf", # Rich Text Format - ".pages", # Apple Pages - ".wpd", - ".wps", # WordPerfect, Microsoft Works - ".abw", - ".zabw", # AbiWord - ".cwk", - ".hwp", - ".lwp", - ".mcw", - ".mw", - ".sdw", - ".vor", # Other word processors - ]: - # Typical word document: ~50KB per page (conservative) - return max(1, file_size // (50 * 1024)) - - # Presentation Documents - # PowerPoint, Impress, Keynote, etc. - elif file_ext in [ - ".ppt", - ".pptx", - ".pptm", - ".pot", - ".potx", # Microsoft PowerPoint - ".odp", - ".otp", - ".sxi", - ".sti", - ".uop", # OpenDocument/StarOffice Impress - ".key", # Apple Keynote - ".sda", - ".sdd", - ".sdp", # StarOffice Draw/Impress - ]: - # Typical presentation: ~200KB per slide (conservative) - return max(1, file_size // (200 * 1024)) - - # Spreadsheet Documents - # Excel, Calc, Numbers, Lotus, etc. - elif file_ext in [ - ".xls", - ".xlsx", - ".xlsm", - ".xlsb", - ".xlw", - ".xlr", # Microsoft Excel - ".ods", - ".ots", - ".fods", # OpenDocument Spreadsheet - ".numbers", # Apple Numbers - ".123", - ".wk1", - ".wk2", - ".wk3", - ".wk4", - ".wks", # Lotus 1-2-3 - ".wb1", - ".wb2", - ".wb3", - ".wq1", - ".wq2", # Quattro Pro - ".csv", - ".tsv", - ".slk", - ".sylk", - ".dif", - ".dbf", - ".prn", - ".qpw", # Data formats - ".602", - ".et", - ".eth", # Other spreadsheets - ]: - # Spreadsheets typically have 1 sheet = 1 page for ETL - # Conservative: ~100KB per sheet - return max(1, file_size // (100 * 1024)) - - # E-books - elif file_ext in [".epub"]: - # E-books vary widely, estimate by size - # Typical e-book: ~50KB per page - return max(1, file_size // (50 * 1024)) - - # Plain Text and Markup Files - elif file_ext in [ - ".txt", - ".log", # Plain text - ".md", - ".markdown", # Markdown - ".htm", - ".html", - ".xml", # Markup - ]: - # Plain text: ~3000 bytes per page - return max(1, file_size // 3000) - - # Image Files - # Each image is typically processed as 1 page - elif file_ext in [ - ".jpg", - ".jpeg", # JPEG - ".png", # PNG - ".gif", # GIF - ".bmp", # Bitmap - ".tiff", # TIFF - ".webp", # WebP - ".svg", # SVG - ".cgm", # Computer Graphics Metafile - ".odg", - ".pbd", # OpenDocument Graphics - ]: - # Each image = 1 page - return 1 - - # Audio Files (transcription = typically 1 page per minute) - # Note: These should be handled by audio transcription flow, not ETL - elif file_ext in [".mp3", ".m4a", ".wav", ".mpga"]: - # Audio files: estimate based on duration - # Fallback: ~1MB per minute of audio, 1 page per minute transcript - return max(1, file_size // (1024 * 1024)) - - # Video Files (typically not processed for pages, but just in case) - elif file_ext in [".mp4", ".mpeg", ".webm"]: - # Video files: very rough estimate - # Typically wouldn't be page-based, but use conservative estimate - return max(1, file_size // (5 * 1024 * 1024)) - - # Other/Unknown Document Types - else: - # Conservative estimate: ~80KB per page - # This catches: .sgl, .sxg, .uof, .uos1, .uos2, .web, and any future formats - return max(1, file_size // (80 * 1024)) + return self.estimate_pages_from_metadata(file_ext, file_size) diff --git a/surfsense_backend/app/tasks/connector_indexers/base.py b/surfsense_backend/app/tasks/connector_indexers/base.py index ffc8ab72e..6b4bed4b5 100644 --- a/surfsense_backend/app/tasks/connector_indexers/base.py +++ b/surfsense_backend/app/tasks/connector_indexers/base.py @@ -4,7 +4,6 @@ Base functionality and shared imports for connector indexers. import logging from datetime import UTC, datetime, timedelta - from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select diff --git a/surfsense_backend/app/tasks/connector_indexers/dropbox_indexer.py b/surfsense_backend/app/tasks/connector_indexers/dropbox_indexer.py index 1b039add7..87b3c55df 100644 --- a/surfsense_backend/app/tasks/connector_indexers/dropbox_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/dropbox_indexer.py @@ -28,6 +28,7 @@ from app.indexing_pipeline.connector_document import ConnectorDocument from app.indexing_pipeline.document_hashing import compute_identifier_hash from app.indexing_pipeline.indexing_pipeline_service import IndexingPipelineService from app.services.llm_service import get_user_long_context_llm +from app.services.page_limit_service import PageLimitService from app.services.task_logging_service import TaskLoggingService from app.tasks.connector_indexers.base import ( check_document_by_unique_identifier, @@ -278,6 +279,12 @@ async def _index_full_scan( }, ) + page_limit_service = PageLimitService(session) + pages_used, pages_limit = await page_limit_service.get_page_usage(user_id) + remaining_quota = pages_limit - pages_used + batch_estimated_pages = 0 + page_limit_reached = False + renamed_count = 0 skipped = 0 files_to_download: list[dict] = [] @@ -307,6 +314,21 @@ async def _index_full_scan( elif skip_item(file): skipped += 1 continue + + file_pages = PageLimitService.estimate_pages_from_metadata( + file.get("name", ""), file.get("size") + ) + if batch_estimated_pages + file_pages > remaining_quota: + if not page_limit_reached: + logger.warning( + "Page limit reached during Dropbox full scan, " + "skipping remaining files" + ) + page_limit_reached = True + skipped += 1 + continue + + batch_estimated_pages += file_pages files_to_download.append(file) batch_indexed, failed = await _download_and_index( @@ -320,6 +342,14 @@ async def _index_full_scan( on_heartbeat=on_heartbeat_callback, ) + if batch_indexed > 0 and files_to_download and batch_estimated_pages > 0: + pages_to_deduct = max( + 1, batch_estimated_pages * batch_indexed // len(files_to_download) + ) + await page_limit_service.update_page_usage( + user_id, pages_to_deduct, allow_exceed=True + ) + indexed = renamed_count + batch_indexed logger.info( f"Full scan complete: {indexed} indexed, {skipped} skipped, {failed} failed" @@ -340,6 +370,11 @@ async def _index_selected_files( on_heartbeat: HeartbeatCallbackType | None = None, ) -> tuple[int, int, list[str]]: """Index user-selected files using the parallel pipeline.""" + page_limit_service = PageLimitService(session) + pages_used, pages_limit = await page_limit_service.get_page_usage(user_id) + remaining_quota = pages_limit - pages_used + batch_estimated_pages = 0 + files_to_download: list[dict] = [] errors: list[str] = [] renamed_count = 0 @@ -364,6 +399,15 @@ async def _index_selected_files( skipped += 1 continue + file_pages = PageLimitService.estimate_pages_from_metadata( + file.get("name", ""), file.get("size") + ) + if batch_estimated_pages + file_pages > remaining_quota: + display = file_name or file_path + errors.append(f"File '{display}': page limit would be exceeded") + continue + + batch_estimated_pages += file_pages files_to_download.append(file) batch_indexed, _failed = await _download_and_index( @@ -377,6 +421,14 @@ async def _index_selected_files( on_heartbeat=on_heartbeat, ) + if batch_indexed > 0 and files_to_download and batch_estimated_pages > 0: + pages_to_deduct = max( + 1, batch_estimated_pages * batch_indexed // len(files_to_download) + ) + await page_limit_service.update_page_usage( + user_id, pages_to_deduct, allow_exceed=True + ) + return renamed_count + batch_indexed, skipped, errors diff --git a/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py b/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py index b03d305f7..5e9e0f62f 100644 --- a/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py @@ -34,6 +34,7 @@ from app.indexing_pipeline.indexing_pipeline_service import ( PlaceholderInfo, ) from app.services.llm_service import get_user_long_context_llm +from app.services.page_limit_service import PageLimitService from app.services.task_logging_service import TaskLoggingService from app.tasks.connector_indexers.base import ( check_document_by_unique_identifier, @@ -327,6 +328,12 @@ async def _process_single_file( return 1, 0, 0 return 0, 1, 0 + page_limit_service = PageLimitService(session) + estimated_pages = PageLimitService.estimate_pages_from_metadata( + file_name, file.get("size") + ) + await page_limit_service.check_page_limit(user_id, estimated_pages) + markdown, drive_metadata, error = await download_and_extract_content( drive_client, file ) @@ -363,6 +370,9 @@ async def _process_single_file( ) await pipeline.index(document, connector_doc, user_llm) + await page_limit_service.update_page_usage( + user_id, estimated_pages, allow_exceed=True + ) logger.info(f"Successfully indexed Google Drive file: {file_name}") return 1, 0, 0 @@ -466,6 +476,11 @@ async def _index_selected_files( Returns (indexed_count, skipped_count, errors). """ + page_limit_service = PageLimitService(session) + pages_used, pages_limit = await page_limit_service.get_page_usage(user_id) + remaining_quota = pages_limit - pages_used + batch_estimated_pages = 0 + files_to_download: list[dict] = [] errors: list[str] = [] renamed_count = 0 @@ -486,6 +501,15 @@ async def _index_selected_files( skipped += 1 continue + file_pages = PageLimitService.estimate_pages_from_metadata( + file.get("name", ""), file.get("size") + ) + if batch_estimated_pages + file_pages > remaining_quota: + display = file_name or file_id + errors.append(f"File '{display}': page limit would be exceeded") + continue + + batch_estimated_pages += file_pages files_to_download.append(file) await _create_drive_placeholders( @@ -507,6 +531,14 @@ async def _index_selected_files( on_heartbeat=on_heartbeat, ) + if batch_indexed > 0 and files_to_download and batch_estimated_pages > 0: + pages_to_deduct = max( + 1, batch_estimated_pages * batch_indexed // len(files_to_download) + ) + await page_limit_service.update_page_usage( + user_id, pages_to_deduct, allow_exceed=True + ) + return renamed_count + batch_indexed, skipped, errors @@ -545,6 +577,12 @@ async def _index_full_scan( # ------------------------------------------------------------------ # Phase 1 (serial): collect files, run skip checks, track renames # ------------------------------------------------------------------ + page_limit_service = PageLimitService(session) + pages_used, pages_limit = await page_limit_service.get_page_usage(user_id) + remaining_quota = pages_limit - pages_used + batch_estimated_pages = 0 + page_limit_reached = False + renamed_count = 0 skipped = 0 files_processed = 0 @@ -593,6 +631,20 @@ async def _index_full_scan( skipped += 1 continue + file_pages = PageLimitService.estimate_pages_from_metadata( + file.get("name", ""), file.get("size") + ) + if batch_estimated_pages + file_pages > remaining_quota: + if not page_limit_reached: + logger.warning( + "Page limit reached during Google Drive full scan, " + "skipping remaining files" + ) + page_limit_reached = True + skipped += 1 + continue + + batch_estimated_pages += file_pages files_to_download.append(file) page_token = next_token @@ -636,6 +688,14 @@ async def _index_full_scan( on_heartbeat=on_heartbeat_callback, ) + if batch_indexed > 0 and files_to_download and batch_estimated_pages > 0: + pages_to_deduct = max( + 1, batch_estimated_pages * batch_indexed // len(files_to_download) + ) + await page_limit_service.update_page_usage( + user_id, pages_to_deduct, allow_exceed=True + ) + indexed = renamed_count + batch_indexed logger.info( f"Full scan complete: {indexed} indexed, {skipped} skipped, {failed} failed" @@ -686,6 +746,12 @@ async def _index_with_delta_sync( # ------------------------------------------------------------------ # Phase 1 (serial): handle removals, collect files for download # ------------------------------------------------------------------ + page_limit_service = PageLimitService(session) + pages_used, pages_limit = await page_limit_service.get_page_usage(user_id) + remaining_quota = pages_limit - pages_used + batch_estimated_pages = 0 + page_limit_reached = False + renamed_count = 0 skipped = 0 files_to_download: list[dict] = [] @@ -715,6 +781,20 @@ async def _index_with_delta_sync( skipped += 1 continue + file_pages = PageLimitService.estimate_pages_from_metadata( + file.get("name", ""), file.get("size") + ) + if batch_estimated_pages + file_pages > remaining_quota: + if not page_limit_reached: + logger.warning( + "Page limit reached during Google Drive delta sync, " + "skipping remaining files" + ) + page_limit_reached = True + skipped += 1 + continue + + batch_estimated_pages += file_pages files_to_download.append(file) # ------------------------------------------------------------------ @@ -742,6 +822,14 @@ async def _index_with_delta_sync( on_heartbeat=on_heartbeat_callback, ) + if batch_indexed > 0 and files_to_download and batch_estimated_pages > 0: + pages_to_deduct = max( + 1, batch_estimated_pages * batch_indexed // len(files_to_download) + ) + await page_limit_service.update_page_usage( + user_id, pages_to_deduct, allow_exceed=True + ) + indexed = renamed_count + batch_indexed logger.info( f"Delta sync complete: {indexed} indexed, {skipped} skipped, {failed} failed" diff --git a/surfsense_backend/app/tasks/connector_indexers/onedrive_indexer.py b/surfsense_backend/app/tasks/connector_indexers/onedrive_indexer.py index 748cb0988..2301b6260 100644 --- a/surfsense_backend/app/tasks/connector_indexers/onedrive_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/onedrive_indexer.py @@ -28,6 +28,7 @@ from app.indexing_pipeline.connector_document import ConnectorDocument from app.indexing_pipeline.document_hashing import compute_identifier_hash from app.indexing_pipeline.indexing_pipeline_service import IndexingPipelineService from app.services.llm_service import get_user_long_context_llm +from app.services.page_limit_service import PageLimitService from app.services.task_logging_service import TaskLoggingService from app.tasks.connector_indexers.base import ( check_document_by_unique_identifier, @@ -291,6 +292,11 @@ async def _index_selected_files( on_heartbeat: HeartbeatCallbackType | None = None, ) -> tuple[int, int, list[str]]: """Index user-selected files using the parallel pipeline.""" + page_limit_service = PageLimitService(session) + pages_used, pages_limit = await page_limit_service.get_page_usage(user_id) + remaining_quota = pages_limit - pages_used + batch_estimated_pages = 0 + files_to_download: list[dict] = [] errors: list[str] = [] renamed_count = 0 @@ -311,6 +317,15 @@ async def _index_selected_files( skipped += 1 continue + file_pages = PageLimitService.estimate_pages_from_metadata( + file.get("name", ""), file.get("size") + ) + if batch_estimated_pages + file_pages > remaining_quota: + display = file_name or file_id + errors.append(f"File '{display}': page limit would be exceeded") + continue + + batch_estimated_pages += file_pages files_to_download.append(file) batch_indexed, _failed = await _download_and_index( @@ -324,6 +339,14 @@ async def _index_selected_files( on_heartbeat=on_heartbeat, ) + if batch_indexed > 0 and files_to_download and batch_estimated_pages > 0: + pages_to_deduct = max( + 1, batch_estimated_pages * batch_indexed // len(files_to_download) + ) + await page_limit_service.update_page_usage( + user_id, pages_to_deduct, allow_exceed=True + ) + return renamed_count + batch_indexed, skipped, errors @@ -358,6 +381,12 @@ async def _index_full_scan( }, ) + page_limit_service = PageLimitService(session) + pages_used, pages_limit = await page_limit_service.get_page_usage(user_id) + remaining_quota = pages_limit - pages_used + batch_estimated_pages = 0 + page_limit_reached = False + renamed_count = 0 skipped = 0 files_to_download: list[dict] = [] @@ -383,6 +412,21 @@ async def _index_full_scan( else: skipped += 1 continue + + file_pages = PageLimitService.estimate_pages_from_metadata( + file.get("name", ""), file.get("size") + ) + if batch_estimated_pages + file_pages > remaining_quota: + if not page_limit_reached: + logger.warning( + "Page limit reached during OneDrive full scan, " + "skipping remaining files" + ) + page_limit_reached = True + skipped += 1 + continue + + batch_estimated_pages += file_pages files_to_download.append(file) batch_indexed, failed = await _download_and_index( @@ -396,6 +440,14 @@ async def _index_full_scan( on_heartbeat=on_heartbeat_callback, ) + if batch_indexed > 0 and files_to_download and batch_estimated_pages > 0: + pages_to_deduct = max( + 1, batch_estimated_pages * batch_indexed // len(files_to_download) + ) + await page_limit_service.update_page_usage( + user_id, pages_to_deduct, allow_exceed=True + ) + indexed = renamed_count + batch_indexed logger.info( f"Full scan complete: {indexed} indexed, {skipped} skipped, {failed} failed" @@ -441,6 +493,12 @@ async def _index_with_delta_sync( logger.info(f"Processing {len(changes)} delta changes") + page_limit_service = PageLimitService(session) + pages_used, pages_limit = await page_limit_service.get_page_usage(user_id) + remaining_quota = pages_limit - pages_used + batch_estimated_pages = 0 + page_limit_reached = False + renamed_count = 0 skipped = 0 files_to_download: list[dict] = [] @@ -471,6 +529,20 @@ async def _index_with_delta_sync( skipped += 1 continue + file_pages = PageLimitService.estimate_pages_from_metadata( + change.get("name", ""), change.get("size") + ) + if batch_estimated_pages + file_pages > remaining_quota: + if not page_limit_reached: + logger.warning( + "Page limit reached during OneDrive delta sync, " + "skipping remaining files" + ) + page_limit_reached = True + skipped += 1 + continue + + batch_estimated_pages += file_pages files_to_download.append(change) batch_indexed, failed = await _download_and_index( @@ -484,6 +556,14 @@ async def _index_with_delta_sync( on_heartbeat=on_heartbeat_callback, ) + if batch_indexed > 0 and files_to_download and batch_estimated_pages > 0: + pages_to_deduct = max( + 1, batch_estimated_pages * batch_indexed // len(files_to_download) + ) + await page_limit_service.update_page_usage( + user_id, pages_to_deduct, allow_exceed=True + ) + indexed = renamed_count + batch_indexed logger.info( f"Delta sync complete: {indexed} indexed, {skipped} skipped, {failed} failed" diff --git a/surfsense_backend/tests/integration/document_upload/conftest.py b/surfsense_backend/tests/integration/document_upload/conftest.py index 1f1c7df59..6640fefdf 100644 --- a/surfsense_backend/tests/integration/document_upload/conftest.py +++ b/surfsense_backend/tests/integration/document_upload/conftest.py @@ -3,6 +3,7 @@ Prerequisites: PostgreSQL + pgvector only. External system boundaries are mocked: + - ETL parsing — LlamaParse (external API) and Docling (heavy library) - LLM summarization, text embedding, text chunking (external APIs) - Redis heartbeat (external infrastructure) - Task dispatch is swapped via DI (InlineTaskDispatcher) @@ -11,6 +12,7 @@ External system boundaries are mocked: from __future__ import annotations import contextlib +import os from collections.abc import AsyncGenerator from unittest.mock import AsyncMock, MagicMock @@ -298,3 +300,64 @@ def _mock_redis_heartbeat(monkeypatch): "app.tasks.celery_tasks.document_tasks._run_heartbeat_loop", AsyncMock(), ) + + +@pytest.fixture(autouse=True) +def _mock_etl_parsing(monkeypatch): + """Mock ETL parsing services — LlamaParse and Docling are external boundaries. + + Preserves the real contract: empty/corrupt files raise an error just like + the actual services would, so tests covering failure paths keep working. + """ + + _MOCK_MARKDOWN = "# Mocked Document\n\nThis is mocked ETL content." + + def _reject_empty(file_path: str) -> None: + if os.path.getsize(file_path) == 0: + raise RuntimeError(f"Cannot parse empty file: {file_path}") + + # -- LlamaParse mock (external API) -------------------------------- + + class _FakeMarkdownDoc: + def __init__(self, text: str): + self.text = text + + class _FakeLlamaParseResult: + async def aget_markdown_documents(self, *, split_by_page=False): + return [_FakeMarkdownDoc(_MOCK_MARKDOWN)] + + async def _fake_llamacloud_parse(**kwargs): + _reject_empty(kwargs["file_path"]) + return _FakeLlamaParseResult() + + monkeypatch.setattr( + "app.tasks.document_processors.file_processors.parse_with_llamacloud_retry", + _fake_llamacloud_parse, + ) + + # -- Docling mock (heavy library boundary) ------------------------- + + async def _fake_docling_parse(file_path: str, filename: str): + _reject_empty(file_path) + return _MOCK_MARKDOWN + + monkeypatch.setattr( + "app.tasks.document_processors.file_processors.parse_with_docling", + _fake_docling_parse, + ) + + class _FakeDoclingResult: + class document: + @staticmethod + def export_to_markdown(): + return _MOCK_MARKDOWN + + class _FakeDocumentConverter: + def convert(self, file_path): + _reject_empty(file_path) + return _FakeDoclingResult() + + monkeypatch.setattr( + "docling.document_converter.DocumentConverter", + _FakeDocumentConverter, + ) diff --git a/surfsense_backend/tests/unit/connector_indexers/test_google_drive_parallel.py b/surfsense_backend/tests/unit/connector_indexers/test_google_drive_parallel.py index 3fe8a183d..20bd3f3d6 100644 --- a/surfsense_backend/tests/unit/connector_indexers/test_google_drive_parallel.py +++ b/surfsense_backend/tests/unit/connector_indexers/test_google_drive_parallel.py @@ -248,12 +248,33 @@ def _folder_dict(file_id: str, name: str) -> dict: } +def _make_page_limit_session(pages_used=0, pages_limit=999_999): + """Build a mock DB session that real PageLimitService can operate against.""" + + class _FakeUser: + def __init__(self, pu, pl): + self.pages_used = pu + self.pages_limit = pl + + fake_user = _FakeUser(pages_used, pages_limit) + session = AsyncMock() + + def _make_result(*_a, **_kw): + r = MagicMock() + r.first.return_value = (fake_user.pages_used, fake_user.pages_limit) + r.unique.return_value.scalar_one_or_none.return_value = fake_user + return r + + session.execute = AsyncMock(side_effect=_make_result) + return session, fake_user + + @pytest.fixture def full_scan_mocks(mock_drive_client, monkeypatch): """Wire up all mocks needed to call _index_full_scan in isolation.""" import app.tasks.connector_indexers.google_drive_indexer as _mod - mock_session = AsyncMock() + mock_session, _ = _make_page_limit_session() mock_connector = MagicMock() mock_task_logger = MagicMock() mock_task_logger.log_task_progress = AsyncMock() @@ -472,7 +493,7 @@ async def test_delta_sync_removals_serial_rest_parallel(monkeypatch): AsyncMock(return_value=MagicMock()), ) - mock_session = AsyncMock() + mock_session, _ = _make_page_limit_session() mock_task_logger = MagicMock() mock_task_logger.log_task_progress = AsyncMock() @@ -512,7 +533,7 @@ def selected_files_mocks(mock_drive_client, monkeypatch): """Wire up mocks for _index_selected_files tests.""" import app.tasks.connector_indexers.google_drive_indexer as _mod - mock_session = AsyncMock() + mock_session, _ = _make_page_limit_session() get_file_results: dict[str, tuple[dict | None, str | None]] = {} diff --git a/surfsense_backend/tests/unit/connector_indexers/test_page_limits.py b/surfsense_backend/tests/unit/connector_indexers/test_page_limits.py new file mode 100644 index 000000000..1c93965f3 --- /dev/null +++ b/surfsense_backend/tests/unit/connector_indexers/test_page_limits.py @@ -0,0 +1,648 @@ +"""Tests for page limit enforcement in connector indexers. + +Covers: + A) PageLimitService.estimate_pages_from_metadata — pure function (no mocks) + B) Page-limit quota gating in _index_selected_files tested through the + real PageLimitService with a mock DB session (system boundary). + Google Drive is the primary, with OneDrive/Dropbox smoke tests. +""" + +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from app.services.page_limit_service import PageLimitService + +pytestmark = pytest.mark.unit + +_USER_ID = "00000000-0000-0000-0000-000000000001" +_CONNECTOR_ID = 42 +_SEARCH_SPACE_ID = 1 + + +# =================================================================== +# A) PageLimitService.estimate_pages_from_metadata — pure function +# No mocks: it's a staticmethod with no I/O. +# =================================================================== + + +class TestEstimatePagesFromMetadata: + """Vertical slices for the page estimation staticmethod.""" + + def test_pdf_100kb_returns_1(self): + assert PageLimitService.estimate_pages_from_metadata(".pdf", 100 * 1024) == 1 + + def test_pdf_500kb_returns_5(self): + assert PageLimitService.estimate_pages_from_metadata(".pdf", 500 * 1024) == 5 + + def test_pdf_1mb(self): + assert PageLimitService.estimate_pages_from_metadata(".pdf", 1024 * 1024) == 10 + + def test_docx_50kb_returns_1(self): + assert PageLimitService.estimate_pages_from_metadata(".docx", 50 * 1024) == 1 + + def test_docx_200kb(self): + assert PageLimitService.estimate_pages_from_metadata(".docx", 200 * 1024) == 4 + + def test_pptx_uses_200kb_per_page(self): + assert PageLimitService.estimate_pages_from_metadata(".pptx", 600 * 1024) == 3 + + def test_xlsx_uses_100kb_per_page(self): + assert PageLimitService.estimate_pages_from_metadata(".xlsx", 300 * 1024) == 3 + + def test_txt_uses_3000_bytes_per_page(self): + assert PageLimitService.estimate_pages_from_metadata(".txt", 9000) == 3 + + def test_image_always_returns_1(self): + for ext in (".jpg", ".png", ".gif", ".webp"): + assert PageLimitService.estimate_pages_from_metadata(ext, 5_000_000) == 1 + + def test_audio_uses_1mb_per_page(self): + assert PageLimitService.estimate_pages_from_metadata(".mp3", 3 * 1024 * 1024) == 3 + + def test_video_uses_5mb_per_page(self): + assert PageLimitService.estimate_pages_from_metadata(".mp4", 15 * 1024 * 1024) == 3 + + def test_unknown_ext_uses_80kb_per_page(self): + assert PageLimitService.estimate_pages_from_metadata(".xyz", 160 * 1024) == 2 + + def test_zero_size_returns_1(self): + assert PageLimitService.estimate_pages_from_metadata(".pdf", 0) == 1 + + def test_negative_size_returns_1(self): + assert PageLimitService.estimate_pages_from_metadata(".pdf", -500) == 1 + + def test_minimum_is_always_1(self): + assert PageLimitService.estimate_pages_from_metadata(".pdf", 50) == 1 + + def test_epub_uses_50kb_per_page(self): + assert PageLimitService.estimate_pages_from_metadata(".epub", 250 * 1024) == 5 + + +# =================================================================== +# B) Page-limit enforcement in connector indexers +# System boundary mocked: DB session (for PageLimitService) +# System boundary mocked: external API clients, download/ETL +# NOT mocked: PageLimitService itself (our own code) +# =================================================================== + + +class _FakeUser: + """Stands in for the User ORM model at the DB boundary.""" + + def __init__(self, pages_used: int = 0, pages_limit: int = 100): + self.pages_used = pages_used + self.pages_limit = pages_limit + + +def _make_page_limit_session(pages_used: int = 0, pages_limit: int = 100): + """Build a mock DB session that real PageLimitService can operate against. + + Every ``session.execute()`` returns a result compatible with both + ``get_page_usage`` (.first() → tuple) and ``update_page_usage`` + (.unique().scalar_one_or_none() → User-like). + """ + fake_user = _FakeUser(pages_used, pages_limit) + session = AsyncMock() + + def _make_result(*_args, **_kwargs): + result = MagicMock() + result.first.return_value = (fake_user.pages_used, fake_user.pages_limit) + result.unique.return_value.scalar_one_or_none.return_value = fake_user + return result + + session.execute = AsyncMock(side_effect=_make_result) + return session, fake_user + + +def _make_gdrive_file(file_id: str, name: str, size: int = 80 * 1024) -> dict: + return { + "id": file_id, + "name": name, + "mimeType": "application/octet-stream", + "size": str(size), + } + + +# --------------------------------------------------------------------------- +# Google Drive: _index_selected_files +# --------------------------------------------------------------------------- + + +@pytest.fixture +def gdrive_selected_mocks(monkeypatch): + """Mocks for Google Drive _index_selected_files — only system boundaries.""" + import app.tasks.connector_indexers.google_drive_indexer as _mod + + session, fake_user = _make_page_limit_session(0, 100) + + get_file_results: dict[str, tuple[dict | None, str | None]] = {} + + async def _fake_get_file(client, file_id): + return get_file_results.get(file_id, (None, f"Not configured: {file_id}")) + + monkeypatch.setattr(_mod, "get_file_by_id", _fake_get_file) + monkeypatch.setattr( + _mod, "_should_skip_file", AsyncMock(return_value=(False, None)) + ) + + download_and_index_mock = AsyncMock(return_value=(0, 0)) + monkeypatch.setattr(_mod, "_download_and_index", download_and_index_mock) + + pipeline_mock = MagicMock() + pipeline_mock.create_placeholder_documents = AsyncMock(return_value=0) + monkeypatch.setattr( + _mod, "IndexingPipelineService", MagicMock(return_value=pipeline_mock) + ) + + return { + "mod": _mod, + "session": session, + "fake_user": fake_user, + "get_file_results": get_file_results, + "download_and_index_mock": download_and_index_mock, + } + + +async def _run_gdrive_selected(mocks, file_ids): + from app.tasks.connector_indexers.google_drive_indexer import ( + _index_selected_files, + ) + + return await _index_selected_files( + MagicMock(), + mocks["session"], + file_ids, + connector_id=_CONNECTOR_ID, + search_space_id=_SEARCH_SPACE_ID, + user_id=_USER_ID, + enable_summary=True, + ) + + +async def test_gdrive_files_within_quota_are_downloaded(gdrive_selected_mocks): + """Files whose cumulative estimated pages fit within remaining quota + are sent to _download_and_index.""" + m = gdrive_selected_mocks + m["fake_user"].pages_used = 0 + m["fake_user"].pages_limit = 100 + + for fid in ("f1", "f2", "f3"): + m["get_file_results"][fid] = ( + _make_gdrive_file(fid, f"{fid}.xyz", size=80 * 1024), None, + ) + m["download_and_index_mock"].return_value = (3, 0) + + indexed, _skipped, errors = await _run_gdrive_selected( + m, [("f1", "f1.xyz"), ("f2", "f2.xyz"), ("f3", "f3.xyz")] + ) + + assert indexed == 3 + assert errors == [] + call_files = m["download_and_index_mock"].call_args[0][2] + assert len(call_files) == 3 + + +async def test_gdrive_files_exceeding_quota_rejected(gdrive_selected_mocks): + """Files whose pages would exceed remaining quota are rejected.""" + m = gdrive_selected_mocks + m["fake_user"].pages_used = 98 + m["fake_user"].pages_limit = 100 + + m["get_file_results"]["big"] = ( + _make_gdrive_file("big", "huge.pdf", size=500 * 1024), None, + ) + + indexed, _skipped, errors = await _run_gdrive_selected(m, [("big", "huge.pdf")]) + + assert indexed == 0 + assert len(errors) == 1 + assert "page limit" in errors[0].lower() + + +async def test_gdrive_quota_mix_partial_indexing(gdrive_selected_mocks): + """3rd file pushes over quota → only first two indexed.""" + m = gdrive_selected_mocks + m["fake_user"].pages_used = 0 + m["fake_user"].pages_limit = 2 + + for fid in ("f1", "f2", "f3"): + m["get_file_results"][fid] = ( + _make_gdrive_file(fid, f"{fid}.xyz", size=80 * 1024), None, + ) + m["download_and_index_mock"].return_value = (2, 0) + + indexed, _skipped, errors = await _run_gdrive_selected( + m, [("f1", "f1.xyz"), ("f2", "f2.xyz"), ("f3", "f3.xyz")] + ) + + assert indexed == 2 + assert len(errors) == 1 + call_files = m["download_and_index_mock"].call_args[0][2] + assert {f["id"] for f in call_files} == {"f1", "f2"} + + +async def test_gdrive_proportional_page_deduction(gdrive_selected_mocks): + """Pages deducted are proportional to successfully indexed files.""" + m = gdrive_selected_mocks + m["fake_user"].pages_used = 0 + m["fake_user"].pages_limit = 100 + + for fid in ("f1", "f2", "f3", "f4"): + m["get_file_results"][fid] = ( + _make_gdrive_file(fid, f"{fid}.xyz", size=80 * 1024), None, + ) + m["download_and_index_mock"].return_value = (2, 2) + + await _run_gdrive_selected( + m, + [("f1", "f1.xyz"), ("f2", "f2.xyz"), ("f3", "f3.xyz"), ("f4", "f4.xyz")], + ) + + assert m["fake_user"].pages_used == 2 + + +async def test_gdrive_no_deduction_when_nothing_indexed(gdrive_selected_mocks): + """If batch_indexed == 0, user's pages_used stays unchanged.""" + m = gdrive_selected_mocks + m["fake_user"].pages_used = 5 + m["fake_user"].pages_limit = 100 + + m["get_file_results"]["f1"] = ( + _make_gdrive_file("f1", "f1.xyz", size=80 * 1024), None, + ) + m["download_and_index_mock"].return_value = (0, 1) + + await _run_gdrive_selected(m, [("f1", "f1.xyz")]) + + assert m["fake_user"].pages_used == 5 + + +async def test_gdrive_zero_quota_rejects_all(gdrive_selected_mocks): + """When pages_used == pages_limit, every file is rejected.""" + m = gdrive_selected_mocks + m["fake_user"].pages_used = 100 + m["fake_user"].pages_limit = 100 + + for fid in ("f1", "f2"): + m["get_file_results"][fid] = ( + _make_gdrive_file(fid, f"{fid}.xyz", size=80 * 1024), None, + ) + + indexed, _skipped, errors = await _run_gdrive_selected( + m, [("f1", "f1.xyz"), ("f2", "f2.xyz")] + ) + + assert indexed == 0 + assert len(errors) == 2 + + +# --------------------------------------------------------------------------- +# Google Drive: _index_full_scan +# --------------------------------------------------------------------------- + + +@pytest.fixture +def gdrive_full_scan_mocks(monkeypatch): + import app.tasks.connector_indexers.google_drive_indexer as _mod + + session, fake_user = _make_page_limit_session(0, 100) + mock_task_logger = MagicMock() + mock_task_logger.log_task_progress = AsyncMock() + + monkeypatch.setattr( + _mod, "_should_skip_file", AsyncMock(return_value=(False, None)) + ) + + download_mock = AsyncMock(return_value=([], 0)) + monkeypatch.setattr(_mod, "_download_files_parallel", download_mock) + + batch_mock = AsyncMock(return_value=([], 0, 0)) + pipeline_mock = MagicMock() + pipeline_mock.index_batch_parallel = batch_mock + pipeline_mock.create_placeholder_documents = AsyncMock(return_value=0) + monkeypatch.setattr( + _mod, "IndexingPipelineService", MagicMock(return_value=pipeline_mock) + ) + monkeypatch.setattr( + _mod, "get_user_long_context_llm", AsyncMock(return_value=MagicMock()) + ) + + return { + "mod": _mod, + "session": session, + "fake_user": fake_user, + "task_logger": mock_task_logger, + "download_mock": download_mock, + "batch_mock": batch_mock, + } + + +async def _run_gdrive_full_scan(mocks, max_files=500): + from app.tasks.connector_indexers.google_drive_indexer import _index_full_scan + + return await _index_full_scan( + MagicMock(), + mocks["session"], + MagicMock(), + _CONNECTOR_ID, + _SEARCH_SPACE_ID, + _USER_ID, + "folder-root", + "My Folder", + mocks["task_logger"], + MagicMock(), + max_files, + include_subfolders=False, + enable_summary=True, + ) + + +async def test_gdrive_full_scan_skips_over_quota(gdrive_full_scan_mocks, monkeypatch): + m = gdrive_full_scan_mocks + m["fake_user"].pages_used = 0 + m["fake_user"].pages_limit = 2 + + page_files = [ + _make_gdrive_file(f"f{i}", f"file{i}.xyz", size=80 * 1024) for i in range(5) + ] + monkeypatch.setattr( + m["mod"], "get_files_in_folder", + AsyncMock(return_value=(page_files, None, None)), + ) + m["download_mock"].return_value = ([], 0) + m["batch_mock"].return_value = ([], 2, 0) + + _indexed, skipped = await _run_gdrive_full_scan(m) + + call_files = m["download_mock"].call_args[0][1] + assert len(call_files) == 2 + assert skipped == 3 + + +async def test_gdrive_full_scan_deducts_after_indexing( + gdrive_full_scan_mocks, monkeypatch +): + m = gdrive_full_scan_mocks + m["fake_user"].pages_used = 0 + m["fake_user"].pages_limit = 100 + + page_files = [ + _make_gdrive_file(f"f{i}", f"file{i}.xyz", size=80 * 1024) for i in range(3) + ] + monkeypatch.setattr( + m["mod"], "get_files_in_folder", + AsyncMock(return_value=(page_files, None, None)), + ) + mock_docs = [MagicMock() for _ in range(3)] + m["download_mock"].return_value = (mock_docs, 0) + m["batch_mock"].return_value = ([], 3, 0) + + await _run_gdrive_full_scan(m) + + assert m["fake_user"].pages_used == 3 + + +# --------------------------------------------------------------------------- +# Google Drive: _index_with_delta_sync +# --------------------------------------------------------------------------- + + +async def test_gdrive_delta_sync_skips_over_quota(monkeypatch): + import app.tasks.connector_indexers.google_drive_indexer as _mod + + session, _ = _make_page_limit_session(0, 2) + + changes = [ + { + "fileId": f"mod{i}", + "file": _make_gdrive_file(f"mod{i}", f"mod{i}.xyz", size=80 * 1024), + } + for i in range(5) + ] + monkeypatch.setattr( + _mod, "fetch_all_changes", + AsyncMock(return_value=(changes, "new-token", None)), + ) + monkeypatch.setattr(_mod, "categorize_change", lambda change: "modified") + monkeypatch.setattr( + _mod, "_should_skip_file", AsyncMock(return_value=(False, None)) + ) + + download_mock = AsyncMock(return_value=([], 0)) + monkeypatch.setattr(_mod, "_download_files_parallel", download_mock) + + batch_mock = AsyncMock(return_value=([], 2, 0)) + pipeline_mock = MagicMock() + pipeline_mock.index_batch_parallel = batch_mock + pipeline_mock.create_placeholder_documents = AsyncMock(return_value=0) + monkeypatch.setattr( + _mod, "IndexingPipelineService", MagicMock(return_value=pipeline_mock) + ) + monkeypatch.setattr( + _mod, "get_user_long_context_llm", AsyncMock(return_value=MagicMock()) + ) + + mock_task_logger = MagicMock() + mock_task_logger.log_task_progress = AsyncMock() + + _indexed, skipped = await _mod._index_with_delta_sync( + MagicMock(), session, MagicMock(), + _CONNECTOR_ID, _SEARCH_SPACE_ID, _USER_ID, + "folder-root", "start-token", + mock_task_logger, MagicMock(), + max_files=500, enable_summary=True, + ) + + call_files = download_mock.call_args[0][1] + assert len(call_files) == 2 + assert skipped == 3 + + +# =================================================================== +# C) OneDrive smoke tests — verify page limit wiring +# =================================================================== + + +def _make_onedrive_file(file_id: str, name: str, size: int = 80 * 1024) -> dict: + return { + "id": file_id, + "name": name, + "file": {"mimeType": "application/octet-stream"}, + "size": str(size), + "lastModifiedDateTime": "2026-01-01T00:00:00Z", + } + + +@pytest.fixture +def onedrive_selected_mocks(monkeypatch): + import app.tasks.connector_indexers.onedrive_indexer as _mod + + session, fake_user = _make_page_limit_session(0, 100) + + get_file_results: dict[str, tuple[dict | None, str | None]] = {} + + async def _fake_get_file(client, file_id): + return get_file_results.get(file_id, (None, f"Not found: {file_id}")) + + monkeypatch.setattr(_mod, "get_file_by_id", _fake_get_file) + monkeypatch.setattr( + _mod, "_should_skip_file", AsyncMock(return_value=(False, None)) + ) + + download_and_index_mock = AsyncMock(return_value=(0, 0)) + monkeypatch.setattr(_mod, "_download_and_index", download_and_index_mock) + + pipeline_mock = MagicMock() + pipeline_mock.create_placeholder_documents = AsyncMock(return_value=0) + monkeypatch.setattr( + _mod, "IndexingPipelineService", MagicMock(return_value=pipeline_mock) + ) + + return { + "session": session, + "fake_user": fake_user, + "get_file_results": get_file_results, + "download_and_index_mock": download_and_index_mock, + } + + +async def _run_onedrive_selected(mocks, file_ids): + from app.tasks.connector_indexers.onedrive_indexer import _index_selected_files + + return await _index_selected_files( + MagicMock(), mocks["session"], file_ids, + connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, + user_id=_USER_ID, enable_summary=True, + ) + + +async def test_onedrive_over_quota_rejected(onedrive_selected_mocks): + """OneDrive: files exceeding quota produce errors, not downloads.""" + m = onedrive_selected_mocks + m["fake_user"].pages_used = 99 + m["fake_user"].pages_limit = 100 + + m["get_file_results"]["big"] = ( + _make_onedrive_file("big", "huge.pdf", size=500 * 1024), None, + ) + + indexed, _skipped, errors = await _run_onedrive_selected(m, [("big", "huge.pdf")]) + + assert indexed == 0 + assert len(errors) == 1 + assert "page limit" in errors[0].lower() + + +async def test_onedrive_deducts_after_success(onedrive_selected_mocks): + """OneDrive: pages_used increases after successful indexing.""" + m = onedrive_selected_mocks + m["fake_user"].pages_used = 0 + m["fake_user"].pages_limit = 100 + + for fid in ("f1", "f2"): + m["get_file_results"][fid] = ( + _make_onedrive_file(fid, f"{fid}.xyz", size=80 * 1024), None, + ) + m["download_and_index_mock"].return_value = (2, 0) + + await _run_onedrive_selected(m, [("f1", "f1.xyz"), ("f2", "f2.xyz")]) + + assert m["fake_user"].pages_used == 2 + + +# =================================================================== +# D) Dropbox smoke tests — verify page limit wiring +# =================================================================== + + +def _make_dropbox_file(file_path: str, name: str, size: int = 80 * 1024) -> dict: + return { + "id": f"id:{file_path}", + "name": name, + ".tag": "file", + "path_lower": file_path, + "size": str(size), + "server_modified": "2026-01-01T00:00:00Z", + "content_hash": f"hash_{name}", + } + + +@pytest.fixture +def dropbox_selected_mocks(monkeypatch): + import app.tasks.connector_indexers.dropbox_indexer as _mod + + session, fake_user = _make_page_limit_session(0, 100) + + get_file_results: dict[str, tuple[dict | None, str | None]] = {} + + async def _fake_get_file(client, file_path): + return get_file_results.get(file_path, (None, f"Not found: {file_path}")) + + monkeypatch.setattr(_mod, "get_file_by_path", _fake_get_file) + monkeypatch.setattr( + _mod, "_should_skip_file", AsyncMock(return_value=(False, None)) + ) + + download_and_index_mock = AsyncMock(return_value=(0, 0)) + monkeypatch.setattr(_mod, "_download_and_index", download_and_index_mock) + + pipeline_mock = MagicMock() + pipeline_mock.create_placeholder_documents = AsyncMock(return_value=0) + monkeypatch.setattr( + _mod, "IndexingPipelineService", MagicMock(return_value=pipeline_mock) + ) + + return { + "session": session, + "fake_user": fake_user, + "get_file_results": get_file_results, + "download_and_index_mock": download_and_index_mock, + } + + +async def _run_dropbox_selected(mocks, file_paths): + from app.tasks.connector_indexers.dropbox_indexer import _index_selected_files + + return await _index_selected_files( + MagicMock(), mocks["session"], file_paths, + connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, + user_id=_USER_ID, enable_summary=True, + ) + + +async def test_dropbox_over_quota_rejected(dropbox_selected_mocks): + """Dropbox: files exceeding quota produce errors, not downloads.""" + m = dropbox_selected_mocks + m["fake_user"].pages_used = 99 + m["fake_user"].pages_limit = 100 + + m["get_file_results"]["/huge.pdf"] = ( + _make_dropbox_file("/huge.pdf", "huge.pdf", size=500 * 1024), None, + ) + + indexed, _skipped, errors = await _run_dropbox_selected( + m, [("/huge.pdf", "huge.pdf")] + ) + + assert indexed == 0 + assert len(errors) == 1 + assert "page limit" in errors[0].lower() + + +async def test_dropbox_deducts_after_success(dropbox_selected_mocks): + """Dropbox: pages_used increases after successful indexing.""" + m = dropbox_selected_mocks + m["fake_user"].pages_used = 0 + m["fake_user"].pages_limit = 100 + + for name in ("f1.xyz", "f2.xyz"): + path = f"/{name}" + m["get_file_results"][path] = ( + _make_dropbox_file(path, name, size=80 * 1024), None, + ) + m["download_and_index_mock"].return_value = (2, 0) + + await _run_dropbox_selected(m, [("/f1.xyz", "f1.xyz"), ("/f2.xyz", "f2.xyz")]) + + assert m["fake_user"].pages_used == 2 From a2b354104631a17c5717cac53837e022843a58bf Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 4 Apr 2026 03:11:56 +0530 Subject: [PATCH 25/33] chore: ran linting --- .../app/services/page_limit_service.py | 96 +++++++++++++++---- .../app/tasks/connector_indexers/base.py | 1 + .../local_folder_indexer.py | 1 + .../integration/document_upload/conftest.py | 15 +-- .../test_local_folder_pipeline.py | 2 +- .../connector_indexers/test_page_limits.py | 84 +++++++++++----- 6 files changed, 150 insertions(+), 49 deletions(-) diff --git a/surfsense_backend/app/services/page_limit_service.py b/surfsense_backend/app/services/page_limit_service.py index ea22067be..47fe07fc6 100644 --- a/surfsense_backend/app/services/page_limit_service.py +++ b/surfsense_backend/app/services/page_limit_service.py @@ -257,28 +257,83 @@ class PageLimitService: return max(1, file_size // (100 * 1024)) if file_ext in { - ".doc", ".docx", ".docm", ".dot", ".dotm", - ".odt", ".ott", ".sxw", ".stw", ".uot", - ".rtf", ".pages", ".wpd", ".wps", - ".abw", ".zabw", ".cwk", ".hwp", ".lwp", - ".mcw", ".mw", ".sdw", ".vor", + ".doc", + ".docx", + ".docm", + ".dot", + ".dotm", + ".odt", + ".ott", + ".sxw", + ".stw", + ".uot", + ".rtf", + ".pages", + ".wpd", + ".wps", + ".abw", + ".zabw", + ".cwk", + ".hwp", + ".lwp", + ".mcw", + ".mw", + ".sdw", + ".vor", }: return max(1, file_size // (50 * 1024)) if file_ext in { - ".ppt", ".pptx", ".pptm", ".pot", ".potx", - ".odp", ".otp", ".sxi", ".sti", ".uop", - ".key", ".sda", ".sdd", ".sdp", + ".ppt", + ".pptx", + ".pptm", + ".pot", + ".potx", + ".odp", + ".otp", + ".sxi", + ".sti", + ".uop", + ".key", + ".sda", + ".sdd", + ".sdp", }: return max(1, file_size // (200 * 1024)) if file_ext in { - ".xls", ".xlsx", ".xlsm", ".xlsb", ".xlw", ".xlr", - ".ods", ".ots", ".fods", ".numbers", - ".123", ".wk1", ".wk2", ".wk3", ".wk4", ".wks", - ".wb1", ".wb2", ".wb3", ".wq1", ".wq2", - ".csv", ".tsv", ".slk", ".sylk", ".dif", ".dbf", - ".prn", ".qpw", ".602", ".et", ".eth", + ".xls", + ".xlsx", + ".xlsm", + ".xlsb", + ".xlw", + ".xlr", + ".ods", + ".ots", + ".fods", + ".numbers", + ".123", + ".wk1", + ".wk2", + ".wk3", + ".wk4", + ".wks", + ".wb1", + ".wb2", + ".wb3", + ".wq1", + ".wq2", + ".csv", + ".tsv", + ".slk", + ".sylk", + ".dif", + ".dbf", + ".prn", + ".qpw", + ".602", + ".et", + ".eth", }: return max(1, file_size // (100 * 1024)) @@ -289,8 +344,17 @@ class PageLimitService: return max(1, file_size // 3000) if file_ext in { - ".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", - ".webp", ".svg", ".cgm", ".odg", ".pbd", + ".jpg", + ".jpeg", + ".png", + ".gif", + ".bmp", + ".tiff", + ".webp", + ".svg", + ".cgm", + ".odg", + ".pbd", }: return 1 diff --git a/surfsense_backend/app/tasks/connector_indexers/base.py b/surfsense_backend/app/tasks/connector_indexers/base.py index 6b4bed4b5..ffc8ab72e 100644 --- a/surfsense_backend/app/tasks/connector_indexers/base.py +++ b/surfsense_backend/app/tasks/connector_indexers/base.py @@ -4,6 +4,7 @@ Base functionality and shared imports for connector indexers. import logging from datetime import UTC, datetime, timedelta + from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select diff --git a/surfsense_backend/app/tasks/connector_indexers/local_folder_indexer.py b/surfsense_backend/app/tasks/connector_indexers/local_folder_indexer.py index acfbce0bf..fa50e86d3 100644 --- a/surfsense_backend/app/tasks/connector_indexers/local_folder_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/local_folder_indexer.py @@ -205,6 +205,7 @@ def _compute_final_pages( actual = page_limit_service.estimate_pages_from_content_length(content_length) return max(estimated_pages, actual) + DEFAULT_EXCLUDE_PATTERNS = [ ".git", "node_modules", diff --git a/surfsense_backend/tests/integration/document_upload/conftest.py b/surfsense_backend/tests/integration/document_upload/conftest.py index 6640fefdf..41c379e58 100644 --- a/surfsense_backend/tests/integration/document_upload/conftest.py +++ b/surfsense_backend/tests/integration/document_upload/conftest.py @@ -302,6 +302,9 @@ def _mock_redis_heartbeat(monkeypatch): ) +_MOCK_ETL_MARKDOWN = "# Mocked Document\n\nThis is mocked ETL content." + + @pytest.fixture(autouse=True) def _mock_etl_parsing(monkeypatch): """Mock ETL parsing services — LlamaParse and Docling are external boundaries. @@ -310,8 +313,6 @@ def _mock_etl_parsing(monkeypatch): the actual services would, so tests covering failure paths keep working. """ - _MOCK_MARKDOWN = "# Mocked Document\n\nThis is mocked ETL content." - def _reject_empty(file_path: str) -> None: if os.path.getsize(file_path) == 0: raise RuntimeError(f"Cannot parse empty file: {file_path}") @@ -324,7 +325,7 @@ def _mock_etl_parsing(monkeypatch): class _FakeLlamaParseResult: async def aget_markdown_documents(self, *, split_by_page=False): - return [_FakeMarkdownDoc(_MOCK_MARKDOWN)] + return [_FakeMarkdownDoc(_MOCK_ETL_MARKDOWN)] async def _fake_llamacloud_parse(**kwargs): _reject_empty(kwargs["file_path"]) @@ -339,7 +340,7 @@ def _mock_etl_parsing(monkeypatch): async def _fake_docling_parse(file_path: str, filename: str): _reject_empty(file_path) - return _MOCK_MARKDOWN + return _MOCK_ETL_MARKDOWN monkeypatch.setattr( "app.tasks.document_processors.file_processors.parse_with_docling", @@ -347,10 +348,12 @@ def _mock_etl_parsing(monkeypatch): ) class _FakeDoclingResult: - class document: + class Document: @staticmethod def export_to_markdown(): - return _MOCK_MARKDOWN + return _MOCK_ETL_MARKDOWN + + document = Document() class _FakeDocumentConverter: def convert(self, file_path): diff --git a/surfsense_backend/tests/integration/indexing_pipeline/test_local_folder_pipeline.py b/surfsense_backend/tests/integration/indexing_pipeline/test_local_folder_pipeline.py index 4d9bda7ee..000f43aa8 100644 --- a/surfsense_backend/tests/integration/indexing_pipeline/test_local_folder_pipeline.py +++ b/surfsense_backend/tests/integration/indexing_pipeline/test_local_folder_pipeline.py @@ -1015,7 +1015,7 @@ class TestPageLimits: (tmp_path / "note.md").write_text("# Hello World\n\nContent here.") - count, _skipped, _root_folder_id, err = await index_local_folder( + count, _skipped, _root_folder_id, _err = await index_local_folder( session=db_session, search_space_id=db_search_space.id, user_id=str(db_user.id), diff --git a/surfsense_backend/tests/unit/connector_indexers/test_page_limits.py b/surfsense_backend/tests/unit/connector_indexers/test_page_limits.py index 1c93965f3..b31a9557f 100644 --- a/surfsense_backend/tests/unit/connector_indexers/test_page_limits.py +++ b/surfsense_backend/tests/unit/connector_indexers/test_page_limits.py @@ -58,10 +58,14 @@ class TestEstimatePagesFromMetadata: assert PageLimitService.estimate_pages_from_metadata(ext, 5_000_000) == 1 def test_audio_uses_1mb_per_page(self): - assert PageLimitService.estimate_pages_from_metadata(".mp3", 3 * 1024 * 1024) == 3 + assert ( + PageLimitService.estimate_pages_from_metadata(".mp3", 3 * 1024 * 1024) == 3 + ) def test_video_uses_5mb_per_page(self): - assert PageLimitService.estimate_pages_from_metadata(".mp4", 15 * 1024 * 1024) == 3 + assert ( + PageLimitService.estimate_pages_from_metadata(".mp4", 15 * 1024 * 1024) == 3 + ) def test_unknown_ext_uses_80kb_per_page(self): assert PageLimitService.estimate_pages_from_metadata(".xyz", 160 * 1024) == 2 @@ -189,7 +193,8 @@ async def test_gdrive_files_within_quota_are_downloaded(gdrive_selected_mocks): for fid in ("f1", "f2", "f3"): m["get_file_results"][fid] = ( - _make_gdrive_file(fid, f"{fid}.xyz", size=80 * 1024), None, + _make_gdrive_file(fid, f"{fid}.xyz", size=80 * 1024), + None, ) m["download_and_index_mock"].return_value = (3, 0) @@ -210,7 +215,8 @@ async def test_gdrive_files_exceeding_quota_rejected(gdrive_selected_mocks): m["fake_user"].pages_limit = 100 m["get_file_results"]["big"] = ( - _make_gdrive_file("big", "huge.pdf", size=500 * 1024), None, + _make_gdrive_file("big", "huge.pdf", size=500 * 1024), + None, ) indexed, _skipped, errors = await _run_gdrive_selected(m, [("big", "huge.pdf")]) @@ -228,7 +234,8 @@ async def test_gdrive_quota_mix_partial_indexing(gdrive_selected_mocks): for fid in ("f1", "f2", "f3"): m["get_file_results"][fid] = ( - _make_gdrive_file(fid, f"{fid}.xyz", size=80 * 1024), None, + _make_gdrive_file(fid, f"{fid}.xyz", size=80 * 1024), + None, ) m["download_and_index_mock"].return_value = (2, 0) @@ -250,7 +257,8 @@ async def test_gdrive_proportional_page_deduction(gdrive_selected_mocks): for fid in ("f1", "f2", "f3", "f4"): m["get_file_results"][fid] = ( - _make_gdrive_file(fid, f"{fid}.xyz", size=80 * 1024), None, + _make_gdrive_file(fid, f"{fid}.xyz", size=80 * 1024), + None, ) m["download_and_index_mock"].return_value = (2, 2) @@ -269,7 +277,8 @@ async def test_gdrive_no_deduction_when_nothing_indexed(gdrive_selected_mocks): m["fake_user"].pages_limit = 100 m["get_file_results"]["f1"] = ( - _make_gdrive_file("f1", "f1.xyz", size=80 * 1024), None, + _make_gdrive_file("f1", "f1.xyz", size=80 * 1024), + None, ) m["download_and_index_mock"].return_value = (0, 1) @@ -286,7 +295,8 @@ async def test_gdrive_zero_quota_rejects_all(gdrive_selected_mocks): for fid in ("f1", "f2"): m["get_file_results"][fid] = ( - _make_gdrive_file(fid, f"{fid}.xyz", size=80 * 1024), None, + _make_gdrive_file(fid, f"{fid}.xyz", size=80 * 1024), + None, ) indexed, _skipped, errors = await _run_gdrive_selected( @@ -367,7 +377,8 @@ async def test_gdrive_full_scan_skips_over_quota(gdrive_full_scan_mocks, monkeyp _make_gdrive_file(f"f{i}", f"file{i}.xyz", size=80 * 1024) for i in range(5) ] monkeypatch.setattr( - m["mod"], "get_files_in_folder", + m["mod"], + "get_files_in_folder", AsyncMock(return_value=(page_files, None, None)), ) m["download_mock"].return_value = ([], 0) @@ -391,7 +402,8 @@ async def test_gdrive_full_scan_deducts_after_indexing( _make_gdrive_file(f"f{i}", f"file{i}.xyz", size=80 * 1024) for i in range(3) ] monkeypatch.setattr( - m["mod"], "get_files_in_folder", + m["mod"], + "get_files_in_folder", AsyncMock(return_value=(page_files, None, None)), ) mock_docs = [MagicMock() for _ in range(3)] @@ -421,7 +433,8 @@ async def test_gdrive_delta_sync_skips_over_quota(monkeypatch): for i in range(5) ] monkeypatch.setattr( - _mod, "fetch_all_changes", + _mod, + "fetch_all_changes", AsyncMock(return_value=(changes, "new-token", None)), ) monkeypatch.setattr(_mod, "categorize_change", lambda change: "modified") @@ -447,11 +460,18 @@ async def test_gdrive_delta_sync_skips_over_quota(monkeypatch): mock_task_logger.log_task_progress = AsyncMock() _indexed, skipped = await _mod._index_with_delta_sync( - MagicMock(), session, MagicMock(), - _CONNECTOR_ID, _SEARCH_SPACE_ID, _USER_ID, - "folder-root", "start-token", - mock_task_logger, MagicMock(), - max_files=500, enable_summary=True, + MagicMock(), + session, + MagicMock(), + _CONNECTOR_ID, + _SEARCH_SPACE_ID, + _USER_ID, + "folder-root", + "start-token", + mock_task_logger, + MagicMock(), + max_files=500, + enable_summary=True, ) call_files = download_mock.call_args[0][1] @@ -511,9 +531,13 @@ async def _run_onedrive_selected(mocks, file_ids): from app.tasks.connector_indexers.onedrive_indexer import _index_selected_files return await _index_selected_files( - MagicMock(), mocks["session"], file_ids, - connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, - user_id=_USER_ID, enable_summary=True, + MagicMock(), + mocks["session"], + file_ids, + connector_id=_CONNECTOR_ID, + search_space_id=_SEARCH_SPACE_ID, + user_id=_USER_ID, + enable_summary=True, ) @@ -524,7 +548,8 @@ async def test_onedrive_over_quota_rejected(onedrive_selected_mocks): m["fake_user"].pages_limit = 100 m["get_file_results"]["big"] = ( - _make_onedrive_file("big", "huge.pdf", size=500 * 1024), None, + _make_onedrive_file("big", "huge.pdf", size=500 * 1024), + None, ) indexed, _skipped, errors = await _run_onedrive_selected(m, [("big", "huge.pdf")]) @@ -542,7 +567,8 @@ async def test_onedrive_deducts_after_success(onedrive_selected_mocks): for fid in ("f1", "f2"): m["get_file_results"][fid] = ( - _make_onedrive_file(fid, f"{fid}.xyz", size=80 * 1024), None, + _make_onedrive_file(fid, f"{fid}.xyz", size=80 * 1024), + None, ) m["download_and_index_mock"].return_value = (2, 0) @@ -605,9 +631,13 @@ async def _run_dropbox_selected(mocks, file_paths): from app.tasks.connector_indexers.dropbox_indexer import _index_selected_files return await _index_selected_files( - MagicMock(), mocks["session"], file_paths, - connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, - user_id=_USER_ID, enable_summary=True, + MagicMock(), + mocks["session"], + file_paths, + connector_id=_CONNECTOR_ID, + search_space_id=_SEARCH_SPACE_ID, + user_id=_USER_ID, + enable_summary=True, ) @@ -618,7 +648,8 @@ async def test_dropbox_over_quota_rejected(dropbox_selected_mocks): m["fake_user"].pages_limit = 100 m["get_file_results"]["/huge.pdf"] = ( - _make_dropbox_file("/huge.pdf", "huge.pdf", size=500 * 1024), None, + _make_dropbox_file("/huge.pdf", "huge.pdf", size=500 * 1024), + None, ) indexed, _skipped, errors = await _run_dropbox_selected( @@ -639,7 +670,8 @@ async def test_dropbox_deducts_after_success(dropbox_selected_mocks): for name in ("f1.xyz", "f2.xyz"): path = f"/{name}" m["get_file_results"][path] = ( - _make_dropbox_file(path, name, size=80 * 1024), None, + _make_dropbox_file(path, name, size=80 * 1024), + None, ) m["download_and_index_mock"].return_value = (2, 0) From 09008c8f1a6ed963bcac17957ce14c8b3bc569e3 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 4 Apr 2026 03:26:22 +0530 Subject: [PATCH 26/33] refactor: remove redundant authenticatedFetch calls in editor panel components --- surfsense_web/components/editor-panel/editor-panel.tsx | 3 --- surfsense_web/components/layout/ui/tabs/DocumentTabContent.tsx | 3 --- 2 files changed, 6 deletions(-) diff --git a/surfsense_web/components/editor-panel/editor-panel.tsx b/surfsense_web/components/editor-panel/editor-panel.tsx index 248fe68eb..c307b3cea 100644 --- a/surfsense_web/components/editor-panel/editor-panel.tsx +++ b/surfsense_web/components/editor-panel/editor-panel.tsx @@ -96,9 +96,6 @@ export function EditorPanelContent({ } try { - const response = await authenticatedFetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-spaces/${searchSpaceId}/documents/${documentId}/editor-content`, - { method: "GET", signal: controller.signal } const url = new URL( `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-spaces/${searchSpaceId}/documents/${documentId}/editor-content` ); diff --git a/surfsense_web/components/layout/ui/tabs/DocumentTabContent.tsx b/surfsense_web/components/layout/ui/tabs/DocumentTabContent.tsx index d2ce3cc64..97c5b7cd9 100644 --- a/surfsense_web/components/layout/ui/tabs/DocumentTabContent.tsx +++ b/surfsense_web/components/layout/ui/tabs/DocumentTabContent.tsx @@ -81,9 +81,6 @@ export function DocumentTabContent({ documentId, searchSpaceId, title }: Documen } try { - const response = await authenticatedFetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-spaces/${searchSpaceId}/documents/${documentId}/editor-content`, - { method: "GET", signal: controller.signal } const url = new URL( `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-spaces/${searchSpaceId}/documents/${documentId}/editor-content` ); From 8e6b1c77eafbbb54c69ccefaf26aa017cb8e2e50 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 4 Apr 2026 03:35:34 +0530 Subject: [PATCH 27/33] feat: implement PKCE support in native Google OAuth flows - Added `generate_code_verifier` function to create a PKCE code verifier for enhanced security. - Updated Google Calendar, Drive, and Gmail connector routes to utilize the PKCE code verifier during OAuth authorization. - Modified state management to include the code verifier for secure state generation and validation. --- .../google_calendar_add_connector_route.py | 18 ++++++++++---- .../google_drive_add_connector_route.py | 24 +++++++++++++------ .../google_gmail_add_connector_route.py | 18 ++++++++++---- surfsense_backend/app/utils/oauth_security.py | 10 ++++++++ 4 files changed, 55 insertions(+), 15 deletions(-) diff --git a/surfsense_backend/app/routes/google_calendar_add_connector_route.py b/surfsense_backend/app/routes/google_calendar_add_connector_route.py index 9a2308bec..725f8decc 100644 --- a/surfsense_backend/app/routes/google_calendar_add_connector_route.py +++ b/surfsense_backend/app/routes/google_calendar_add_connector_route.py @@ -28,7 +28,7 @@ from app.utils.connector_naming import ( check_duplicate_connector, generate_unique_connector_name, ) -from app.utils.oauth_security import OAuthStateManager, TokenEncryption +from app.utils.oauth_security import OAuthStateManager, TokenEncryption, generate_code_verifier logger = logging.getLogger(__name__) @@ -96,9 +96,14 @@ async def connect_calendar(space_id: int, user: User = Depends(current_active_us flow = get_google_flow() - # Generate secure state parameter with HMAC signature + code_verifier = generate_code_verifier() + flow.code_verifier = code_verifier + + # Generate secure state parameter with HMAC signature (includes PKCE code_verifier) state_manager = get_state_manager() - state_encoded = state_manager.generate_secure_state(space_id, user.id) + state_encoded = state_manager.generate_secure_state( + space_id, user.id, code_verifier=code_verifier + ) auth_url, _ = flow.authorization_url( access_type="offline", @@ -146,8 +151,11 @@ async def reauth_calendar( flow = get_google_flow() + code_verifier = generate_code_verifier() + flow.code_verifier = code_verifier + state_manager = get_state_manager() - extra: dict = {"connector_id": connector_id} + extra: dict = {"connector_id": connector_id, "code_verifier": code_verifier} if return_url and return_url.startswith("/"): extra["return_url"] = return_url state_encoded = state_manager.generate_secure_state(space_id, user.id, **extra) @@ -225,6 +233,7 @@ async def calendar_callback( user_id = UUID(data["user_id"]) space_id = data["space_id"] + code_verifier = data.get("code_verifier") # Validate redirect URI (security: ensure it matches configured value) if not config.GOOGLE_CALENDAR_REDIRECT_URI: @@ -233,6 +242,7 @@ async def calendar_callback( ) flow = get_google_flow() + flow.code_verifier = code_verifier flow.fetch_token(code=code) creds = flow.credentials diff --git a/surfsense_backend/app/routes/google_drive_add_connector_route.py b/surfsense_backend/app/routes/google_drive_add_connector_route.py index 1c9391610..921f84af9 100644 --- a/surfsense_backend/app/routes/google_drive_add_connector_route.py +++ b/surfsense_backend/app/routes/google_drive_add_connector_route.py @@ -41,7 +41,7 @@ from app.utils.connector_naming import ( check_duplicate_connector, generate_unique_connector_name, ) -from app.utils.oauth_security import OAuthStateManager, TokenEncryption +from app.utils.oauth_security import OAuthStateManager, TokenEncryption, generate_code_verifier # Relax token scope validation for Google OAuth os.environ["OAUTHLIB_RELAX_TOKEN_SCOPE"] = "1" @@ -127,14 +127,19 @@ async def connect_drive(space_id: int, user: User = Depends(current_active_user) flow = get_google_flow() - # Generate secure state parameter with HMAC signature + code_verifier = generate_code_verifier() + flow.code_verifier = code_verifier + + # Generate secure state parameter with HMAC signature (includes PKCE code_verifier) state_manager = get_state_manager() - state_encoded = state_manager.generate_secure_state(space_id, user.id) + state_encoded = state_manager.generate_secure_state( + space_id, user.id, code_verifier=code_verifier + ) # Generate authorization URL auth_url, _ = flow.authorization_url( - access_type="offline", # Get refresh token - prompt="consent", # Force consent screen to get refresh token + access_type="offline", + prompt="consent", include_granted_scopes="true", state=state_encoded, ) @@ -193,8 +198,11 @@ async def reauth_drive( flow = get_google_flow() + code_verifier = generate_code_verifier() + flow.code_verifier = code_verifier + state_manager = get_state_manager() - extra: dict = {"connector_id": connector_id} + extra: dict = {"connector_id": connector_id, "code_verifier": code_verifier} if return_url and return_url.startswith("/"): extra["return_url"] = return_url state_encoded = state_manager.generate_secure_state(space_id, user.id, **extra) @@ -285,6 +293,7 @@ async def drive_callback( space_id = data["space_id"] reauth_connector_id = data.get("connector_id") reauth_return_url = data.get("return_url") + code_verifier = data.get("code_verifier") logger.info( f"Processing Google Drive callback for user {user_id}, space {space_id}" @@ -296,8 +305,9 @@ async def drive_callback( status_code=500, detail="GOOGLE_DRIVE_REDIRECT_URI not configured" ) - # Exchange authorization code for tokens + # Exchange authorization code for tokens (restore PKCE code_verifier from state) flow = get_google_flow() + flow.code_verifier = code_verifier flow.fetch_token(code=code) creds = flow.credentials diff --git a/surfsense_backend/app/routes/google_gmail_add_connector_route.py b/surfsense_backend/app/routes/google_gmail_add_connector_route.py index 750a64819..9fe0c0de6 100644 --- a/surfsense_backend/app/routes/google_gmail_add_connector_route.py +++ b/surfsense_backend/app/routes/google_gmail_add_connector_route.py @@ -28,7 +28,7 @@ from app.utils.connector_naming import ( check_duplicate_connector, generate_unique_connector_name, ) -from app.utils.oauth_security import OAuthStateManager, TokenEncryption +from app.utils.oauth_security import OAuthStateManager, TokenEncryption, generate_code_verifier logger = logging.getLogger(__name__) @@ -109,9 +109,14 @@ async def connect_gmail(space_id: int, user: User = Depends(current_active_user) flow = get_google_flow() - # Generate secure state parameter with HMAC signature + code_verifier = generate_code_verifier() + flow.code_verifier = code_verifier + + # Generate secure state parameter with HMAC signature (includes PKCE code_verifier) state_manager = get_state_manager() - state_encoded = state_manager.generate_secure_state(space_id, user.id) + state_encoded = state_manager.generate_secure_state( + space_id, user.id, code_verifier=code_verifier + ) auth_url, _ = flow.authorization_url( access_type="offline", @@ -164,8 +169,11 @@ async def reauth_gmail( flow = get_google_flow() + code_verifier = generate_code_verifier() + flow.code_verifier = code_verifier + state_manager = get_state_manager() - extra: dict = {"connector_id": connector_id} + extra: dict = {"connector_id": connector_id, "code_verifier": code_verifier} if return_url and return_url.startswith("/"): extra["return_url"] = return_url state_encoded = state_manager.generate_secure_state(space_id, user.id, **extra) @@ -256,6 +264,7 @@ async def gmail_callback( user_id = UUID(data["user_id"]) space_id = data["space_id"] + code_verifier = data.get("code_verifier") # Validate redirect URI (security: ensure it matches configured value) if not config.GOOGLE_GMAIL_REDIRECT_URI: @@ -264,6 +273,7 @@ async def gmail_callback( ) flow = get_google_flow() + flow.code_verifier = code_verifier flow.fetch_token(code=code) creds = flow.credentials diff --git a/surfsense_backend/app/utils/oauth_security.py b/surfsense_backend/app/utils/oauth_security.py index 5135cdef4..0ad9d3bd9 100644 --- a/surfsense_backend/app/utils/oauth_security.py +++ b/surfsense_backend/app/utils/oauth_security.py @@ -11,6 +11,8 @@ import hmac import json import logging import time +from random import SystemRandom +from string import ascii_letters, digits from uuid import UUID from cryptography.fernet import Fernet @@ -18,6 +20,14 @@ from fastapi import HTTPException logger = logging.getLogger(__name__) +_PKCE_CHARS = ascii_letters + digits + "-._~" +_PKCE_RNG = SystemRandom() + + +def generate_code_verifier(length: int = 128) -> str: + """Generate a PKCE code_verifier (RFC 7636, 43-128 unreserved chars).""" + return "".join(_PKCE_RNG.choice(_PKCE_CHARS) for _ in range(length)) + class OAuthStateManager: """Manages secure OAuth state parameters with HMAC signatures.""" From e814540727bbc457bef59e5a6b63f3a2aec2f957 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 4 Apr 2026 03:36:54 +0530 Subject: [PATCH 28/33] refactor: move PKCE pair generatio for airtable - Removed the `generate_pkce_pair` function from `airtable_add_connector_route.py` and relocated it to `oauth_security.py` for better organization. - Updated imports in `airtable_add_connector_route.py` to reflect the new location of the PKCE generation function. --- .../routes/airtable_add_connector_route.py | 26 +------------------ surfsense_backend/app/utils/oauth_security.py | 11 ++++++++ 2 files changed, 12 insertions(+), 25 deletions(-) diff --git a/surfsense_backend/app/routes/airtable_add_connector_route.py b/surfsense_backend/app/routes/airtable_add_connector_route.py index fe359d2f3..d2d25d006 100644 --- a/surfsense_backend/app/routes/airtable_add_connector_route.py +++ b/surfsense_backend/app/routes/airtable_add_connector_route.py @@ -1,7 +1,5 @@ import base64 -import hashlib import logging -import secrets from datetime import UTC, datetime, timedelta from uuid import UUID @@ -26,7 +24,7 @@ from app.utils.connector_naming import ( check_duplicate_connector, generate_unique_connector_name, ) -from app.utils.oauth_security import OAuthStateManager, TokenEncryption +from app.utils.oauth_security import OAuthStateManager, TokenEncryption, generate_pkce_pair logger = logging.getLogger(__name__) @@ -75,28 +73,6 @@ def make_basic_auth_header(client_id: str, client_secret: str) -> str: return f"Basic {b64}" -def generate_pkce_pair() -> tuple[str, str]: - """ - Generate PKCE code verifier and code challenge. - - Returns: - Tuple of (code_verifier, code_challenge) - """ - # Generate code verifier (43-128 characters) - code_verifier = ( - base64.urlsafe_b64encode(secrets.token_bytes(32)).decode("utf-8").rstrip("=") - ) - - # Generate code challenge (SHA256 hash of verifier, base64url encoded) - code_challenge = ( - base64.urlsafe_b64encode(hashlib.sha256(code_verifier.encode("utf-8")).digest()) - .decode("utf-8") - .rstrip("=") - ) - - return code_verifier, code_challenge - - @router.get("/auth/airtable/connector/add") async def connect_airtable(space_id: int, user: User = Depends(current_active_user)): """ diff --git a/surfsense_backend/app/utils/oauth_security.py b/surfsense_backend/app/utils/oauth_security.py index 0ad9d3bd9..c39b1e9b1 100644 --- a/surfsense_backend/app/utils/oauth_security.py +++ b/surfsense_backend/app/utils/oauth_security.py @@ -29,6 +29,17 @@ def generate_code_verifier(length: int = 128) -> str: return "".join(_PKCE_RNG.choice(_PKCE_CHARS) for _ in range(length)) +def generate_pkce_pair(length: int = 128) -> tuple[str, str]: + """Generate a PKCE code_verifier and its S256 code_challenge.""" + verifier = generate_code_verifier(length) + challenge = ( + base64.urlsafe_b64encode(hashlib.sha256(verifier.encode()).digest()) + .decode() + .rstrip("=") + ) + return verifier, challenge + + class OAuthStateManager: """Manages secure OAuth state parameters with HMAC signatures.""" From 82d4d3e27234fb876701a35f22b15c99e5608b15 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sat, 4 Apr 2026 03:37:33 +0530 Subject: [PATCH 29/33] chore: ran linting --- .../app/routes/airtable_add_connector_route.py | 6 +++++- .../app/routes/google_calendar_add_connector_route.py | 6 +++++- .../app/routes/google_drive_add_connector_route.py | 6 +++++- .../app/routes/google_gmail_add_connector_route.py | 6 +++++- 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/surfsense_backend/app/routes/airtable_add_connector_route.py b/surfsense_backend/app/routes/airtable_add_connector_route.py index d2d25d006..1e0b1eb5d 100644 --- a/surfsense_backend/app/routes/airtable_add_connector_route.py +++ b/surfsense_backend/app/routes/airtable_add_connector_route.py @@ -24,7 +24,11 @@ from app.utils.connector_naming import ( check_duplicate_connector, generate_unique_connector_name, ) -from app.utils.oauth_security import OAuthStateManager, TokenEncryption, generate_pkce_pair +from app.utils.oauth_security import ( + OAuthStateManager, + TokenEncryption, + generate_pkce_pair, +) logger = logging.getLogger(__name__) diff --git a/surfsense_backend/app/routes/google_calendar_add_connector_route.py b/surfsense_backend/app/routes/google_calendar_add_connector_route.py index 725f8decc..d7ccf62ca 100644 --- a/surfsense_backend/app/routes/google_calendar_add_connector_route.py +++ b/surfsense_backend/app/routes/google_calendar_add_connector_route.py @@ -28,7 +28,11 @@ from app.utils.connector_naming import ( check_duplicate_connector, generate_unique_connector_name, ) -from app.utils.oauth_security import OAuthStateManager, TokenEncryption, generate_code_verifier +from app.utils.oauth_security import ( + OAuthStateManager, + TokenEncryption, + generate_code_verifier, +) logger = logging.getLogger(__name__) diff --git a/surfsense_backend/app/routes/google_drive_add_connector_route.py b/surfsense_backend/app/routes/google_drive_add_connector_route.py index 921f84af9..8706326b7 100644 --- a/surfsense_backend/app/routes/google_drive_add_connector_route.py +++ b/surfsense_backend/app/routes/google_drive_add_connector_route.py @@ -41,7 +41,11 @@ from app.utils.connector_naming import ( check_duplicate_connector, generate_unique_connector_name, ) -from app.utils.oauth_security import OAuthStateManager, TokenEncryption, generate_code_verifier +from app.utils.oauth_security import ( + OAuthStateManager, + TokenEncryption, + generate_code_verifier, +) # Relax token scope validation for Google OAuth os.environ["OAUTHLIB_RELAX_TOKEN_SCOPE"] = "1" diff --git a/surfsense_backend/app/routes/google_gmail_add_connector_route.py b/surfsense_backend/app/routes/google_gmail_add_connector_route.py index 9fe0c0de6..dd8feb1c7 100644 --- a/surfsense_backend/app/routes/google_gmail_add_connector_route.py +++ b/surfsense_backend/app/routes/google_gmail_add_connector_route.py @@ -28,7 +28,11 @@ from app.utils.connector_naming import ( check_duplicate_connector, generate_unique_connector_name, ) -from app.utils.oauth_security import OAuthStateManager, TokenEncryption, generate_code_verifier +from app.utils.oauth_security import ( + OAuthStateManager, + TokenEncryption, + generate_code_verifier, +) logger = logging.getLogger(__name__) From 46e8134b23ec5bb56dce475811437a283bcaec47 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sat, 4 Apr 2026 08:40:55 +0200 Subject: [PATCH 30/33] catch non-vision model errors with actionable user message --- .../app/services/vision_autocomplete_service.py | 12 ++++++++++-- surfsense_web/app/desktop/suggestion/page.tsx | 2 ++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/surfsense_backend/app/services/vision_autocomplete_service.py b/surfsense_backend/app/services/vision_autocomplete_service.py index 0804df7fb..e172c6522 100644 --- a/surfsense_backend/app/services/vision_autocomplete_service.py +++ b/surfsense_backend/app/services/vision_autocomplete_service.py @@ -186,6 +186,14 @@ async def stream_vision_autocomplete( yield streaming.format_done() except Exception as e: - logger.error(f"Vision autocomplete streaming error: {e}") - yield streaming.format_error(str(e)) + error_str = str(e).lower() + if "content must be a string" in error_str or "does not support image" in error_str: + logger.warning(f"Vision autocomplete: selected model does not support vision: {e}") + yield streaming.format_error( + "The selected model does not support vision. " + "Please set a vision-capable model (e.g. GPT-4o, Gemini) in your search space settings." + ) + else: + logger.error(f"Vision autocomplete streaming error: {e}") + yield streaming.format_error(str(e)) yield streaming.format_done() diff --git a/surfsense_web/app/desktop/suggestion/page.tsx b/surfsense_web/app/desktop/suggestion/page.tsx index b7d9b97bd..7188b73c6 100644 --- a/surfsense_web/app/desktop/suggestion/page.tsx +++ b/surfsense_web/app/desktop/suggestion/page.tsx @@ -24,6 +24,8 @@ function friendlyError(raw: string | number): string { return "Please sign in to use suggestions."; if (lower.includes("no vision llm configured") || lower.includes("no llm configured")) return "No Vision LLM configured. Set one in search space settings."; + if (lower.includes("does not support vision")) + return "Selected model doesn\u2019t support vision. Set a vision-capable model in settings."; if (lower.includes("fetch") || lower.includes("network") || lower.includes("econnrefused")) return "Can\u2019t reach the server. Check your connection."; return "Something went wrong. Try again."; From 18103417bb10751c1b55b745bcd711596191dede Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sat, 4 Apr 2026 08:57:09 +0200 Subject: [PATCH 31/33] fix: harden autocomplete endpoint security and error handling --- .../app/routes/autocomplete_routes.py | 11 +++-- .../services/vision_autocomplete_service.py | 46 +++++++++++++++---- surfsense_web/app/desktop/suggestion/page.tsx | 18 ++++++++ .../app/desktop/suggestion/suggestion.css | 3 +- 4 files changed, 64 insertions(+), 14 deletions(-) diff --git a/surfsense_backend/app/routes/autocomplete_routes.py b/surfsense_backend/app/routes/autocomplete_routes.py index a6f66f408..bb56709cb 100644 --- a/surfsense_backend/app/routes/autocomplete_routes.py +++ b/surfsense_backend/app/routes/autocomplete_routes.py @@ -1,18 +1,21 @@ -from fastapi import APIRouter, Depends +from fastapi import APIRouter, Depends, HTTPException from fastapi.responses import StreamingResponse -from pydantic import BaseModel +from pydantic import BaseModel, Field from sqlalchemy.ext.asyncio import AsyncSession from app.db import User, get_async_session from app.services.new_streaming_service import VercelStreamingService from app.services.vision_autocomplete_service import stream_vision_autocomplete from app.users import current_active_user +from app.utils.rbac import check_search_space_access router = APIRouter(prefix="/autocomplete", tags=["autocomplete"]) +MAX_SCREENSHOT_SIZE = 20 * 1024 * 1024 # 20 MB base64 ceiling + class VisionAutocompleteRequest(BaseModel): - screenshot: str + screenshot: str = Field(..., max_length=MAX_SCREENSHOT_SIZE) search_space_id: int app_name: str = "" window_title: str = "" @@ -24,6 +27,8 @@ async def vision_autocomplete_stream( user: User = Depends(current_active_user), session: AsyncSession = Depends(get_async_session), ): + await check_search_space_access(session, user, body.search_space_id) + return StreamingResponse( stream_vision_autocomplete( body.screenshot, body.search_space_id, session, diff --git a/surfsense_backend/app/services/vision_autocomplete_service.py b/surfsense_backend/app/services/vision_autocomplete_service.py index e172c6522..2ddb56f51 100644 --- a/surfsense_backend/app/services/vision_autocomplete_service.py +++ b/surfsense_backend/app/services/vision_autocomplete_service.py @@ -61,11 +61,21 @@ def _build_system_prompt(app_name: str, window_title: str, kb_context: str) -> s return prompt +def _is_vision_unsupported_error(e: Exception) -> bool: + """Check if an exception indicates the model doesn't support vision/images.""" + msg = str(e).lower() + return "content must be a string" in msg or "does not support image" in msg + + async def _extract_query_from_screenshot( llm, screenshot_data_url: str, app_name: str = "", window_title: str = "", ) -> str | None: - """Ask the Vision LLM to describe what the user is working on.""" + """Ask the Vision LLM to describe what the user is working on. + + Raises vision-unsupported errors so the caller can return a + friendly message immediately instead of retrying with astream. + """ if app_name: prompt_text = EXTRACT_QUERY_PROMPT_WITH_APP.format( app_name=app_name, window_title=window_title, @@ -83,6 +93,8 @@ async def _extract_query_from_screenshot( query = response.content.strip() if hasattr(response, "content") else "" return query if query else None except Exception as e: + if _is_vision_unsupported_error(e): + raise logger.warning(f"Failed to extract query from screenshot: {e}") return None @@ -140,6 +152,10 @@ async def stream_vision_autocomplete( 3. Stream the final completion with screenshot + KB + app context """ streaming = VercelStreamingService() + vision_error_msg = ( + "The selected model does not support vision. " + "Please set a vision-capable model (e.g. GPT-4o, Gemini) in your search space settings." + ) llm = await get_vision_llm(session, search_space_id) if not llm: @@ -149,9 +165,17 @@ async def stream_vision_autocomplete( return kb_context = "" - query = await _extract_query_from_screenshot( - llm, screenshot_data_url, app_name=app_name, window_title=window_title, - ) + try: + query = await _extract_query_from_screenshot( + llm, screenshot_data_url, app_name=app_name, window_title=window_title, + ) + except Exception as e: + logger.warning(f"Vision autocomplete: selected model does not support vision: {e}") + yield streaming.format_message_start() + yield streaming.format_error(vision_error_msg) + yield streaming.format_done() + return + if query: kb_context = await _search_knowledge_base(session, search_space_id, query) @@ -171,10 +195,13 @@ async def stream_vision_autocomplete( ]), ] + text_started = False + text_id = "" try: yield streaming.format_message_start() text_id = streaming.generate_text_id() yield streaming.format_text_start(text_id) + text_started = True async for chunk in llm.astream(messages): token = chunk.content if hasattr(chunk, "content") else str(chunk) @@ -186,13 +213,12 @@ async def stream_vision_autocomplete( yield streaming.format_done() except Exception as e: - error_str = str(e).lower() - if "content must be a string" in error_str or "does not support image" in error_str: + if text_started: + yield streaming.format_text_end(text_id) + + if _is_vision_unsupported_error(e): logger.warning(f"Vision autocomplete: selected model does not support vision: {e}") - yield streaming.format_error( - "The selected model does not support vision. " - "Please set a vision-capable model (e.g. GPT-4o, Gemini) in your search space settings." - ) + yield streaming.format_error(vision_error_msg) else: logger.error(f"Vision autocomplete streaming error: {e}") yield streaming.format_error(str(e)) diff --git a/surfsense_web/app/desktop/suggestion/page.tsx b/surfsense_web/app/desktop/suggestion/page.tsx index 7188b73c6..03944867f 100644 --- a/surfsense_web/app/desktop/suggestion/page.tsx +++ b/surfsense_web/app/desktop/suggestion/page.tsx @@ -36,9 +36,17 @@ const AUTO_DISMISS_MS = 3000; export default function SuggestionPage() { const [suggestion, setSuggestion] = useState(""); const [isLoading, setIsLoading] = useState(true); + const [isDesktop, setIsDesktop] = useState(true); const [error, setError] = useState(null); const abortRef = useRef(null); + useEffect(() => { + if (!window.electronAPI?.onAutocompleteContext) { + setIsDesktop(false); + setIsLoading(false); + } + }, []); + useEffect(() => { if (!error) return; const timer = setTimeout(() => { @@ -153,6 +161,16 @@ export default function SuggestionPage() { return cleanup; }, [fetchSuggestion]); + if (!isDesktop) { + return ( +
+ + This page is only available in the SurfSense desktop app. + +
+ ); + } + if (error) { return (
diff --git a/surfsense_web/app/desktop/suggestion/suggestion.css b/surfsense_web/app/desktop/suggestion/suggestion.css index 712d12618..62f4d2ea7 100644 --- a/surfsense_web/app/desktop/suggestion/suggestion.css +++ b/surfsense_web/app/desktop/suggestion/suggestion.css @@ -1,4 +1,5 @@ -html, body { +html:has(.suggestion-body), +body:has(.suggestion-body) { margin: 0 !important; padding: 0 !important; background: transparent !important; From a180bf5576dc424646ea55a81e37b16c1c84743c Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sat, 4 Apr 2026 09:39:20 +0200 Subject: [PATCH 32/33] fix merge conflicts --- ...=> 119_add_vision_llm_id_to_search_spaces.py} | 10 +++++----- .../app/services/vision_autocomplete_service.py | 4 ++-- .../src/modules/autocomplete/index.ts | 16 +++++++++++----- .../components/editor-panel/editor-panel.tsx | 3 --- .../layout/ui/tabs/DocumentTabContent.tsx | 3 --- 5 files changed, 18 insertions(+), 18 deletions(-) rename surfsense_backend/alembic/versions/{117_add_vision_llm_id_to_search_spaces.py => 119_add_vision_llm_id_to_search_spaces.py} (86%) diff --git a/surfsense_backend/alembic/versions/117_add_vision_llm_id_to_search_spaces.py b/surfsense_backend/alembic/versions/119_add_vision_llm_id_to_search_spaces.py similarity index 86% rename from surfsense_backend/alembic/versions/117_add_vision_llm_id_to_search_spaces.py rename to surfsense_backend/alembic/versions/119_add_vision_llm_id_to_search_spaces.py index 2bec374c6..8e41d5e67 100644 --- a/surfsense_backend/alembic/versions/117_add_vision_llm_id_to_search_spaces.py +++ b/surfsense_backend/alembic/versions/119_add_vision_llm_id_to_search_spaces.py @@ -1,7 +1,7 @@ -"""117_add_vision_llm_id_to_search_spaces +"""119_add_vision_llm_id_to_search_spaces -Revision ID: 117 -Revises: 116 +Revision ID: 119 +Revises: 118 Adds vision_llm_id column to search_spaces for vision/screenshot analysis LLM role assignment. Defaults to 0 (Auto mode), same convention as @@ -16,8 +16,8 @@ import sqlalchemy as sa from alembic import op -revision: str = "117" -down_revision: str | None = "116" +revision: str = "119" +down_revision: str | None = "118" branch_labels: str | Sequence[str] | None = None depends_on: str | Sequence[str] | None = None diff --git a/surfsense_backend/app/services/vision_autocomplete_service.py b/surfsense_backend/app/services/vision_autocomplete_service.py index 2ddb56f51..f24a5c848 100644 --- a/surfsense_backend/app/services/vision_autocomplete_service.py +++ b/surfsense_backend/app/services/vision_autocomplete_service.py @@ -220,6 +220,6 @@ async def stream_vision_autocomplete( logger.warning(f"Vision autocomplete: selected model does not support vision: {e}") yield streaming.format_error(vision_error_msg) else: - logger.error(f"Vision autocomplete streaming error: {e}") - yield streaming.format_error(str(e)) + logger.error(f"Vision autocomplete streaming error: {e}", exc_info=True) + yield streaming.format_error("Autocomplete failed. Please try again.") yield streaming.format_done() diff --git a/surfsense_desktop/src/modules/autocomplete/index.ts b/surfsense_desktop/src/modules/autocomplete/index.ts index 0d5073de4..01a4cf913 100644 --- a/surfsense_desktop/src/modules/autocomplete/index.ts +++ b/surfsense_desktop/src/modules/autocomplete/index.ts @@ -11,6 +11,7 @@ const SHORTCUT = 'CommandOrControl+Shift+Space'; let autocompleteEnabled = true; let savedClipboard = ''; let sourceApp = ''; +let lastSearchSpaceId: string | null = null; function isSurfSenseWindow(): boolean { const app = getFrontmostApp(); @@ -36,19 +37,24 @@ async function triggerAutocomplete(): Promise { return; } - const cursor = screen.getCursorScreenPoint(); - const win = createSuggestionWindow(cursor.x, cursor.y); - - let searchSpaceId = '1'; const mainWin = getMainWindow(); if (mainWin && !mainWin.isDestroyed()) { const mainUrl = mainWin.webContents.getURL(); const match = mainUrl.match(/\/dashboard\/(\d+)/); if (match) { - searchSpaceId = match[1]; + lastSearchSpaceId = match[1]; } } + if (!lastSearchSpaceId) { + console.warn('[autocomplete] No active search space. Open a search space first.'); + return; + } + + const searchSpaceId = lastSearchSpaceId; + const cursor = screen.getCursorScreenPoint(); + const win = createSuggestionWindow(cursor.x, cursor.y); + win.webContents.once('did-finish-load', () => { const sw = getSuggestionWindow(); setTimeout(() => { diff --git a/surfsense_web/components/editor-panel/editor-panel.tsx b/surfsense_web/components/editor-panel/editor-panel.tsx index 248fe68eb..c307b3cea 100644 --- a/surfsense_web/components/editor-panel/editor-panel.tsx +++ b/surfsense_web/components/editor-panel/editor-panel.tsx @@ -96,9 +96,6 @@ export function EditorPanelContent({ } try { - const response = await authenticatedFetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-spaces/${searchSpaceId}/documents/${documentId}/editor-content`, - { method: "GET", signal: controller.signal } const url = new URL( `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-spaces/${searchSpaceId}/documents/${documentId}/editor-content` ); diff --git a/surfsense_web/components/layout/ui/tabs/DocumentTabContent.tsx b/surfsense_web/components/layout/ui/tabs/DocumentTabContent.tsx index d2ce3cc64..97c5b7cd9 100644 --- a/surfsense_web/components/layout/ui/tabs/DocumentTabContent.tsx +++ b/surfsense_web/components/layout/ui/tabs/DocumentTabContent.tsx @@ -81,9 +81,6 @@ export function DocumentTabContent({ documentId, searchSpaceId, title }: Documen } try { - const response = await authenticatedFetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-spaces/${searchSpaceId}/documents/${documentId}/editor-content`, - { method: "GET", signal: controller.signal } const url = new URL( `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-spaces/${searchSpaceId}/documents/${documentId}/editor-content` ); From 6b5b45d08d58e81c5cdadf76fd2f79f0dfe3821e Mon Sep 17 00:00:00 2001 From: Trevin Chow Date: Thu, 2 Apr 2026 05:05:04 -0700 Subject: [PATCH 33/33] perf: add passive option to scroll and touch event listeners Browsers wait for scroll/touch listeners to finish to check if preventDefault() is called, which delays scrolling. Since these handlers (navbar scroll detection, click-outside detection, and onboarding tour position updates) never call preventDefault(), marking them as passive lets the browser scroll without waiting. Fixes #1053 --- surfsense_web/components/homepage/navbar.tsx | 4 ++-- surfsense_web/components/onboarding-tour.tsx | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/surfsense_web/components/homepage/navbar.tsx b/surfsense_web/components/homepage/navbar.tsx index 6ed5265bc..ec00de47f 100644 --- a/surfsense_web/components/homepage/navbar.tsx +++ b/surfsense_web/components/homepage/navbar.tsx @@ -32,7 +32,7 @@ export const Navbar = ({ scrolledBgClassName }: NavbarProps = {}) => { }; handleScroll(); - window.addEventListener("scroll", handleScroll); + window.addEventListener("scroll", handleScroll, { passive: true }); return () => window.removeEventListener("scroll", handleScroll); }, []); @@ -132,7 +132,7 @@ const MobileNav = ({ navItems, isScrolled, scrolledBgClassName }: any) => { }; document.addEventListener("mousedown", handleClickOutside); - document.addEventListener("touchstart", handleClickOutside); + document.addEventListener("touchstart", handleClickOutside, { passive: true }); return () => { document.removeEventListener("mousedown", handleClickOutside); document.removeEventListener("touchstart", handleClickOutside); diff --git a/surfsense_web/components/onboarding-tour.tsx b/surfsense_web/components/onboarding-tour.tsx index a7d8f108b..178b6c97e 100644 --- a/surfsense_web/components/onboarding-tour.tsx +++ b/surfsense_web/components/onboarding-tour.tsx @@ -602,11 +602,11 @@ export function OnboardingTour() { }; window.addEventListener("resize", handleUpdate); - window.addEventListener("scroll", handleUpdate, true); + window.addEventListener("scroll", handleUpdate, { capture: true, passive: true }); return () => { window.removeEventListener("resize", handleUpdate); - window.removeEventListener("scroll", handleUpdate, true); + window.removeEventListener("scroll", handleUpdate, { capture: true }); }; }, [isActive, targetEl, currentStep?.placement]);