From f4d197f7022b0ceec560925f38fe6c125a77b0fc Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 2 Apr 2026 13:18:20 +0200 Subject: [PATCH 01/21] feat: add native module support for desktop autocomplete --- surfsense_desktop/.npmrc | 1 + surfsense_desktop/electron-builder.yml | 13 +++++ surfsense_desktop/package.json | 8 +++- surfsense_desktop/pnpm-lock.yaml | 50 ++++++++++++++++++++ surfsense_desktop/scripts/build-electron.mjs | 2 +- 5 files changed, 71 insertions(+), 3 deletions(-) create mode 100644 surfsense_desktop/.npmrc diff --git a/surfsense_desktop/.npmrc b/surfsense_desktop/.npmrc new file mode 100644 index 000000000..d67f37488 --- /dev/null +++ b/surfsense_desktop/.npmrc @@ -0,0 +1 @@ +node-linker=hoisted diff --git a/surfsense_desktop/electron-builder.yml b/surfsense_desktop/electron-builder.yml index eaca0f19b..74c69d223 100644 --- a/surfsense_desktop/electron-builder.yml +++ b/surfsense_desktop/electron-builder.yml @@ -9,6 +9,16 @@ directories: files: - dist/**/* - "!node_modules" + - node_modules/uiohook-napi/**/* + - "!node_modules/uiohook-napi/build" + - "!node_modules/uiohook-napi/src" + - "!node_modules/uiohook-napi/libuiohook" + - "!node_modules/uiohook-napi/binding.gyp" + - node_modules/node-gyp-build/**/* + - node_modules/node-mac-permissions/**/* + - "!node_modules/node-mac-permissions/build" + - "!node_modules/node-mac-permissions/src" + - "!node_modules/node-mac-permissions/binding.gyp" - "!src" - "!scripts" - "!release" @@ -29,6 +39,9 @@ extraResources: filter: ["**/*"] asarUnpack: - "**/*.node" + - "node_modules/uiohook-napi/**/*" + - "node_modules/node-gyp-build/**/*" + - "node_modules/node-mac-permissions/**/*" mac: icon: assets/icon.icns category: public.app-category.productivity diff --git a/surfsense_desktop/package.json b/surfsense_desktop/package.json index bd0cc67ab..a2e452b7c 100644 --- a/surfsense_desktop/package.json +++ b/surfsense_desktop/package.json @@ -11,12 +11,14 @@ "dist:mac": "pnpm build && electron-builder --mac --config electron-builder.yml", "dist:win": "pnpm build && electron-builder --win --config electron-builder.yml", "dist:linux": "pnpm build && electron-builder --linux --config electron-builder.yml", - "typecheck": "tsc --noEmit" + "typecheck": "tsc --noEmit", + "postinstall": "electron-rebuild" }, "author": "MODSetter", "license": "MIT", "packageManager": "pnpm@10.24.0", "devDependencies": { + "@electron/rebuild": "^4.0.3", "@types/node": "^25.5.0", "concurrently": "^9.2.1", "dotenv": "^17.3.1", @@ -28,6 +30,8 @@ }, "dependencies": { "electron-updater": "^6.8.3", - "get-port-please": "^3.2.0" + "get-port-please": "^3.2.0", + "node-mac-permissions": "^2.5.0", + "uiohook-napi": "^1.5.5" } } diff --git a/surfsense_desktop/pnpm-lock.yaml b/surfsense_desktop/pnpm-lock.yaml index ea65be0bb..82bad9456 100644 --- a/surfsense_desktop/pnpm-lock.yaml +++ b/surfsense_desktop/pnpm-lock.yaml @@ -14,7 +14,16 @@ importers: get-port-please: specifier: ^3.2.0 version: 3.2.0 + node-mac-permissions: + specifier: ^2.5.0 + version: 2.5.0 + uiohook-napi: + specifier: ^1.5.5 + version: 1.5.5 devDependencies: + '@electron/rebuild': + specifier: ^4.0.3 + version: 4.0.3 '@types/node': specifier: ^25.5.0 version: 25.5.0 @@ -343,6 +352,7 @@ packages: '@xmldom/xmldom@0.8.11': resolution: {integrity: sha512-cQzWCtO6C8TQiYl1ruKNn2U6Ao4o4WBBcbL61yJl84x+j5sOWWFU9X7DpND8XZG3daDppSsigMdfAIl2upQBRw==} engines: {node: '>=10.0.0'} + deprecated: this version has critical issues, please update to the latest version abbrev@3.0.1: resolution: {integrity: sha512-AO2ac6pjRB3SJmGJo+v5/aK6Omggp6fsLrs6wN9bd35ulu4cCwaAU9+7ZhXjeqHVkaHThLuzH0nZr0YpCDhygg==} @@ -424,6 +434,9 @@ packages: base64-js@1.5.1: resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} + bindings@1.5.0: + resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==} + bl@4.1.0: resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} @@ -738,6 +751,9 @@ packages: picomatch: optional: true + file-uri-to-path@1.0.0: + resolution: {integrity: sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==} + filelist@1.0.6: resolution: {integrity: sha512-5giy2PkLYY1cP39p17Ech+2xlpTRL9HLspOfEgm0L6CwBXBTgsK5ou0JtzYuepxkaQ/tvhCFIJ5uXo0OrM2DxA==} @@ -1103,14 +1119,25 @@ packages: node-addon-api@1.7.2: resolution: {integrity: sha512-ibPK3iA+vaY1eEjESkQkM0BbCqFOaZMiXRTtdB0u7b4djtY6JnsjvPdUHVMg6xQt3B8fpTTWHI9A+ADjM9frzg==} + node-addon-api@7.1.1: + resolution: {integrity: sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==} + node-api-version@0.2.1: resolution: {integrity: sha512-2xP/IGGMmmSQpI1+O/k72jF/ykvZ89JeuKX3TLJAYPDVLUalrshrLHkeVcCCZqG/eEa635cr8IBYzgnDvM2O8Q==} + node-gyp-build@4.8.4: + resolution: {integrity: sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==} + hasBin: true + node-gyp@11.5.0: resolution: {integrity: sha512-ra7Kvlhxn5V9Slyus0ygMa2h+UqExPqUIkfk7Pc8QTLT956JLSy51uWFwHtIYy0vI8cB4BDhc/S03+880My/LQ==} engines: {node: ^18.17.0 || >=20.5.0} hasBin: true + node-mac-permissions@2.5.0: + resolution: {integrity: sha512-zR8SVCaN3WqV1xwWd04XVAdzm3UTdjbxciLrZtB0Cc7F2Kd34AJfhPD4hm1HU0YH3oGUZO4X9OBLY5ijSTHsGw==} + os: [darwin] + nopt@8.1.0: resolution: {integrity: sha512-ieGu42u/Qsa4TFktmaKEwM6MQH0pOWnaB3htzh0JRtx84+Mebc0cbZYN5bC+6WTZ4+77xrL9Pn5m7CV6VIkV7A==} engines: {node: ^18.17.0 || >=20.5.0} @@ -1424,6 +1451,10 @@ packages: engines: {node: '>=14.17'} hasBin: true + uiohook-napi@1.5.5: + resolution: {integrity: sha512-oSlTdnECw2GBfsJPTbBQBeE4v/EXP0EZmX6BJq5nzH/JgFaBE8JpFwEA/kLhiEP7HxQw28FViWiYgdIZzWuuJQ==} + engines: {node: '>= 16'} + undici-types@7.16.0: resolution: {integrity: sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==} @@ -1923,6 +1954,10 @@ snapshots: base64-js@1.5.1: {} + bindings@1.5.0: + dependencies: + file-uri-to-path: 1.0.0 + bl@4.1.0: dependencies: buffer: 5.7.1 @@ -2348,6 +2383,8 @@ snapshots: optionalDependencies: picomatch: 4.0.3 + file-uri-to-path@1.0.0: {} + filelist@1.0.6: dependencies: minimatch: 5.1.9 @@ -2739,10 +2776,14 @@ snapshots: node-addon-api@1.7.2: optional: true + node-addon-api@7.1.1: {} + node-api-version@0.2.1: dependencies: semver: 7.7.4 + node-gyp-build@4.8.4: {} + node-gyp@11.5.0: dependencies: env-paths: 2.2.1 @@ -2758,6 +2799,11 @@ snapshots: transitivePeerDependencies: - supports-color + node-mac-permissions@2.5.0: + dependencies: + bindings: 1.5.0 + node-addon-api: 7.1.1 + nopt@8.1.0: dependencies: abbrev: 3.0.1 @@ -3064,6 +3110,10 @@ snapshots: typescript@5.9.3: {} + uiohook-napi@1.5.5: + dependencies: + node-gyp-build: 4.8.4 + undici-types@7.16.0: {} undici-types@7.18.2: {} diff --git a/surfsense_desktop/scripts/build-electron.mjs b/surfsense_desktop/scripts/build-electron.mjs index 923830296..83d941dd2 100644 --- a/surfsense_desktop/scripts/build-electron.mjs +++ b/surfsense_desktop/scripts/build-electron.mjs @@ -104,7 +104,7 @@ async function buildElectron() { bundle: true, platform: 'node', target: 'node18', - external: ['electron'], + external: ['electron', 'uiohook-napi', 'node-mac-permissions'], sourcemap: true, minify: false, define: { From fbd033d0a4ed756a789879b1bba69c08bc71506d Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 2 Apr 2026 13:19:21 +0200 Subject: [PATCH 02/21] feat: add autocomplete streaming endpoint with KB context --- surfsense_backend/app/routes/__init__.py | 2 + .../app/routes/autocomplete_routes.py | 136 ++++++++++++++++++ 2 files changed, 138 insertions(+) create mode 100644 surfsense_backend/app/routes/autocomplete_routes.py diff --git a/surfsense_backend/app/routes/__init__.py b/surfsense_backend/app/routes/__init__.py index 1937f11cb..a063b5976 100644 --- a/surfsense_backend/app/routes/__init__.py +++ b/surfsense_backend/app/routes/__init__.py @@ -3,6 +3,7 @@ from fastapi import APIRouter from .airtable_add_connector_route import ( router as airtable_add_connector_router, ) +from .autocomplete_routes import router as autocomplete_router from .chat_comments_routes import router as chat_comments_router from .circleback_webhook_route import router as circleback_webhook_router from .clickup_add_connector_route import router as clickup_add_connector_router @@ -95,3 +96,4 @@ router.include_router(incentive_tasks_router) # Incentive tasks for earning fre router.include_router(stripe_router) # Stripe checkout for additional page packs router.include_router(youtube_router) # YouTube playlist resolution router.include_router(prompts_router) +router.include_router(autocomplete_router) # Lightweight autocomplete with KB context diff --git a/surfsense_backend/app/routes/autocomplete_routes.py b/surfsense_backend/app/routes/autocomplete_routes.py new file mode 100644 index 000000000..9a285a723 --- /dev/null +++ b/surfsense_backend/app/routes/autocomplete_routes.py @@ -0,0 +1,136 @@ +import logging +from typing import AsyncGenerator + +from fastapi import APIRouter, Depends, Query +from fastapi.responses import StreamingResponse +from langchain_core.messages import HumanMessage, SystemMessage +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db import User, get_async_session +from app.retriever.chunks_hybrid_search import ChucksHybridSearchRetriever +from app.services.llm_service import get_agent_llm +from app.services.new_streaming_service import VercelStreamingService +from app.users import current_active_user + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/autocomplete", tags=["autocomplete"]) + +AUTOCOMPLETE_SYSTEM_PROMPT = """You are an inline text autocomplete engine. Your job is to complete the user's text naturally. + +Rules: +- Output ONLY the continuation text. Do NOT repeat what the user already typed. +- Keep completions concise: 1-3 sentences maximum. +- Match the user's tone, style, and language. +- If knowledge base context is provided, use it to make the completion factually accurate and personalized. +- Do NOT add quotes, explanations, or meta-commentary. +- Do NOT start with a space unless grammatically required. +- If you cannot produce a useful completion, output nothing.""" + +KB_CONTEXT_TEMPLATE = """ +Relevant knowledge base context (use this to personalize the completion): +--- +{kb_context} +--- +""" + + +async def _stream_autocomplete( + text: str, + cursor_position: int, + search_space_id: int, + session: AsyncSession, +) -> AsyncGenerator[str, None]: + """Stream an autocomplete response with KB context.""" + streaming_service = VercelStreamingService() + + try: + # Text before cursor is what we're completing + text_before_cursor = text[:cursor_position] if cursor_position >= 0 else text + + if not text_before_cursor.strip(): + yield streaming_service.format_message_start() + yield streaming_service.format_finish() + yield streaming_service.format_done() + return + + # Fast KB lookup: vector-only search, top 3 chunks, no planner LLM + kb_context = "" + try: + retriever = ChucksHybridSearchRetriever(session) + chunks = await retriever.vector_search( + query_text=text_before_cursor[-200:], # last 200 chars for relevance + top_k=3, + search_space_id=search_space_id, + ) + if chunks: + kb_snippets = [] + for chunk in chunks: + content = getattr(chunk, "content", None) or getattr(chunk, "chunk_text", "") + if content: + kb_snippets.append(content[:300]) + if kb_snippets: + kb_context = KB_CONTEXT_TEMPLATE.format( + kb_context="\n\n".join(kb_snippets) + ) + except Exception as e: + logger.warning(f"KB search failed for autocomplete, proceeding without context: {e}") + + # Get the search space's configured LLM + llm = await get_agent_llm(session, search_space_id) + if not llm: + yield streaming_service.format_message_start() + error_msg = "No LLM configured for this search space" + yield streaming_service.format_error(error_msg) + yield streaming_service.format_done() + return + + system_prompt = AUTOCOMPLETE_SYSTEM_PROMPT + if kb_context: + system_prompt += kb_context + + messages = [ + SystemMessage(content=system_prompt), + HumanMessage(content=f"Complete this text:\n{text_before_cursor}"), + ] + + # Stream the response + yield streaming_service.format_message_start() + text_id = streaming_service.generate_text_id() + yield streaming_service.format_text_start(text_id) + + async for chunk in llm.astream(messages): + token = chunk.content if hasattr(chunk, "content") else str(chunk) + if token: + yield streaming_service.format_text_delta(text_id, token) + + yield streaming_service.format_text_end(text_id) + yield streaming_service.format_finish() + yield streaming_service.format_done() + + except Exception as e: + logger.error(f"Autocomplete streaming error: {e}") + yield streaming_service.format_error(str(e)) + yield streaming_service.format_done() + + +@router.post("/stream") +async def autocomplete_stream( + text: str = Query(..., description="Current text in the input field"), + cursor_position: int = Query(-1, description="Cursor position in the text (-1 for end)"), + search_space_id: int = Query(..., description="Search space ID for KB context and LLM config"), + user: User = Depends(current_active_user), + session: AsyncSession = Depends(get_async_session), +): + """Stream an autocomplete suggestion based on the current text and KB context.""" + if cursor_position < 0: + cursor_position = len(text) + + return StreamingResponse( + _stream_autocomplete(text, cursor_position, search_space_id, session), + media_type="text/event-stream", + headers={ + **VercelStreamingService.get_response_headers(), + "X-Accel-Buffering": "no", + }, + ) From bcc227a4ddc34fbd6af3fb1115e01af0997bac85 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 2 Apr 2026 13:19:59 +0200 Subject: [PATCH 03/21] feat: add suggestion tooltip UI and autocomplete API types --- surfsense_web/app/suggestion/layout.tsx | 13 ++ surfsense_web/app/suggestion/page.tsx | 160 ++++++++++++++++++++ surfsense_web/app/suggestion/suggestion.css | 96 ++++++++++++ surfsense_web/types/window.d.ts | 6 + 4 files changed, 275 insertions(+) create mode 100644 surfsense_web/app/suggestion/layout.tsx create mode 100644 surfsense_web/app/suggestion/page.tsx create mode 100644 surfsense_web/app/suggestion/suggestion.css diff --git a/surfsense_web/app/suggestion/layout.tsx b/surfsense_web/app/suggestion/layout.tsx new file mode 100644 index 000000000..36b7e037b --- /dev/null +++ b/surfsense_web/app/suggestion/layout.tsx @@ -0,0 +1,13 @@ +import "./suggestion.css"; + +export const metadata = { + title: "SurfSense Suggestion", +}; + +export default function SuggestionLayout({ + children, +}: { + children: React.ReactNode; +}) { + return
{children}
; +} diff --git a/surfsense_web/app/suggestion/page.tsx b/surfsense_web/app/suggestion/page.tsx new file mode 100644 index 000000000..14dfab3af --- /dev/null +++ b/surfsense_web/app/suggestion/page.tsx @@ -0,0 +1,160 @@ +"use client"; + +import { useCallback, useEffect, useRef, useState } from "react"; +import { getBearerToken } from "@/lib/auth-utils"; + +type SSEEvent = + | { type: "text-delta"; id: string; delta: string } + | { type: "text-start"; id: string } + | { type: "text-end"; id: string } + | { type: "start"; messageId: string } + | { type: "finish" } + | { type: "error"; errorText: string }; + +export default function SuggestionPage() { + const [suggestion, setSuggestion] = useState(""); + const [isLoading, setIsLoading] = useState(true); + const [error, setError] = useState(null); + const abortRef = useRef(null); + + const fetchSuggestion = useCallback( + async (text: string, cursorPosition: number, searchSpaceId: string) => { + abortRef.current?.abort(); + const controller = new AbortController(); + abortRef.current = controller; + + setIsLoading(true); + setSuggestion(""); + setError(null); + + const token = getBearerToken(); + if (!token) { + setError("Not authenticated"); + setIsLoading(false); + return; + } + + const backendUrl = + process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL || "http://localhost:8000"; + + const params = new URLSearchParams({ + text, + cursor_position: String(cursorPosition), + search_space_id: searchSpaceId, + }); + + try { + const response = await fetch( + `${backendUrl}/api/v1/autocomplete/stream?${params}`, + { + method: "POST", + headers: { + Authorization: `Bearer ${token}`, + "Content-Type": "application/json", + }, + signal: controller.signal, + }, + ); + + if (!response.ok) { + setError(`Error: ${response.status}`); + setIsLoading(false); + return; + } + + if (!response.body) { + setError("No response body"); + setIsLoading(false); + return; + } + + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ""; + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + buffer += decoder.decode(value, { stream: true }); + const events = buffer.split(/\r?\n\r?\n/); + buffer = events.pop() || ""; + + for (const event of events) { + const lines = event.split(/\r?\n/); + for (const line of lines) { + if (!line.startsWith("data: ")) continue; + const data = line.slice(6).trim(); + if (!data || data === "[DONE]") continue; + + try { + const parsed: SSEEvent = JSON.parse(data); + if (parsed.type === "text-delta") { + setSuggestion((prev) => { + const updated = prev + parsed.delta; + window.electronAPI?.updateSuggestionText?.(updated); + return updated; + }); + } else if (parsed.type === "error") { + setError(parsed.errorText); + } + } catch { + continue; + } + } + } + } + } catch (err) { + if (err instanceof DOMException && err.name === "AbortError") return; + setError("Failed to get suggestion"); + } finally { + setIsLoading(false); + } + }, + [], + ); + + useEffect(() => { + if (!window.electronAPI?.onAutocompleteContext) return; + + const cleanup = window.electronAPI.onAutocompleteContext((data) => { + const searchSpaceId = data.searchSpaceId || "1"; + fetchSuggestion(data.text, data.cursorPosition, searchSpaceId); + }); + + return cleanup; + }, [fetchSuggestion]); + + if (error) { + return ( +
+ {error} +
+ ); + } + + if (isLoading && !suggestion) { + return ( +
+
+ + + +
+
+ ); + } + + if (!suggestion) return null; + + return ( +
+

{suggestion}

+
+ Tab accept + · + Esc dismiss +
+
+ ); +} diff --git a/surfsense_web/app/suggestion/suggestion.css b/surfsense_web/app/suggestion/suggestion.css new file mode 100644 index 000000000..e9471e7f8 --- /dev/null +++ b/surfsense_web/app/suggestion/suggestion.css @@ -0,0 +1,96 @@ +.suggestion-body { + margin: 0; + padding: 0; + background: transparent; + overflow: hidden; + font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; + -webkit-font-smoothing: antialiased; + user-select: none; + -webkit-app-region: no-drag; +} + +.suggestion-tooltip { + background: rgba(30, 30, 30, 0.95); + backdrop-filter: blur(12px); + -webkit-backdrop-filter: blur(12px); + border: 1px solid rgba(255, 255, 255, 0.1); + border-radius: 10px; + padding: 10px 14px; + margin: 4px; + max-width: 400px; + box-shadow: 0 8px 32px rgba(0, 0, 0, 0.4), + 0 2px 8px rgba(0, 0, 0, 0.2); +} + +.suggestion-text { + color: rgba(255, 255, 255, 0.9); + font-size: 13px; + line-height: 1.5; + margin: 0 0 8px 0; + word-wrap: break-word; + white-space: pre-wrap; +} + +.suggestion-hint { + color: rgba(255, 255, 255, 0.4); + font-size: 11px; + display: flex; + align-items: center; + gap: 4px; +} + +.suggestion-key { + background: rgba(255, 255, 255, 0.1); + border: 1px solid rgba(255, 255, 255, 0.15); + border-radius: 3px; + padding: 1px 5px; + font-size: 10px; + font-weight: 500; + color: rgba(255, 255, 255, 0.6); +} + +.suggestion-separator { + margin: 0 2px; +} + +.suggestion-error { + border-color: rgba(255, 80, 80, 0.3); +} + +.suggestion-error-text { + color: rgba(255, 120, 120, 0.9); + font-size: 12px; +} + +.suggestion-loading { + display: flex; + gap: 4px; + padding: 4px 0; +} + +.suggestion-dot { + width: 5px; + height: 5px; + border-radius: 50%; + background: rgba(255, 255, 255, 0.4); + animation: suggestion-pulse 1.2s infinite ease-in-out; +} + +.suggestion-dot:nth-child(2) { + animation-delay: 0.15s; +} + +.suggestion-dot:nth-child(3) { + animation-delay: 0.3s; +} + +@keyframes suggestion-pulse { + 0%, 80%, 100% { + opacity: 0.3; + transform: scale(0.8); + } + 40% { + opacity: 1; + transform: scale(1); + } +} diff --git a/surfsense_web/types/window.d.ts b/surfsense_web/types/window.d.ts index 9cf1aa596..a30358527 100644 --- a/surfsense_web/types/window.d.ts +++ b/surfsense_web/types/window.d.ts @@ -14,6 +14,12 @@ interface ElectronAPI { setQuickAskMode: (mode: string) => Promise; getQuickAskMode: () => Promise; replaceText: (text: string) => Promise; + onAutocompleteContext: (callback: (data: { text: string; cursorPosition: number; searchSpaceId?: string }) => void) => () => void; + acceptSuggestion: (text: string) => Promise; + dismissSuggestion: () => Promise; + updateSuggestionText: (text: string) => Promise; + setAutocompleteEnabled: (enabled: boolean) => Promise; + getAutocompleteEnabled: () => Promise; } declare global { From ec2b7851b6393147b09a9f24273fe84d4314371f Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 2 Apr 2026 13:26:32 +0200 Subject: [PATCH 04/21] feat: add macOS permission infrastructure for autocomplete --- surfsense_desktop/src/ipc/channels.ts | 5 ++ surfsense_desktop/src/ipc/handlers.ts | 22 +++++++++ surfsense_desktop/src/modules/permissions.ts | 50 ++++++++++++++++++++ surfsense_desktop/src/preload.ts | 5 ++ surfsense_web/types/window.d.ts | 9 ++++ 5 files changed, 91 insertions(+) create mode 100644 surfsense_desktop/src/modules/permissions.ts diff --git a/surfsense_desktop/src/ipc/channels.ts b/surfsense_desktop/src/ipc/channels.ts index 25ec1bc0e..a5209dcf3 100644 --- a/surfsense_desktop/src/ipc/channels.ts +++ b/surfsense_desktop/src/ipc/channels.ts @@ -6,4 +6,9 @@ export const IPC_CHANNELS = { SET_QUICK_ASK_MODE: 'set-quick-ask-mode', GET_QUICK_ASK_MODE: 'get-quick-ask-mode', REPLACE_TEXT: 'replace-text', + // Permissions + GET_PERMISSIONS_STATUS: 'get-permissions-status', + REQUEST_ACCESSIBILITY: 'request-accessibility', + REQUEST_INPUT_MONITORING: 'request-input-monitoring', + RESTART_APP: 'restart-app', } as const; diff --git a/surfsense_desktop/src/ipc/handlers.ts b/surfsense_desktop/src/ipc/handlers.ts index 18e343719..fc31329f1 100644 --- a/surfsense_desktop/src/ipc/handlers.ts +++ b/surfsense_desktop/src/ipc/handlers.ts @@ -1,5 +1,11 @@ import { app, ipcMain, shell } from 'electron'; import { IPC_CHANNELS } from './channels'; +import { + getPermissionsStatus, + requestAccessibility, + requestInputMonitoring, + restartApp, +} from '../modules/permissions'; export function registerIpcHandlers(): void { ipcMain.on(IPC_CHANNELS.OPEN_EXTERNAL, (_event, url: string) => { @@ -16,4 +22,20 @@ export function registerIpcHandlers(): void { ipcMain.handle(IPC_CHANNELS.GET_APP_VERSION, () => { return app.getVersion(); }); + + ipcMain.handle(IPC_CHANNELS.GET_PERMISSIONS_STATUS, () => { + return getPermissionsStatus(); + }); + + ipcMain.handle(IPC_CHANNELS.REQUEST_ACCESSIBILITY, () => { + requestAccessibility(); + }); + + ipcMain.handle(IPC_CHANNELS.REQUEST_INPUT_MONITORING, () => { + requestInputMonitoring(); + }); + + ipcMain.handle(IPC_CHANNELS.RESTART_APP, () => { + restartApp(); + }); } diff --git a/surfsense_desktop/src/modules/permissions.ts b/surfsense_desktop/src/modules/permissions.ts new file mode 100644 index 000000000..9a6159c9a --- /dev/null +++ b/surfsense_desktop/src/modules/permissions.ts @@ -0,0 +1,50 @@ +import { app } from 'electron'; + +type PermissionStatus = 'authorized' | 'denied' | 'not determined' | 'restricted' | 'limited'; + +export interface PermissionsStatus { + accessibility: PermissionStatus; + inputMonitoring: PermissionStatus; +} + +function isMac(): boolean { + return process.platform === 'darwin'; +} + +function getNodeMacPermissions() { + return require('node-mac-permissions'); +} + +export function getPermissionsStatus(): PermissionsStatus { + if (!isMac()) { + return { accessibility: 'authorized', inputMonitoring: 'authorized' }; + } + + const perms = getNodeMacPermissions(); + return { + accessibility: perms.getAuthStatus('accessibility'), + inputMonitoring: perms.getAuthStatus('input-monitoring'), + }; +} + +export function allPermissionsGranted(): boolean { + const status = getPermissionsStatus(); + return status.accessibility === 'authorized' && status.inputMonitoring === 'authorized'; +} + +export function requestAccessibility(): void { + if (!isMac()) return; + const perms = getNodeMacPermissions(); + perms.askForAccessibilityAccess(); +} + +export async function requestInputMonitoring(): Promise { + if (!isMac()) return 'authorized'; + const perms = getNodeMacPermissions(); + return perms.askForInputMonitoringAccess('listen'); +} + +export function restartApp(): void { + app.relaunch(); + app.exit(0); +} diff --git a/surfsense_desktop/src/preload.ts b/surfsense_desktop/src/preload.ts index 264ec25b3..069276489 100644 --- a/surfsense_desktop/src/preload.ts +++ b/surfsense_desktop/src/preload.ts @@ -21,4 +21,9 @@ contextBridge.exposeInMainWorld('electronAPI', { setQuickAskMode: (mode: string) => ipcRenderer.invoke(IPC_CHANNELS.SET_QUICK_ASK_MODE, mode), getQuickAskMode: () => ipcRenderer.invoke(IPC_CHANNELS.GET_QUICK_ASK_MODE), replaceText: (text: string) => ipcRenderer.invoke(IPC_CHANNELS.REPLACE_TEXT, text), + // Permissions + getPermissionsStatus: () => ipcRenderer.invoke(IPC_CHANNELS.GET_PERMISSIONS_STATUS), + requestAccessibility: () => ipcRenderer.invoke(IPC_CHANNELS.REQUEST_ACCESSIBILITY), + requestInputMonitoring: () => ipcRenderer.invoke(IPC_CHANNELS.REQUEST_INPUT_MONITORING), + restartApp: () => ipcRenderer.invoke(IPC_CHANNELS.RESTART_APP), }); diff --git a/surfsense_web/types/window.d.ts b/surfsense_web/types/window.d.ts index a30358527..8cf331b42 100644 --- a/surfsense_web/types/window.d.ts +++ b/surfsense_web/types/window.d.ts @@ -14,6 +14,15 @@ interface ElectronAPI { setQuickAskMode: (mode: string) => Promise; getQuickAskMode: () => Promise; replaceText: (text: string) => Promise; + // Permissions + getPermissionsStatus: () => Promise<{ + accessibility: 'authorized' | 'denied' | 'not determined' | 'restricted' | 'limited'; + inputMonitoring: 'authorized' | 'denied' | 'not determined' | 'restricted' | 'limited'; + }>; + requestAccessibility: () => Promise; + requestInputMonitoring: () => Promise; + restartApp: () => Promise; + // Autocomplete onAutocompleteContext: (callback: (data: { text: string; cursorPosition: number; searchSpaceId?: string }) => void) => () => void; acceptSuggestion: (text: string) => Promise; dismissSuggestion: () => Promise; From eaabad38fcd2bfc8d6bef87f8ea60ea4d8192d78 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 2 Apr 2026 13:44:57 +0200 Subject: [PATCH 05/21] feat: add permission onboarding page and startup routing for macOS --- surfsense_desktop/src/ipc/handlers.ts | 4 +- surfsense_desktop/src/main.ts | 15 +- surfsense_desktop/src/modules/window.ts | 4 +- .../app/desktop/permissions/page.tsx | 212 ++++++++++++++++++ .../app/{ => desktop}/suggestion/layout.tsx | 0 .../app/{ => desktop}/suggestion/page.tsx | 0 .../{ => desktop}/suggestion/suggestion.css | 0 7 files changed, 228 insertions(+), 7 deletions(-) create mode 100644 surfsense_web/app/desktop/permissions/page.tsx rename surfsense_web/app/{ => desktop}/suggestion/layout.tsx (100%) rename surfsense_web/app/{ => desktop}/suggestion/page.tsx (100%) rename surfsense_web/app/{ => desktop}/suggestion/suggestion.css (100%) diff --git a/surfsense_desktop/src/ipc/handlers.ts b/surfsense_desktop/src/ipc/handlers.ts index fc31329f1..a6d82be4b 100644 --- a/surfsense_desktop/src/ipc/handlers.ts +++ b/surfsense_desktop/src/ipc/handlers.ts @@ -31,8 +31,8 @@ export function registerIpcHandlers(): void { requestAccessibility(); }); - ipcMain.handle(IPC_CHANNELS.REQUEST_INPUT_MONITORING, () => { - requestInputMonitoring(); + ipcMain.handle(IPC_CHANNELS.REQUEST_INPUT_MONITORING, async () => { + return await requestInputMonitoring(); }); ipcMain.handle(IPC_CHANNELS.RESTART_APP, () => { diff --git a/surfsense_desktop/src/main.ts b/surfsense_desktop/src/main.ts index 3ab41073b..bc164758b 100644 --- a/surfsense_desktop/src/main.ts +++ b/surfsense_desktop/src/main.ts @@ -7,6 +7,7 @@ import { setupAutoUpdater } from './modules/auto-updater'; import { setupMenu } from './modules/menu'; import { registerQuickAsk, unregisterQuickAsk } from './modules/quick-ask'; import { registerIpcHandlers } from './ipc/handlers'; +import { allPermissionsGranted } from './modules/permissions'; registerGlobalErrorHandlers(); @@ -16,7 +17,13 @@ if (!setupDeepLinks()) { registerIpcHandlers(); -// App lifecycle +function getInitialPath(): string { + if (process.platform === 'darwin' && !allPermissionsGranted()) { + return '/desktop/permissions'; + } + return '/dashboard'; +} + app.whenReady().then(async () => { setupMenu(); try { @@ -26,7 +33,9 @@ app.whenReady().then(async () => { setTimeout(() => app.quit(), 0); return; } - createMainWindow(); + + const initialPath = getInitialPath(); + createMainWindow(initialPath); registerQuickAsk(); setupAutoUpdater(); @@ -34,7 +43,7 @@ app.whenReady().then(async () => { app.on('activate', () => { if (BrowserWindow.getAllWindows().length === 0) { - createMainWindow(); + createMainWindow(getInitialPath()); } }); }); diff --git a/surfsense_desktop/src/modules/window.ts b/surfsense_desktop/src/modules/window.ts index 245814cad..7a77773d8 100644 --- a/surfsense_desktop/src/modules/window.ts +++ b/surfsense_desktop/src/modules/window.ts @@ -12,7 +12,7 @@ export function getMainWindow(): BrowserWindow | null { return mainWindow; } -export function createMainWindow(): BrowserWindow { +export function createMainWindow(initialPath = '/dashboard'): BrowserWindow { mainWindow = new BrowserWindow({ width: 1280, height: 800, @@ -33,7 +33,7 @@ export function createMainWindow(): BrowserWindow { mainWindow?.show(); }); - mainWindow.loadURL(`http://localhost:${getServerPort()}/dashboard`); + mainWindow.loadURL(`http://localhost:${getServerPort()}${initialPath}`); mainWindow.webContents.setWindowOpenHandler(({ url }) => { if (url.startsWith('http://localhost')) { diff --git a/surfsense_web/app/desktop/permissions/page.tsx b/surfsense_web/app/desktop/permissions/page.tsx new file mode 100644 index 000000000..2bcdc42df --- /dev/null +++ b/surfsense_web/app/desktop/permissions/page.tsx @@ -0,0 +1,212 @@ +"use client"; + +import { useEffect, useState } from "react"; +import { useRouter } from "next/navigation"; +import { Logo } from "@/components/Logo"; +import { Button } from "@/components/ui/button"; +import { Spinner } from "@/components/ui/spinner"; + +type PermissionStatus = "authorized" | "denied" | "not determined" | "restricted" | "limited"; + +interface PermissionsStatus { + accessibility: PermissionStatus; + inputMonitoring: PermissionStatus; +} + +const STEPS = [ + { + id: "input-monitoring", + title: "Input Monitoring", + description: "Helps you write faster by enriching your text with suggestions from your knowledge base.", + action: "requestInputMonitoring", + field: "inputMonitoring" as const, + }, + { + id: "accessibility", + title: "Accessibility", + description: "Lets you accept suggestions seamlessly, right where you're typing.", + action: "requestAccessibility", + field: "accessibility" as const, + }, +]; + +function StatusBadge({ status }: { status: PermissionStatus }) { + if (status === "authorized") { + return ( + + + Granted + + ); + } + if (status === "denied") { + return ( + + + Denied + + ); + } + return ( + + + Pending + + ); +} + +export default function DesktopPermissionsPage() { + const router = useRouter(); + const [permissions, setPermissions] = useState(null); + const [isElectron, setIsElectron] = useState(false); + + useEffect(() => { + if (!window.electronAPI) return; + setIsElectron(true); + + let interval: ReturnType | null = null; + + const isResolved = (s: string) => s === "authorized" || s === "restricted"; + + const poll = async () => { + const status = await window.electronAPI!.getPermissionsStatus(); + setPermissions(status); + + if (isResolved(status.accessibility) && isResolved(status.inputMonitoring)) { + if (interval) clearInterval(interval); + } + }; + + poll(); + interval = setInterval(poll, 2000); + return () => { if (interval) clearInterval(interval); }; + }, []); + + if (!isElectron) { + return ( +
+

This page is only available in the desktop app.

+
+ ); + } + + if (!permissions) { + return ( +
+ +
+ ); + } + + const allGranted = permissions.accessibility === "authorized" && permissions.inputMonitoring === "authorized"; + + const handleRequest = async (action: string) => { + if (action === "requestInputMonitoring") { + await window.electronAPI!.requestInputMonitoring(); + } else if (action === "requestAccessibility") { + await window.electronAPI!.requestAccessibility(); + } + }; + + const handleContinue = () => { + if (allGranted) { + window.electronAPI!.restartApp(); + } + }; + + const handleSkip = () => { + router.push("/dashboard"); + }; + + return ( +
+
+ {/* Header */} +
+ +
+

System Permissions

+

+ SurfSense needs two macOS permissions to provide system-wide autocomplete. +

+
+
+ + {/* Steps */} +
+ {STEPS.map((step, index) => { + const status = permissions[step.field]; + const isGranted = status === "authorized"; + + return ( +
+
+
+ + {isGranted ? "✓" : index + 1} + +
+

{step.title}

+

{step.description}

+
+
+ +
+ {!isGranted && ( +
+ + {status === "denied" && ( +

+ Toggle SurfSense on in System Settings to continue. +

+ )} +
+ )} +
+ ); + })} +
+ + {/* Footer */} +
+ {allGranted ? ( + <> + +

+ A restart is needed for permissions to take effect. +

+ + ) : ( + <> + + + + )} +
+
+
+ ); +} diff --git a/surfsense_web/app/suggestion/layout.tsx b/surfsense_web/app/desktop/suggestion/layout.tsx similarity index 100% rename from surfsense_web/app/suggestion/layout.tsx rename to surfsense_web/app/desktop/suggestion/layout.tsx diff --git a/surfsense_web/app/suggestion/page.tsx b/surfsense_web/app/desktop/suggestion/page.tsx similarity index 100% rename from surfsense_web/app/suggestion/page.tsx rename to surfsense_web/app/desktop/suggestion/page.tsx diff --git a/surfsense_web/app/suggestion/suggestion.css b/surfsense_web/app/desktop/suggestion/suggestion.css similarity index 100% rename from surfsense_web/app/suggestion/suggestion.css rename to surfsense_web/app/desktop/suggestion/suggestion.css From b2706b00a1bf793e8d1aa63235a0c53d5cc6766c Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 2 Apr 2026 14:29:12 +0200 Subject: [PATCH 06/21] feat: add autocomplete module with keystroke monitoring and IPC wiring --- surfsense_desktop/src/ipc/channels.ts | 7 + surfsense_desktop/src/main.ts | 3 + surfsense_desktop/src/modules/autocomplete.ts | 267 ++++++++++++++++++ surfsense_desktop/src/modules/platform.ts | 40 +++ surfsense_desktop/src/preload.ts | 13 + 5 files changed, 330 insertions(+) create mode 100644 surfsense_desktop/src/modules/autocomplete.ts diff --git a/surfsense_desktop/src/ipc/channels.ts b/surfsense_desktop/src/ipc/channels.ts index a5209dcf3..2965f516f 100644 --- a/surfsense_desktop/src/ipc/channels.ts +++ b/surfsense_desktop/src/ipc/channels.ts @@ -11,4 +11,11 @@ export const IPC_CHANNELS = { REQUEST_ACCESSIBILITY: 'request-accessibility', REQUEST_INPUT_MONITORING: 'request-input-monitoring', RESTART_APP: 'restart-app', + // Autocomplete + AUTOCOMPLETE_CONTEXT: 'autocomplete-context', + ACCEPT_SUGGESTION: 'accept-suggestion', + DISMISS_SUGGESTION: 'dismiss-suggestion', + UPDATE_SUGGESTION_TEXT: 'update-suggestion-text', + SET_AUTOCOMPLETE_ENABLED: 'set-autocomplete-enabled', + GET_AUTOCOMPLETE_ENABLED: 'get-autocomplete-enabled', } as const; diff --git a/surfsense_desktop/src/main.ts b/surfsense_desktop/src/main.ts index bc164758b..9623be82e 100644 --- a/surfsense_desktop/src/main.ts +++ b/surfsense_desktop/src/main.ts @@ -6,6 +6,7 @@ import { setupDeepLinks, handlePendingDeepLink } from './modules/deep-links'; import { setupAutoUpdater } from './modules/auto-updater'; import { setupMenu } from './modules/menu'; import { registerQuickAsk, unregisterQuickAsk } from './modules/quick-ask'; +import { registerAutocomplete, unregisterAutocomplete } from './modules/autocomplete'; import { registerIpcHandlers } from './ipc/handlers'; import { allPermissionsGranted } from './modules/permissions'; @@ -37,6 +38,7 @@ app.whenReady().then(async () => { const initialPath = getInitialPath(); createMainWindow(initialPath); registerQuickAsk(); + registerAutocomplete(); setupAutoUpdater(); handlePendingDeepLink(); @@ -56,4 +58,5 @@ app.on('window-all-closed', () => { app.on('will-quit', () => { unregisterQuickAsk(); + unregisterAutocomplete(); }); diff --git a/surfsense_desktop/src/modules/autocomplete.ts b/surfsense_desktop/src/modules/autocomplete.ts new file mode 100644 index 000000000..2b877723f --- /dev/null +++ b/surfsense_desktop/src/modules/autocomplete.ts @@ -0,0 +1,267 @@ +import { BrowserWindow, clipboard, ipcMain, screen, shell } from 'electron'; +import path from 'path'; +import { IPC_CHANNELS } from '../ipc/channels'; +import { allPermissionsGranted } from './permissions'; +import { getFieldContent, getFrontmostApp, hasAccessibilityPermission, simulatePaste } from './platform'; +import { getServerPort } from './server'; +import { getMainWindow } from './window'; + +const DEBOUNCE_MS = 600; +const TOOLTIP_WIDTH = 420; +const TOOLTIP_HEIGHT = 140; + +let uIOhook: any = null; +let UiohookKey: any = {}; +let IGNORED_KEYCODES: Set = new Set(); + +let suggestionWindow: BrowserWindow | null = null; +let debounceTimer: ReturnType | null = null; +let hookStarted = false; +let autocompleteEnabled = true; +let savedClipboard = ''; +let sourceApp = ''; +let pendingSuggestionText = ''; + +function loadUiohook(): boolean { + if (uIOhook) return true; + try { + const mod = require('uiohook-napi'); + uIOhook = mod.uIOhook; + UiohookKey = mod.UiohookKey; + IGNORED_KEYCODES = new Set([ + UiohookKey.Shift, UiohookKey.ShiftRight, + UiohookKey.Ctrl, UiohookKey.CtrlRight, + UiohookKey.Alt, UiohookKey.AltRight, + UiohookKey.Meta, UiohookKey.MetaRight, + UiohookKey.CapsLock, UiohookKey.NumLock, UiohookKey.ScrollLock, + UiohookKey.F1, UiohookKey.F2, UiohookKey.F3, UiohookKey.F4, + UiohookKey.F5, UiohookKey.F6, UiohookKey.F7, UiohookKey.F8, + UiohookKey.F9, UiohookKey.F10, UiohookKey.F11, UiohookKey.F12, + UiohookKey.PrintScreen, + UiohookKey.Insert, UiohookKey.Delete, + UiohookKey.Home, UiohookKey.End, + UiohookKey.PageUp, UiohookKey.PageDown, + UiohookKey.ArrowUp, UiohookKey.ArrowDown, + UiohookKey.ArrowLeft, UiohookKey.ArrowRight, + ]); + console.log('[autocomplete] uiohook-napi loaded'); + return true; + } catch (err) { + console.error('[autocomplete] Failed to load uiohook-napi:', err); + return false; + } +} + +function destroySuggestion(): void { + if (suggestionWindow && !suggestionWindow.isDestroyed()) { + suggestionWindow.close(); + } + suggestionWindow = null; +} + +function clampToScreen(x: number, y: number, w: number, h: number): { x: number; y: number } { + const display = screen.getDisplayNearestPoint({ x, y }); + const { x: dx, y: dy, width: dw, height: dh } = display.workArea; + return { + x: Math.max(dx, Math.min(x, dx + dw - w)), + y: Math.max(dy, Math.min(y, dy + dh - h)), + }; +} + +function createSuggestionWindow(x: number, y: number): BrowserWindow { + destroySuggestion(); + + const pos = clampToScreen(x, y + 20, TOOLTIP_WIDTH, TOOLTIP_HEIGHT); + + suggestionWindow = new BrowserWindow({ + width: TOOLTIP_WIDTH, + height: TOOLTIP_HEIGHT, + x: pos.x, + y: pos.y, + frame: false, + transparent: true, + focusable: false, + alwaysOnTop: true, + skipTaskbar: true, + resizable: false, + hasShadow: true, + type: 'panel', + webPreferences: { + preload: path.join(__dirname, 'preload.js'), + contextIsolation: true, + nodeIntegration: false, + sandbox: true, + }, + show: false, + }); + + suggestionWindow.loadURL(`http://localhost:${getServerPort()}/desktop/suggestion?t=${Date.now()}`); + + suggestionWindow.once('ready-to-show', () => { + suggestionWindow?.showInactive(); + }); + + suggestionWindow.webContents.setWindowOpenHandler(({ url }) => { + if (url.startsWith('http://localhost')) { + return { action: 'allow' }; + } + shell.openExternal(url); + return { action: 'deny' }; + }); + + suggestionWindow.on('closed', () => { + suggestionWindow = null; + }); + + return suggestionWindow; +} + +function clearDebounce(): void { + if (debounceTimer) { + clearTimeout(debounceTimer); + debounceTimer = null; + } +} + +function isSurfSenseWindow(): boolean { + const app = getFrontmostApp(); + return app === 'Electron' || app === 'SurfSense' || app === 'surfsense-desktop'; +} + +function onKeyDown(event: { keycode: number; ctrlKey?: boolean; metaKey?: boolean; altKey?: boolean }): void { + if (!autocompleteEnabled) return; + + if (event.keycode === UiohookKey.Tab && suggestionWindow && !suggestionWindow.isDestroyed()) { + if (pendingSuggestionText) { + acceptAndInject(pendingSuggestionText); + } + return; + } + + if (event.keycode === UiohookKey.Escape) { + if (suggestionWindow && !suggestionWindow.isDestroyed()) { + destroySuggestion(); + pendingSuggestionText = ''; + } + clearDebounce(); + return; + } + + if (IGNORED_KEYCODES.has(event.keycode)) return; + if (event.ctrlKey || event.metaKey || event.altKey) return; + if (isSurfSenseWindow()) return; + + if (suggestionWindow && !suggestionWindow.isDestroyed()) { + destroySuggestion(); + } + + clearDebounce(); + debounceTimer = setTimeout(() => { + triggerAutocomplete(); + }, DEBOUNCE_MS); +} + +async function triggerAutocomplete(): Promise { + if (!hasAccessibilityPermission()) return; + if (isSurfSenseWindow()) return; + + const fieldContent = getFieldContent(); + if (!fieldContent || !fieldContent.text.trim()) return; + if (fieldContent.text.trim().length < 5) return; + + sourceApp = getFrontmostApp(); + savedClipboard = clipboard.readText(); + + const cursor = screen.getCursorScreenPoint(); + const win = createSuggestionWindow(cursor.x, cursor.y); + + let searchSpaceId = '1'; + const mainWin = getMainWindow(); + if (mainWin && !mainWin.isDestroyed()) { + const mainUrl = mainWin.webContents.getURL(); + const match = mainUrl.match(/\/dashboard\/(\d+)/); + if (match) { + searchSpaceId = match[1]; + } + } + + win.webContents.once('did-finish-load', () => { + if (suggestionWindow && !suggestionWindow.isDestroyed()) { + suggestionWindow.webContents.send(IPC_CHANNELS.AUTOCOMPLETE_CONTEXT, { + text: fieldContent.text, + cursorPosition: fieldContent.cursorPosition, + searchSpaceId, + }); + } + }); +} + +async function acceptAndInject(text: string): Promise { + if (!sourceApp) return; + if (!hasAccessibilityPermission()) return; + + clipboard.writeText(text); + destroySuggestion(); + pendingSuggestionText = ''; + + try { + await new Promise((r) => setTimeout(r, 50)); + simulatePaste(); + await new Promise((r) => setTimeout(r, 100)); + clipboard.writeText(savedClipboard); + } catch { + clipboard.writeText(savedClipboard); + } +} + +function registerIpcHandlers(): void { + ipcMain.handle(IPC_CHANNELS.ACCEPT_SUGGESTION, async (_event, text: string) => { + await acceptAndInject(text); + }); + ipcMain.handle(IPC_CHANNELS.DISMISS_SUGGESTION, () => { + destroySuggestion(); + pendingSuggestionText = ''; + }); + ipcMain.handle(IPC_CHANNELS.UPDATE_SUGGESTION_TEXT, (_event, text: string) => { + pendingSuggestionText = text; + }); + ipcMain.handle(IPC_CHANNELS.SET_AUTOCOMPLETE_ENABLED, (_event, enabled: boolean) => { + autocompleteEnabled = enabled; + if (!enabled) { + clearDebounce(); + destroySuggestion(); + } + }); + ipcMain.handle(IPC_CHANNELS.GET_AUTOCOMPLETE_ENABLED, () => autocompleteEnabled); +} + +export function registerAutocomplete(): void { + registerIpcHandlers(); + + if (!allPermissionsGranted()) { + console.log('[autocomplete] Permissions not granted — hook not started'); + return; + } + + if (!loadUiohook()) { + console.error('[autocomplete] Cannot start: uiohook-napi failed to load'); + return; + } + + uIOhook.on('keydown', onKeyDown); + try { + uIOhook.start(); + hookStarted = true; + console.log('[autocomplete] uIOhook started'); + } catch (err) { + console.error('[autocomplete] uIOhook.start() failed:', err); + } +} + +export function unregisterAutocomplete(): void { + clearDebounce(); + destroySuggestion(); + if (uIOhook && hookStarted) { + try { uIOhook.stop(); } catch { /* already stopped */ } + } +} diff --git a/surfsense_desktop/src/modules/platform.ts b/surfsense_desktop/src/modules/platform.ts index 37e126799..262866d07 100644 --- a/surfsense_desktop/src/modules/platform.ts +++ b/surfsense_desktop/src/modules/platform.ts @@ -53,3 +53,43 @@ export function checkAccessibilityPermission(): boolean { if (process.platform !== 'darwin') return true; return systemPreferences.isTrustedAccessibilityClient(true); } + +export function hasAccessibilityPermission(): boolean { + if (process.platform !== 'darwin') return true; + return systemPreferences.isTrustedAccessibilityClient(false); +} + +export interface FieldContent { + text: string; + cursorPosition: number; +} + +export function getFieldContent(): FieldContent | null { + if (process.platform !== 'darwin') return null; + + try { + const text = execSync( + 'osascript -e \'tell application "System Events" to get value of attribute "AXValue" of focused UI element of first application process whose frontmost is true\'', + { timeout: 500 } + ).toString().trim(); + + let cursorPosition = text.length; + try { + const rangeStr = execSync( + 'osascript -e \'tell application "System Events" to get value of attribute "AXSelectedTextRange" of focused UI element of first application process whose frontmost is true\'', + { timeout: 500 } + ).toString().trim(); + + const locationMatch = rangeStr.match(/location[:\s]*(\d+)/i); + if (locationMatch) { + cursorPosition = parseInt(locationMatch[1], 10); + } + } catch { + // Fall back to end of text + } + + return { text, cursorPosition }; + } catch { + return null; + } +} diff --git a/surfsense_desktop/src/preload.ts b/surfsense_desktop/src/preload.ts index 069276489..956afcc46 100644 --- a/surfsense_desktop/src/preload.ts +++ b/surfsense_desktop/src/preload.ts @@ -26,4 +26,17 @@ contextBridge.exposeInMainWorld('electronAPI', { requestAccessibility: () => ipcRenderer.invoke(IPC_CHANNELS.REQUEST_ACCESSIBILITY), requestInputMonitoring: () => ipcRenderer.invoke(IPC_CHANNELS.REQUEST_INPUT_MONITORING), restartApp: () => ipcRenderer.invoke(IPC_CHANNELS.RESTART_APP), + // Autocomplete + onAutocompleteContext: (callback: (data: { text: string; cursorPosition: number; searchSpaceId?: string }) => void) => { + const listener = (_event: unknown, data: { text: string; cursorPosition: number; searchSpaceId?: string }) => callback(data); + ipcRenderer.on(IPC_CHANNELS.AUTOCOMPLETE_CONTEXT, listener); + return () => { + ipcRenderer.removeListener(IPC_CHANNELS.AUTOCOMPLETE_CONTEXT, listener); + }; + }, + acceptSuggestion: (text: string) => ipcRenderer.invoke(IPC_CHANNELS.ACCEPT_SUGGESTION, text), + dismissSuggestion: () => ipcRenderer.invoke(IPC_CHANNELS.DISMISS_SUGGESTION), + updateSuggestionText: (text: string) => ipcRenderer.invoke(IPC_CHANNELS.UPDATE_SUGGESTION_TEXT, text), + setAutocompleteEnabled: (enabled: boolean) => ipcRenderer.invoke(IPC_CHANNELS.SET_AUTOCOMPLETE_ENABLED, enabled), + getAutocompleteEnabled: () => ipcRenderer.invoke(IPC_CHANNELS.GET_AUTOCOMPLETE_ENABLED), }); From 6899134a20605b619e372564666a25eba5bb76fa Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 2 Apr 2026 14:37:26 +0200 Subject: [PATCH 07/21] feat: add autocomplete toggle in desktop settings --- .../components/DesktopContent.tsx | 79 +++++++++++++++++++ .../settings/user-settings-dialog.tsx | 7 +- 2 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 surfsense_web/app/dashboard/[search_space_id]/user-settings/components/DesktopContent.tsx diff --git a/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/DesktopContent.tsx b/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/DesktopContent.tsx new file mode 100644 index 000000000..1522e153f --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/DesktopContent.tsx @@ -0,0 +1,79 @@ +"use client"; + +import { useEffect, useState } from "react"; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; +import { Label } from "@/components/ui/label"; +import { Switch } from "@/components/ui/switch"; +import { Spinner } from "@/components/ui/spinner"; + +export function DesktopContent() { + const [isElectron, setIsElectron] = useState(false); + const [loading, setLoading] = useState(true); + const [enabled, setEnabled] = useState(true); + + useEffect(() => { + if (!window.electronAPI) { + setLoading(false); + return; + } + setIsElectron(true); + + window.electronAPI.getAutocompleteEnabled().then((val) => { + setEnabled(val); + setLoading(false); + }); + }, []); + + if (!isElectron) { + return ( +
+

+ Desktop settings are only available in the SurfSense desktop app. +

+
+ ); + } + + if (loading) { + return ( +
+ +
+ ); + } + + const handleToggle = async (checked: boolean) => { + setEnabled(checked); + await window.electronAPI!.setAutocompleteEnabled(checked); + }; + + return ( +
+ + + Autocomplete + + Get inline writing suggestions powered by your knowledge base as you type in any app. + + + +
+
+ +

+ Show suggestions while typing in other applications. +

+
+ +
+ + +
+ ); +} diff --git a/surfsense_web/components/settings/user-settings-dialog.tsx b/surfsense_web/components/settings/user-settings-dialog.tsx index 389ebc5fd..b74ff973b 100644 --- a/surfsense_web/components/settings/user-settings-dialog.tsx +++ b/surfsense_web/components/settings/user-settings-dialog.tsx @@ -1,13 +1,14 @@ "use client"; import { useAtom } from "jotai"; -import { Globe, KeyRound, Receipt, Sparkles, User } from "lucide-react"; +import { Globe, KeyRound, Monitor, Receipt, Sparkles, User } from "lucide-react"; import { useTranslations } from "next-intl"; import { ApiKeyContent } from "@/app/dashboard/[search_space_id]/user-settings/components/ApiKeyContent"; import { CommunityPromptsContent } from "@/app/dashboard/[search_space_id]/user-settings/components/CommunityPromptsContent"; import { ProfileContent } from "@/app/dashboard/[search_space_id]/user-settings/components/ProfileContent"; import { PromptsContent } from "@/app/dashboard/[search_space_id]/user-settings/components/PromptsContent"; import { PurchaseHistoryContent } from "@/app/dashboard/[search_space_id]/user-settings/components/PurchaseHistoryContent"; +import { DesktopContent } from "@/app/dashboard/[search_space_id]/user-settings/components/DesktopContent"; import { userSettingsDialogAtom } from "@/atoms/settings/settings-dialog.atoms"; import { SettingsDialog } from "@/components/settings/settings-dialog"; @@ -37,6 +38,9 @@ export function UserSettingsDialog() { label: "Purchase History", icon: , }, + ...(typeof window !== "undefined" && window.electronAPI + ? [{ value: "desktop", label: "Desktop", icon: }] + : []), ]; return ( @@ -54,6 +58,7 @@ export function UserSettingsDialog() { {state.initialTab === "prompts" && } {state.initialTab === "community-prompts" && } {state.initialTab === "purchases" && } + {state.initialTab === "desktop" && } ); From 9c1d9357c4e3b0fe5eb25f737069d6494cea2188 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 2 Apr 2026 20:19:16 +0200 Subject: [PATCH 08/21] refactor: fix dynamic tooltip resizing and split autocomplete into SPR modules --- surfsense_desktop/electron-builder.yml | 12 +- surfsense_desktop/package.json | 1 + surfsense_desktop/pnpm-lock.yaml | 3 + surfsense_desktop/scripts/build-electron.mjs | 2 +- surfsense_desktop/src/main.ts | 3 +- .../index.ts} | 181 ++++++++---------- .../modules/autocomplete/keystroke-buffer.ts | 76 ++++++++ .../modules/autocomplete/suggestion-window.ts | 103 ++++++++++ surfsense_desktop/src/modules/platform.ts | 49 ----- .../app/desktop/permissions/page.tsx | 13 +- surfsense_web/app/desktop/suggestion/page.tsx | 6 +- .../app/desktop/suggestion/suggestion.css | 70 ++++--- 12 files changed, 326 insertions(+), 193 deletions(-) rename surfsense_desktop/src/modules/{autocomplete.ts => autocomplete/index.ts} (55%) create mode 100644 surfsense_desktop/src/modules/autocomplete/keystroke-buffer.ts create mode 100644 surfsense_desktop/src/modules/autocomplete/suggestion-window.ts diff --git a/surfsense_desktop/electron-builder.yml b/surfsense_desktop/electron-builder.yml index 74c69d223..115b69c8e 100644 --- a/surfsense_desktop/electron-builder.yml +++ b/surfsense_desktop/electron-builder.yml @@ -10,13 +10,13 @@ files: - dist/**/* - "!node_modules" - node_modules/uiohook-napi/**/* - - "!node_modules/uiohook-napi/build" - "!node_modules/uiohook-napi/src" - "!node_modules/uiohook-napi/libuiohook" - "!node_modules/uiohook-napi/binding.gyp" - node_modules/node-gyp-build/**/* + - node_modules/bindings/**/* + - node_modules/file-uri-to-path/**/* - node_modules/node-mac-permissions/**/* - - "!node_modules/node-mac-permissions/build" - "!node_modules/node-mac-permissions/src" - "!node_modules/node-mac-permissions/binding.gyp" - "!src" @@ -41,13 +41,19 @@ asarUnpack: - "**/*.node" - "node_modules/uiohook-napi/**/*" - "node_modules/node-gyp-build/**/*" + - "node_modules/bindings/**/*" + - "node_modules/file-uri-to-path/**/*" - "node_modules/node-mac-permissions/**/*" mac: icon: assets/icon.icns category: public.app-category.productivity artifactName: "${productName}-${version}-${arch}.${ext}" - hardenedRuntime: true + hardenedRuntime: false gatekeeperAssess: false + extendInfo: + NSInputMonitoringUsageDescription: "SurfSense uses input monitoring to provide system-wide autocomplete suggestions as you type." + NSAccessibilityUsageDescription: "SurfSense uses accessibility features to read text fields and insert suggestions." + NSAppleEventsUsageDescription: "SurfSense uses Apple Events to read text from the active application and insert autocomplete suggestions." target: - target: dmg arch: [x64, arm64] diff --git a/surfsense_desktop/package.json b/surfsense_desktop/package.json index a2e452b7c..01a63b265 100644 --- a/surfsense_desktop/package.json +++ b/surfsense_desktop/package.json @@ -29,6 +29,7 @@ "wait-on": "^9.0.4" }, "dependencies": { + "bindings": "^1.5.0", "electron-updater": "^6.8.3", "get-port-please": "^3.2.0", "node-mac-permissions": "^2.5.0", diff --git a/surfsense_desktop/pnpm-lock.yaml b/surfsense_desktop/pnpm-lock.yaml index 82bad9456..d0b453d31 100644 --- a/surfsense_desktop/pnpm-lock.yaml +++ b/surfsense_desktop/pnpm-lock.yaml @@ -8,6 +8,9 @@ importers: .: dependencies: + bindings: + specifier: ^1.5.0 + version: 1.5.0 electron-updater: specifier: ^6.8.3 version: 6.8.3 diff --git a/surfsense_desktop/scripts/build-electron.mjs b/surfsense_desktop/scripts/build-electron.mjs index 83d941dd2..c2869ec46 100644 --- a/surfsense_desktop/scripts/build-electron.mjs +++ b/surfsense_desktop/scripts/build-electron.mjs @@ -104,7 +104,7 @@ async function buildElectron() { bundle: true, platform: 'node', target: 'node18', - external: ['electron', 'uiohook-napi', 'node-mac-permissions'], + external: ['electron', 'uiohook-napi', 'node-mac-permissions', 'bindings', 'file-uri-to-path'], sourcemap: true, minify: false, define: { diff --git a/surfsense_desktop/src/main.ts b/surfsense_desktop/src/main.ts index 9623be82e..c96453c6d 100644 --- a/surfsense_desktop/src/main.ts +++ b/surfsense_desktop/src/main.ts @@ -19,7 +19,8 @@ if (!setupDeepLinks()) { registerIpcHandlers(); function getInitialPath(): string { - if (process.platform === 'darwin' && !allPermissionsGranted()) { + const granted = allPermissionsGranted(); + if (process.platform === 'darwin' && !granted) { return '/desktop/permissions'; } return '/dashboard'; diff --git a/surfsense_desktop/src/modules/autocomplete.ts b/surfsense_desktop/src/modules/autocomplete/index.ts similarity index 55% rename from surfsense_desktop/src/modules/autocomplete.ts rename to surfsense_desktop/src/modules/autocomplete/index.ts index 2b877723f..2ea37d051 100644 --- a/surfsense_desktop/src/modules/autocomplete.ts +++ b/surfsense_desktop/src/modules/autocomplete/index.ts @@ -1,20 +1,19 @@ -import { BrowserWindow, clipboard, ipcMain, screen, shell } from 'electron'; -import path from 'path'; -import { IPC_CHANNELS } from '../ipc/channels'; -import { allPermissionsGranted } from './permissions'; -import { getFieldContent, getFrontmostApp, hasAccessibilityPermission, simulatePaste } from './platform'; -import { getServerPort } from './server'; -import { getMainWindow } from './window'; +import { clipboard, ipcMain, screen } from 'electron'; +import { IPC_CHANNELS } from '../../ipc/channels'; +import { getFrontmostApp, hasAccessibilityPermission, simulatePaste } from '../platform'; +import { getMainWindow } from '../window'; +import { + appendToBuffer, buildKeycodeMap, getBuffer, getBufferTrimmed, + getLastTrackedApp, removeLastChar, resetBuffer, resolveChar, setLastTrackedApp, +} from './keystroke-buffer'; +import { createSuggestionWindow, destroySuggestion, getSuggestionWindow } from './suggestion-window'; const DEBOUNCE_MS = 600; -const TOOLTIP_WIDTH = 420; -const TOOLTIP_HEIGHT = 140; let uIOhook: any = null; let UiohookKey: any = {}; let IGNORED_KEYCODES: Set = new Set(); -let suggestionWindow: BrowserWindow | null = null; let debounceTimer: ReturnType | null = null; let hookStarted = false; let autocompleteEnabled = true; @@ -38,12 +37,8 @@ function loadUiohook(): boolean { UiohookKey.F5, UiohookKey.F6, UiohookKey.F7, UiohookKey.F8, UiohookKey.F9, UiohookKey.F10, UiohookKey.F11, UiohookKey.F12, UiohookKey.PrintScreen, - UiohookKey.Insert, UiohookKey.Delete, - UiohookKey.Home, UiohookKey.End, - UiohookKey.PageUp, UiohookKey.PageDown, - UiohookKey.ArrowUp, UiohookKey.ArrowDown, - UiohookKey.ArrowLeft, UiohookKey.ArrowRight, ]); + buildKeycodeMap(); console.log('[autocomplete] uiohook-napi loaded'); return true; } catch (err) { @@ -52,70 +47,6 @@ function loadUiohook(): boolean { } } -function destroySuggestion(): void { - if (suggestionWindow && !suggestionWindow.isDestroyed()) { - suggestionWindow.close(); - } - suggestionWindow = null; -} - -function clampToScreen(x: number, y: number, w: number, h: number): { x: number; y: number } { - const display = screen.getDisplayNearestPoint({ x, y }); - const { x: dx, y: dy, width: dw, height: dh } = display.workArea; - return { - x: Math.max(dx, Math.min(x, dx + dw - w)), - y: Math.max(dy, Math.min(y, dy + dh - h)), - }; -} - -function createSuggestionWindow(x: number, y: number): BrowserWindow { - destroySuggestion(); - - const pos = clampToScreen(x, y + 20, TOOLTIP_WIDTH, TOOLTIP_HEIGHT); - - suggestionWindow = new BrowserWindow({ - width: TOOLTIP_WIDTH, - height: TOOLTIP_HEIGHT, - x: pos.x, - y: pos.y, - frame: false, - transparent: true, - focusable: false, - alwaysOnTop: true, - skipTaskbar: true, - resizable: false, - hasShadow: true, - type: 'panel', - webPreferences: { - preload: path.join(__dirname, 'preload.js'), - contextIsolation: true, - nodeIntegration: false, - sandbox: true, - }, - show: false, - }); - - suggestionWindow.loadURL(`http://localhost:${getServerPort()}/desktop/suggestion?t=${Date.now()}`); - - suggestionWindow.once('ready-to-show', () => { - suggestionWindow?.showInactive(); - }); - - suggestionWindow.webContents.setWindowOpenHandler(({ url }) => { - if (url.startsWith('http://localhost')) { - return { action: 'allow' }; - } - shell.openExternal(url); - return { action: 'deny' }; - }); - - suggestionWindow.on('closed', () => { - suggestionWindow = null; - }); - - return suggestionWindow; -} - function clearDebounce(): void { if (debounceTimer) { clearTimeout(debounceTimer); @@ -128,10 +59,24 @@ function isSurfSenseWindow(): boolean { return app === 'Electron' || app === 'SurfSense' || app === 'surfsense-desktop'; } -function onKeyDown(event: { keycode: number; ctrlKey?: boolean; metaKey?: boolean; altKey?: boolean }): void { +function onKeyDown(event: { + keycode: number; + shiftKey?: boolean; + ctrlKey?: boolean; + metaKey?: boolean; + altKey?: boolean; +}): void { if (!autocompleteEnabled) return; - if (event.keycode === UiohookKey.Tab && suggestionWindow && !suggestionWindow.isDestroyed()) { + const currentApp = getFrontmostApp(); + if (currentApp !== getLastTrackedApp()) { + resetBuffer(); + setLastTrackedApp(currentApp); + } + + const win = getSuggestionWindow(); + + if (event.keycode === UiohookKey.Tab && win && !win.isDestroyed()) { if (pendingSuggestionText) { acceptAndInject(pendingSuggestionText); } @@ -139,7 +84,7 @@ function onKeyDown(event: { keycode: number; ctrlKey?: boolean; metaKey?: boolea } if (event.keycode === UiohookKey.Escape) { - if (suggestionWindow && !suggestionWindow.isDestroyed()) { + if (win && !win.isDestroyed()) { destroySuggestion(); pendingSuggestionText = ''; } @@ -147,11 +92,41 @@ function onKeyDown(event: { keycode: number; ctrlKey?: boolean; metaKey?: boolea return; } - if (IGNORED_KEYCODES.has(event.keycode)) return; - if (event.ctrlKey || event.metaKey || event.altKey) return; - if (isSurfSenseWindow()) return; + if (currentApp === 'Electron' || currentApp === 'SurfSense' || currentApp === 'surfsense-desktop') { + return; + } - if (suggestionWindow && !suggestionWindow.isDestroyed()) { + if (event.ctrlKey || event.metaKey || event.altKey) { + resetBuffer(); + clearDebounce(); + return; + } + + if (event.keycode === UiohookKey.Backspace) { + removeLastChar(); + } else if (event.keycode === UiohookKey.Delete) { + // forward delete doesn't affect our trailing buffer + } else if (event.keycode === UiohookKey.Enter) { + appendToBuffer('\n'); + } else if (event.keycode === UiohookKey.Space) { + appendToBuffer(' '); + } else if ( + event.keycode === UiohookKey.ArrowLeft || event.keycode === UiohookKey.ArrowRight || + event.keycode === UiohookKey.ArrowUp || event.keycode === UiohookKey.ArrowDown || + event.keycode === UiohookKey.Home || event.keycode === UiohookKey.End || + event.keycode === UiohookKey.PageUp || event.keycode === UiohookKey.PageDown + ) { + resetBuffer(); + clearDebounce(); + return; + } else if (IGNORED_KEYCODES.has(event.keycode)) { + return; + } else { + const ch = resolveChar(event.keycode, !!event.shiftKey); + if (ch) appendToBuffer(ch); + } + + if (win && !win.isDestroyed()) { destroySuggestion(); } @@ -161,13 +136,16 @@ function onKeyDown(event: { keycode: number; ctrlKey?: boolean; metaKey?: boolea }, DEBOUNCE_MS); } +function onMouseClick(): void { + resetBuffer(); +} + async function triggerAutocomplete(): Promise { if (!hasAccessibilityPermission()) return; if (isSurfSenseWindow()) return; - const fieldContent = getFieldContent(); - if (!fieldContent || !fieldContent.text.trim()) return; - if (fieldContent.text.trim().length < 5) return; + const text = getBufferTrimmed(); + if (!text || text.length < 5) return; sourceApp = getFrontmostApp(); savedClipboard = clipboard.readText(); @@ -186,13 +164,16 @@ async function triggerAutocomplete(): Promise { } win.webContents.once('did-finish-load', () => { - if (suggestionWindow && !suggestionWindow.isDestroyed()) { - suggestionWindow.webContents.send(IPC_CHANNELS.AUTOCOMPLETE_CONTEXT, { - text: fieldContent.text, - cursorPosition: fieldContent.cursorPosition, - searchSpaceId, - }); - } + const sw = getSuggestionWindow(); + setTimeout(() => { + if (sw && !sw.isDestroyed()) { + sw.webContents.send(IPC_CHANNELS.AUTOCOMPLETE_CONTEXT, { + text: getBuffer(), + cursorPosition: getBuffer().length, + searchSpaceId, + }); + } + }, 300); }); } @@ -209,6 +190,7 @@ async function acceptAndInject(text: string): Promise { simulatePaste(); await new Promise((r) => setTimeout(r, 100)); clipboard.writeText(savedClipboard); + appendToBuffer(text); } catch { clipboard.writeText(savedClipboard); } @@ -238,21 +220,16 @@ function registerIpcHandlers(): void { export function registerAutocomplete(): void { registerIpcHandlers(); - if (!allPermissionsGranted()) { - console.log('[autocomplete] Permissions not granted — hook not started'); - return; - } - if (!loadUiohook()) { console.error('[autocomplete] Cannot start: uiohook-napi failed to load'); return; } uIOhook.on('keydown', onKeyDown); + uIOhook.on('click', onMouseClick); try { uIOhook.start(); hookStarted = true; - console.log('[autocomplete] uIOhook started'); } catch (err) { console.error('[autocomplete] uIOhook.start() failed:', err); } diff --git a/surfsense_desktop/src/modules/autocomplete/keystroke-buffer.ts b/surfsense_desktop/src/modules/autocomplete/keystroke-buffer.ts new file mode 100644 index 000000000..ca232d307 --- /dev/null +++ b/surfsense_desktop/src/modules/autocomplete/keystroke-buffer.ts @@ -0,0 +1,76 @@ +const MAX_BUFFER_LENGTH = 4000; +const KEYCODE_TO_CHAR: Record = {}; + +let keystrokeBuffer = ''; +let lastTrackedApp = ''; + +export function buildKeycodeMap(): void { + const letters: [string, number][] = [ + ['q', 16], ['w', 17], ['e', 18], ['r', 19], ['t', 20], + ['y', 21], ['u', 22], ['i', 23], ['o', 24], ['p', 25], + ['a', 30], ['s', 31], ['d', 32], ['f', 33], ['g', 34], + ['h', 35], ['j', 36], ['k', 37], ['l', 38], + ['z', 44], ['x', 45], ['c', 46], ['v', 47], + ['b', 48], ['n', 49], ['m', 50], + ]; + for (const [ch, code] of letters) { + KEYCODE_TO_CHAR[code] = [ch, ch.toUpperCase()]; + } + + const digits: [string, string, number][] = [ + ['1', '!', 2], ['2', '@', 3], ['3', '#', 4], ['4', '$', 5], + ['5', '%', 6], ['6', '^', 7], ['7', '&', 8], ['8', '*', 9], + ['9', '(', 10], ['0', ')', 11], + ]; + for (const [norm, shifted, code] of digits) { + KEYCODE_TO_CHAR[code] = [norm, shifted]; + } + + const punctuation: [string, string, number][] = [ + [';', ':', 39], ['=', '+', 13], [',', '<', 51], ['-', '_', 12], + ['.', '>', 52], ['/', '?', 53], ['`', '~', 41], ['[', '{', 26], + ['\\', '|', 43], [']', '}', 27], ["'", '"', 40], + ]; + for (const [norm, shifted, code] of punctuation) { + KEYCODE_TO_CHAR[code] = [norm, shifted]; + } +} + +export function resetBuffer(): void { + keystrokeBuffer = ''; +} + +export function appendToBuffer(char: string): void { + keystrokeBuffer += char; + if (keystrokeBuffer.length > MAX_BUFFER_LENGTH) { + keystrokeBuffer = keystrokeBuffer.slice(-MAX_BUFFER_LENGTH); + } +} + +export function removeLastChar(): void { + if (keystrokeBuffer.length > 0) { + keystrokeBuffer = keystrokeBuffer.slice(0, -1); + } +} + +export function getBuffer(): string { + return keystrokeBuffer; +} + +export function getBufferTrimmed(): string { + return keystrokeBuffer.trim(); +} + +export function getLastTrackedApp(): string { + return lastTrackedApp; +} + +export function setLastTrackedApp(app: string): void { + lastTrackedApp = app; +} + +export function resolveChar(keycode: number, shift: boolean): string | null { + const mapping = KEYCODE_TO_CHAR[keycode]; + if (!mapping) return null; + return shift ? mapping[1] : mapping[0]; +} diff --git a/surfsense_desktop/src/modules/autocomplete/suggestion-window.ts b/surfsense_desktop/src/modules/autocomplete/suggestion-window.ts new file mode 100644 index 000000000..f03930cf6 --- /dev/null +++ b/surfsense_desktop/src/modules/autocomplete/suggestion-window.ts @@ -0,0 +1,103 @@ +import { BrowserWindow, screen, shell } from 'electron'; +import path from 'path'; +import { getServerPort } from '../server'; + +const TOOLTIP_WIDTH = 420; +const TOOLTIP_HEIGHT = 38; +const MAX_HEIGHT = 400; + +let suggestionWindow: BrowserWindow | null = null; +let resizeTimer: ReturnType | null = null; + +function clampToScreen(x: number, y: number, w: number, h: number): { x: number; y: number } { + const display = screen.getDisplayNearestPoint({ x, y }); + const { x: dx, y: dy, width: dw, height: dh } = display.workArea; + return { + x: Math.max(dx, Math.min(x, dx + dw - w)), + y: Math.max(dy, Math.min(y, dy + dh - h)), + }; +} + +function stopResizePolling(): void { + if (resizeTimer) { clearInterval(resizeTimer); resizeTimer = null; } +} + +function startResizePolling(win: BrowserWindow): void { + stopResizePolling(); + let lastH = 0; + resizeTimer = setInterval(async () => { + if (!win || win.isDestroyed()) { stopResizePolling(); return; } + try { + const h: number = await win.webContents.executeJavaScript( + `document.body.scrollHeight` + ); + if (h > 0 && h !== lastH) { + lastH = h; + const clamped = Math.min(h, MAX_HEIGHT); + const bounds = win.getBounds(); + win.setBounds({ x: bounds.x, y: bounds.y, width: TOOLTIP_WIDTH, height: clamped }); + } + } catch {} + }, 150); +} + +export function getSuggestionWindow(): BrowserWindow | null { + return suggestionWindow; +} + +export function destroySuggestion(): void { + stopResizePolling(); + if (suggestionWindow && !suggestionWindow.isDestroyed()) { + suggestionWindow.close(); + } + suggestionWindow = null; +} + +export function createSuggestionWindow(x: number, y: number): BrowserWindow { + destroySuggestion(); + + const pos = clampToScreen(x, y + 20, TOOLTIP_WIDTH, TOOLTIP_HEIGHT); + + suggestionWindow = new BrowserWindow({ + width: TOOLTIP_WIDTH, + height: TOOLTIP_HEIGHT, + x: pos.x, + y: pos.y, + frame: false, + transparent: true, + focusable: false, + alwaysOnTop: true, + skipTaskbar: true, + hasShadow: true, + type: 'panel', + webPreferences: { + preload: path.join(__dirname, '..', 'preload.js'), + contextIsolation: true, + nodeIntegration: false, + sandbox: true, + }, + show: false, + }); + + suggestionWindow.loadURL(`http://localhost:${getServerPort()}/desktop/suggestion?t=${Date.now()}`); + + suggestionWindow.once('ready-to-show', () => { + suggestionWindow?.showInactive(); + if (suggestionWindow) startResizePolling(suggestionWindow); + }); + + suggestionWindow.webContents.setWindowOpenHandler(({ url }) => { + if (url.startsWith('http://localhost')) { + return { action: 'allow' }; + } + shell.openExternal(url); + return { action: 'deny' }; + }); + + suggestionWindow.on('closed', () => { + stopResizePolling(); + suggestionWindow = null; + }); + + return suggestionWindow; +} diff --git a/surfsense_desktop/src/modules/platform.ts b/surfsense_desktop/src/modules/platform.ts index 262866d07..1ab0c38fb 100644 --- a/surfsense_desktop/src/modules/platform.ts +++ b/surfsense_desktop/src/modules/platform.ts @@ -19,20 +19,6 @@ export function getFrontmostApp(): string { return ''; } -export function getSelectedText(): string { - try { - if (process.platform === 'darwin') { - return execSync( - 'osascript -e \'tell application "System Events" to get value of attribute "AXSelectedText" of focused UI element of first application process whose frontmost is true\'' - ).toString().trim(); - } - // Windows: no reliable accessibility API for selected text across apps - } catch { - return ''; - } - return ''; -} - export function simulateCopy(): void { if (process.platform === 'darwin') { execSync('osascript -e \'tell application "System Events" to keystroke "c" using command down\''); @@ -58,38 +44,3 @@ export function hasAccessibilityPermission(): boolean { if (process.platform !== 'darwin') return true; return systemPreferences.isTrustedAccessibilityClient(false); } - -export interface FieldContent { - text: string; - cursorPosition: number; -} - -export function getFieldContent(): FieldContent | null { - if (process.platform !== 'darwin') return null; - - try { - const text = execSync( - 'osascript -e \'tell application "System Events" to get value of attribute "AXValue" of focused UI element of first application process whose frontmost is true\'', - { timeout: 500 } - ).toString().trim(); - - let cursorPosition = text.length; - try { - const rangeStr = execSync( - 'osascript -e \'tell application "System Events" to get value of attribute "AXSelectedTextRange" of focused UI element of first application process whose frontmost is true\'', - { timeout: 500 } - ).toString().trim(); - - const locationMatch = rangeStr.match(/location[:\s]*(\d+)/i); - if (locationMatch) { - cursorPosition = parseInt(locationMatch[1], 10); - } - } catch { - // Fall back to end of text - } - - return { text, cursorPosition }; - } catch { - return null; - } -} diff --git a/surfsense_web/app/desktop/permissions/page.tsx b/surfsense_web/app/desktop/permissions/page.tsx index 2bcdc42df..8bde63357 100644 --- a/surfsense_web/app/desktop/permissions/page.tsx +++ b/surfsense_web/app/desktop/permissions/page.tsx @@ -169,11 +169,14 @@ export default function DesktopPermissionsPage() { > Open System Settings - {status === "denied" && ( -

- Toggle SurfSense on in System Settings to continue. -

- )} + {status === "denied" && ( +

+ Toggle SurfSense on in System Settings to continue. +

+ )} +

+ If SurfSense doesn't appear in the list, click + and select it from Applications. +

)} diff --git a/surfsense_web/app/desktop/suggestion/page.tsx b/surfsense_web/app/desktop/suggestion/page.tsx index 14dfab3af..69a19e3f1 100644 --- a/surfsense_web/app/desktop/suggestion/page.tsx +++ b/surfsense_web/app/desktop/suggestion/page.tsx @@ -151,9 +151,9 @@ export default function SuggestionPage() {

{suggestion}

- Tab accept - · - Esc dismiss + Tab accept + + Esc dismiss
); diff --git a/surfsense_web/app/desktop/suggestion/suggestion.css b/surfsense_web/app/desktop/suggestion/suggestion.css index e9471e7f8..0d3332103 100644 --- a/surfsense_web/app/desktop/suggestion/suggestion.css +++ b/surfsense_web/app/desktop/suggestion/suggestion.css @@ -1,8 +1,16 @@ +html, body { + margin: 0 !important; + padding: 0 !important; + background: transparent !important; + overflow: hidden !important; + height: auto !important; + width: 100% !important; +} + .suggestion-body { margin: 0; padding: 0; background: transparent; - overflow: hidden; font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; -webkit-font-smoothing: antialiased; user-select: none; @@ -10,69 +18,73 @@ } .suggestion-tooltip { - background: rgba(30, 30, 30, 0.95); - backdrop-filter: blur(12px); - -webkit-backdrop-filter: blur(12px); - border: 1px solid rgba(255, 255, 255, 0.1); - border-radius: 10px; - padding: 10px 14px; + background: #1e1e1e; + border: 1px solid #3c3c3c; + border-radius: 8px; + padding: 8px 12px; margin: 4px; max-width: 400px; - box-shadow: 0 8px 32px rgba(0, 0, 0, 0.4), - 0 2px 8px rgba(0, 0, 0, 0.2); + box-shadow: 0 4px 16px rgba(0, 0, 0, 0.5); } .suggestion-text { - color: rgba(255, 255, 255, 0.9); + color: #d4d4d4; font-size: 13px; - line-height: 1.5; - margin: 0 0 8px 0; + line-height: 1.45; + margin: 0 0 6px 0; word-wrap: break-word; white-space: pre-wrap; } .suggestion-hint { - color: rgba(255, 255, 255, 0.4); + color: #666; font-size: 11px; display: flex; align-items: center; - gap: 4px; + gap: 6px; + border-top: 1px solid #2a2a2a; + padding-top: 6px; } -.suggestion-key { - background: rgba(255, 255, 255, 0.1); - border: 1px solid rgba(255, 255, 255, 0.15); +.suggestion-hint kbd { + background: #2a2a2a; + border: 1px solid #3c3c3c; border-radius: 3px; - padding: 1px 5px; + padding: 0 4px; + font-family: inherit; font-size: 10px; - font-weight: 500; - color: rgba(255, 255, 255, 0.6); + font-weight: 600; + color: #999; + line-height: 18px; } .suggestion-separator { - margin: 0 2px; + width: 1px; + height: 10px; + background: #333; } .suggestion-error { - border-color: rgba(255, 80, 80, 0.3); + border-color: #5c2626; } .suggestion-error-text { - color: rgba(255, 120, 120, 0.9); + color: #f48771; font-size: 12px; } .suggestion-loading { display: flex; - gap: 4px; - padding: 4px 0; + gap: 5px; + padding: 2px 0; + justify-content: center; } .suggestion-dot { - width: 5px; - height: 5px; + width: 4px; + height: 4px; border-radius: 50%; - background: rgba(255, 255, 255, 0.4); + background: #666; animation: suggestion-pulse 1.2s infinite ease-in-out; } @@ -91,6 +103,6 @@ } 40% { opacity: 1; - transform: scale(1); + transform: scale(1.1); } } From 3e68d4aa3ed04c87f155a4da08a6610251755f74 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 2 Apr 2026 20:38:09 +0200 Subject: [PATCH 09/21] refactor: extract autocomplete service and fix tooltip screen-edge positioning --- .../app/routes/autocomplete_routes.py | 109 +---------------- .../app/services/autocomplete_service.py | 110 ++++++++++++++++++ .../modules/autocomplete/suggestion-window.ts | 27 +++-- 3 files changed, 130 insertions(+), 116 deletions(-) create mode 100644 surfsense_backend/app/services/autocomplete_service.py diff --git a/surfsense_backend/app/routes/autocomplete_routes.py b/surfsense_backend/app/routes/autocomplete_routes.py index 9a285a723..68c56d0e0 100644 --- a/surfsense_backend/app/routes/autocomplete_routes.py +++ b/surfsense_backend/app/routes/autocomplete_routes.py @@ -1,118 +1,14 @@ -import logging -from typing import AsyncGenerator - from fastapi import APIRouter, Depends, Query from fastapi.responses import StreamingResponse -from langchain_core.messages import HumanMessage, SystemMessage from sqlalchemy.ext.asyncio import AsyncSession from app.db import User, get_async_session -from app.retriever.chunks_hybrid_search import ChucksHybridSearchRetriever -from app.services.llm_service import get_agent_llm +from app.services.autocomplete_service import stream_autocomplete from app.services.new_streaming_service import VercelStreamingService from app.users import current_active_user -logger = logging.getLogger(__name__) - router = APIRouter(prefix="/autocomplete", tags=["autocomplete"]) -AUTOCOMPLETE_SYSTEM_PROMPT = """You are an inline text autocomplete engine. Your job is to complete the user's text naturally. - -Rules: -- Output ONLY the continuation text. Do NOT repeat what the user already typed. -- Keep completions concise: 1-3 sentences maximum. -- Match the user's tone, style, and language. -- If knowledge base context is provided, use it to make the completion factually accurate and personalized. -- Do NOT add quotes, explanations, or meta-commentary. -- Do NOT start with a space unless grammatically required. -- If you cannot produce a useful completion, output nothing.""" - -KB_CONTEXT_TEMPLATE = """ -Relevant knowledge base context (use this to personalize the completion): ---- -{kb_context} ---- -""" - - -async def _stream_autocomplete( - text: str, - cursor_position: int, - search_space_id: int, - session: AsyncSession, -) -> AsyncGenerator[str, None]: - """Stream an autocomplete response with KB context.""" - streaming_service = VercelStreamingService() - - try: - # Text before cursor is what we're completing - text_before_cursor = text[:cursor_position] if cursor_position >= 0 else text - - if not text_before_cursor.strip(): - yield streaming_service.format_message_start() - yield streaming_service.format_finish() - yield streaming_service.format_done() - return - - # Fast KB lookup: vector-only search, top 3 chunks, no planner LLM - kb_context = "" - try: - retriever = ChucksHybridSearchRetriever(session) - chunks = await retriever.vector_search( - query_text=text_before_cursor[-200:], # last 200 chars for relevance - top_k=3, - search_space_id=search_space_id, - ) - if chunks: - kb_snippets = [] - for chunk in chunks: - content = getattr(chunk, "content", None) or getattr(chunk, "chunk_text", "") - if content: - kb_snippets.append(content[:300]) - if kb_snippets: - kb_context = KB_CONTEXT_TEMPLATE.format( - kb_context="\n\n".join(kb_snippets) - ) - except Exception as e: - logger.warning(f"KB search failed for autocomplete, proceeding without context: {e}") - - # Get the search space's configured LLM - llm = await get_agent_llm(session, search_space_id) - if not llm: - yield streaming_service.format_message_start() - error_msg = "No LLM configured for this search space" - yield streaming_service.format_error(error_msg) - yield streaming_service.format_done() - return - - system_prompt = AUTOCOMPLETE_SYSTEM_PROMPT - if kb_context: - system_prompt += kb_context - - messages = [ - SystemMessage(content=system_prompt), - HumanMessage(content=f"Complete this text:\n{text_before_cursor}"), - ] - - # Stream the response - yield streaming_service.format_message_start() - text_id = streaming_service.generate_text_id() - yield streaming_service.format_text_start(text_id) - - async for chunk in llm.astream(messages): - token = chunk.content if hasattr(chunk, "content") else str(chunk) - if token: - yield streaming_service.format_text_delta(text_id, token) - - yield streaming_service.format_text_end(text_id) - yield streaming_service.format_finish() - yield streaming_service.format_done() - - except Exception as e: - logger.error(f"Autocomplete streaming error: {e}") - yield streaming_service.format_error(str(e)) - yield streaming_service.format_done() - @router.post("/stream") async def autocomplete_stream( @@ -122,12 +18,11 @@ async def autocomplete_stream( user: User = Depends(current_active_user), session: AsyncSession = Depends(get_async_session), ): - """Stream an autocomplete suggestion based on the current text and KB context.""" if cursor_position < 0: cursor_position = len(text) return StreamingResponse( - _stream_autocomplete(text, cursor_position, search_space_id, session), + stream_autocomplete(text, cursor_position, search_space_id, session), media_type="text/event-stream", headers={ **VercelStreamingService.get_response_headers(), diff --git a/surfsense_backend/app/services/autocomplete_service.py b/surfsense_backend/app/services/autocomplete_service.py new file mode 100644 index 000000000..7c172275d --- /dev/null +++ b/surfsense_backend/app/services/autocomplete_service.py @@ -0,0 +1,110 @@ +import logging +from typing import AsyncGenerator + +from langchain_core.messages import HumanMessage, SystemMessage +from sqlalchemy.ext.asyncio import AsyncSession + +from app.retriever.chunks_hybrid_search import ChucksHybridSearchRetriever +from app.services.llm_service import get_agent_llm +from app.services.new_streaming_service import VercelStreamingService + +logger = logging.getLogger(__name__) + +SYSTEM_PROMPT = """You are an inline text autocomplete engine. Your job is to complete the user's text naturally. + +Rules: +- Output ONLY the continuation text. Do NOT repeat what the user already typed. +- Keep completions concise: 1-3 sentences maximum. +- Match the user's tone, style, and language. +- If knowledge base context is provided, use it to make the completion factually accurate and personalized. +- Do NOT add quotes, explanations, or meta-commentary. +- Do NOT start with a space unless grammatically required. +- If you cannot produce a useful completion, output nothing.""" + +KB_CONTEXT_TEMPLATE = """ +Relevant knowledge base context (use this to personalize the completion): +--- +{kb_context} +--- +""" + + +async def _retrieve_kb_context( + session: AsyncSession, + text: str, + search_space_id: int, +) -> str: + try: + retriever = ChucksHybridSearchRetriever(session) + chunks = await retriever.vector_search( + query_text=text[-200:], + top_k=3, + search_space_id=search_space_id, + ) + if not chunks: + return "" + snippets = [] + for chunk in chunks: + content = getattr(chunk, "content", None) or getattr(chunk, "chunk_text", "") + if content: + snippets.append(content[:300]) + if not snippets: + return "" + return KB_CONTEXT_TEMPLATE.format(kb_context="\n\n".join(snippets)) + except Exception as e: + logger.warning(f"KB search failed for autocomplete, proceeding without context: {e}") + return "" + + +async def stream_autocomplete( + text: str, + cursor_position: int, + search_space_id: int, + session: AsyncSession, +) -> AsyncGenerator[str, None]: + """Build context, call the LLM, and yield SSE-formatted tokens.""" + streaming = VercelStreamingService() + text_before_cursor = text[:cursor_position] if cursor_position >= 0 else text + + if not text_before_cursor.strip(): + yield streaming.format_message_start() + yield streaming.format_finish() + yield streaming.format_done() + return + + kb_context = await _retrieve_kb_context(session, text_before_cursor, search_space_id) + + llm = await get_agent_llm(session, search_space_id) + if not llm: + yield streaming.format_message_start() + yield streaming.format_error("No LLM configured for this search space") + yield streaming.format_done() + return + + system_prompt = SYSTEM_PROMPT + if kb_context: + system_prompt += kb_context + + messages = [ + SystemMessage(content=system_prompt), + HumanMessage(content=f"Complete this text:\n{text_before_cursor}"), + ] + + try: + yield streaming.format_message_start() + text_id = streaming.generate_text_id() + yield streaming.format_text_start(text_id) + + async for chunk in llm.astream(messages): + token = chunk.content if hasattr(chunk, "content") else str(chunk) + if token: + yield streaming.format_text_delta(text_id, token) + + yield streaming.format_text_end(text_id) + yield streaming.format_finish() + yield streaming.format_done() + + except Exception as e: + logger.error(f"Autocomplete streaming error: {e}") + yield streaming.format_error(str(e)) + yield streaming.format_done() diff --git a/surfsense_desktop/src/modules/autocomplete/suggestion-window.ts b/surfsense_desktop/src/modules/autocomplete/suggestion-window.ts index f03930cf6..e8a2f3a91 100644 --- a/surfsense_desktop/src/modules/autocomplete/suggestion-window.ts +++ b/surfsense_desktop/src/modules/autocomplete/suggestion-window.ts @@ -8,14 +8,22 @@ const MAX_HEIGHT = 400; let suggestionWindow: BrowserWindow | null = null; let resizeTimer: ReturnType | null = null; +let cursorOrigin = { x: 0, y: 0 }; -function clampToScreen(x: number, y: number, w: number, h: number): { x: number; y: number } { - const display = screen.getDisplayNearestPoint({ x, y }); +const CURSOR_GAP = 20; + +function positionOnScreen(cursorX: number, cursorY: number, w: number, h: number): { x: number; y: number } { + const display = screen.getDisplayNearestPoint({ x: cursorX, y: cursorY }); const { x: dx, y: dy, width: dw, height: dh } = display.workArea; - return { - x: Math.max(dx, Math.min(x, dx + dw - w)), - y: Math.max(dy, Math.min(y, dy + dh - h)), - }; + + const x = Math.max(dx, Math.min(cursorX, dx + dw - w)); + + const spaceBelow = (dy + dh) - (cursorY + CURSOR_GAP); + const y = spaceBelow >= h + ? cursorY + CURSOR_GAP + : cursorY - h - CURSOR_GAP; + + return { x, y: Math.max(dy, y) }; } function stopResizePolling(): void { @@ -34,8 +42,8 @@ function startResizePolling(win: BrowserWindow): void { if (h > 0 && h !== lastH) { lastH = h; const clamped = Math.min(h, MAX_HEIGHT); - const bounds = win.getBounds(); - win.setBounds({ x: bounds.x, y: bounds.y, width: TOOLTIP_WIDTH, height: clamped }); + const pos = positionOnScreen(cursorOrigin.x, cursorOrigin.y, TOOLTIP_WIDTH, clamped); + win.setBounds({ x: pos.x, y: pos.y, width: TOOLTIP_WIDTH, height: clamped }); } } catch {} }, 150); @@ -55,8 +63,9 @@ export function destroySuggestion(): void { export function createSuggestionWindow(x: number, y: number): BrowserWindow { destroySuggestion(); + cursorOrigin = { x, y }; - const pos = clampToScreen(x, y + 20, TOOLTIP_WIDTH, TOOLTIP_HEIGHT); + const pos = positionOnScreen(x, y, TOOLTIP_WIDTH, TOOLTIP_HEIGHT); suggestionWindow = new BrowserWindow({ width: TOOLTIP_WIDTH, From a99d999a3658c09ce133940cc0dd15a3353d6cd7 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 2 Apr 2026 21:29:05 +0200 Subject: [PATCH 10/21] fix: correct preload.js path after autocomplete module restructure --- surfsense_desktop/src/modules/autocomplete/suggestion-window.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/surfsense_desktop/src/modules/autocomplete/suggestion-window.ts b/surfsense_desktop/src/modules/autocomplete/suggestion-window.ts index e8a2f3a91..8f61b2901 100644 --- a/surfsense_desktop/src/modules/autocomplete/suggestion-window.ts +++ b/surfsense_desktop/src/modules/autocomplete/suggestion-window.ts @@ -80,7 +80,7 @@ export function createSuggestionWindow(x: number, y: number): BrowserWindow { hasShadow: true, type: 'panel', webPreferences: { - preload: path.join(__dirname, '..', 'preload.js'), + preload: path.join(__dirname, 'preload.js'), contextIsolation: true, nodeIntegration: false, sandbox: true, From 8ba571566d2e1c3c296934c52e28a4202ede1dda Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 3 Apr 2026 16:10:52 +0200 Subject: [PATCH 11/21] remove uiohook-napi and keystroke monitoring --- surfsense_desktop/electron-builder.yml | 10 +- surfsense_desktop/package.json | 3 +- surfsense_desktop/pnpm-lock.yaml | 17 -- surfsense_desktop/scripts/build-electron.mjs | 2 +- surfsense_desktop/src/ipc/channels.ts | 1 - surfsense_desktop/src/ipc/handlers.ts | 5 - .../src/modules/autocomplete/index.ts | 156 +----------------- .../modules/autocomplete/keystroke-buffer.ts | 76 --------- surfsense_desktop/src/modules/permissions.ts | 12 +- surfsense_desktop/src/preload.ts | 1 - .../app/desktop/permissions/page.tsx | 122 ++++++-------- surfsense_web/types/window.d.ts | 2 - 12 files changed, 57 insertions(+), 350 deletions(-) delete mode 100644 surfsense_desktop/src/modules/autocomplete/keystroke-buffer.ts diff --git a/surfsense_desktop/electron-builder.yml b/surfsense_desktop/electron-builder.yml index 115b69c8e..3de0f266d 100644 --- a/surfsense_desktop/electron-builder.yml +++ b/surfsense_desktop/electron-builder.yml @@ -9,10 +9,6 @@ directories: files: - dist/**/* - "!node_modules" - - node_modules/uiohook-napi/**/* - - "!node_modules/uiohook-napi/src" - - "!node_modules/uiohook-napi/libuiohook" - - "!node_modules/uiohook-napi/binding.gyp" - node_modules/node-gyp-build/**/* - node_modules/bindings/**/* - node_modules/file-uri-to-path/**/* @@ -39,7 +35,6 @@ extraResources: filter: ["**/*"] asarUnpack: - "**/*.node" - - "node_modules/uiohook-napi/**/*" - "node_modules/node-gyp-build/**/*" - "node_modules/bindings/**/*" - "node_modules/file-uri-to-path/**/*" @@ -51,9 +46,8 @@ mac: hardenedRuntime: false gatekeeperAssess: false extendInfo: - NSInputMonitoringUsageDescription: "SurfSense uses input monitoring to provide system-wide autocomplete suggestions as you type." - NSAccessibilityUsageDescription: "SurfSense uses accessibility features to read text fields and insert suggestions." - NSAppleEventsUsageDescription: "SurfSense uses Apple Events to read text from the active application and insert autocomplete suggestions." + NSAccessibilityUsageDescription: "SurfSense uses accessibility features to insert suggestions into the active application." + NSAppleEventsUsageDescription: "SurfSense uses Apple Events to interact with the active application." target: - target: dmg arch: [x64, arm64] diff --git a/surfsense_desktop/package.json b/surfsense_desktop/package.json index 01a63b265..ab4fa0b8f 100644 --- a/surfsense_desktop/package.json +++ b/surfsense_desktop/package.json @@ -32,7 +32,6 @@ "bindings": "^1.5.0", "electron-updater": "^6.8.3", "get-port-please": "^3.2.0", - "node-mac-permissions": "^2.5.0", - "uiohook-napi": "^1.5.5" + "node-mac-permissions": "^2.5.0" } } diff --git a/surfsense_desktop/pnpm-lock.yaml b/surfsense_desktop/pnpm-lock.yaml index d0b453d31..96541c579 100644 --- a/surfsense_desktop/pnpm-lock.yaml +++ b/surfsense_desktop/pnpm-lock.yaml @@ -20,9 +20,6 @@ importers: node-mac-permissions: specifier: ^2.5.0 version: 2.5.0 - uiohook-napi: - specifier: ^1.5.5 - version: 1.5.5 devDependencies: '@electron/rebuild': specifier: ^4.0.3 @@ -1128,10 +1125,6 @@ packages: node-api-version@0.2.1: resolution: {integrity: sha512-2xP/IGGMmmSQpI1+O/k72jF/ykvZ89JeuKX3TLJAYPDVLUalrshrLHkeVcCCZqG/eEa635cr8IBYzgnDvM2O8Q==} - node-gyp-build@4.8.4: - resolution: {integrity: sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==} - hasBin: true - node-gyp@11.5.0: resolution: {integrity: sha512-ra7Kvlhxn5V9Slyus0ygMa2h+UqExPqUIkfk7Pc8QTLT956JLSy51uWFwHtIYy0vI8cB4BDhc/S03+880My/LQ==} engines: {node: ^18.17.0 || >=20.5.0} @@ -1454,10 +1447,6 @@ packages: engines: {node: '>=14.17'} hasBin: true - uiohook-napi@1.5.5: - resolution: {integrity: sha512-oSlTdnECw2GBfsJPTbBQBeE4v/EXP0EZmX6BJq5nzH/JgFaBE8JpFwEA/kLhiEP7HxQw28FViWiYgdIZzWuuJQ==} - engines: {node: '>= 16'} - undici-types@7.16.0: resolution: {integrity: sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==} @@ -2785,8 +2774,6 @@ snapshots: dependencies: semver: 7.7.4 - node-gyp-build@4.8.4: {} - node-gyp@11.5.0: dependencies: env-paths: 2.2.1 @@ -3113,10 +3100,6 @@ snapshots: typescript@5.9.3: {} - uiohook-napi@1.5.5: - dependencies: - node-gyp-build: 4.8.4 - undici-types@7.16.0: {} undici-types@7.18.2: {} diff --git a/surfsense_desktop/scripts/build-electron.mjs b/surfsense_desktop/scripts/build-electron.mjs index c2869ec46..9f507ea37 100644 --- a/surfsense_desktop/scripts/build-electron.mjs +++ b/surfsense_desktop/scripts/build-electron.mjs @@ -104,7 +104,7 @@ async function buildElectron() { bundle: true, platform: 'node', target: 'node18', - external: ['electron', 'uiohook-napi', 'node-mac-permissions', 'bindings', 'file-uri-to-path'], + external: ['electron', 'node-mac-permissions', 'bindings', 'file-uri-to-path'], sourcemap: true, minify: false, define: { diff --git a/surfsense_desktop/src/ipc/channels.ts b/surfsense_desktop/src/ipc/channels.ts index 2965f516f..905a84bc3 100644 --- a/surfsense_desktop/src/ipc/channels.ts +++ b/surfsense_desktop/src/ipc/channels.ts @@ -9,7 +9,6 @@ export const IPC_CHANNELS = { // Permissions GET_PERMISSIONS_STATUS: 'get-permissions-status', REQUEST_ACCESSIBILITY: 'request-accessibility', - REQUEST_INPUT_MONITORING: 'request-input-monitoring', RESTART_APP: 'restart-app', // Autocomplete AUTOCOMPLETE_CONTEXT: 'autocomplete-context', diff --git a/surfsense_desktop/src/ipc/handlers.ts b/surfsense_desktop/src/ipc/handlers.ts index a6d82be4b..8597a39e8 100644 --- a/surfsense_desktop/src/ipc/handlers.ts +++ b/surfsense_desktop/src/ipc/handlers.ts @@ -3,7 +3,6 @@ import { IPC_CHANNELS } from './channels'; import { getPermissionsStatus, requestAccessibility, - requestInputMonitoring, restartApp, } from '../modules/permissions'; @@ -31,10 +30,6 @@ export function registerIpcHandlers(): void { requestAccessibility(); }); - ipcMain.handle(IPC_CHANNELS.REQUEST_INPUT_MONITORING, async () => { - return await requestInputMonitoring(); - }); - ipcMain.handle(IPC_CHANNELS.RESTART_APP, () => { restartApp(); }); diff --git a/surfsense_desktop/src/modules/autocomplete/index.ts b/surfsense_desktop/src/modules/autocomplete/index.ts index 2ea37d051..3d9d67eef 100644 --- a/surfsense_desktop/src/modules/autocomplete/index.ts +++ b/surfsense_desktop/src/modules/autocomplete/index.ts @@ -1,152 +1,23 @@ -import { clipboard, ipcMain, screen } from 'electron'; +import { clipboard, globalShortcut, ipcMain, screen } from 'electron'; import { IPC_CHANNELS } from '../../ipc/channels'; import { getFrontmostApp, hasAccessibilityPermission, simulatePaste } from '../platform'; import { getMainWindow } from '../window'; -import { - appendToBuffer, buildKeycodeMap, getBuffer, getBufferTrimmed, - getLastTrackedApp, removeLastChar, resetBuffer, resolveChar, setLastTrackedApp, -} from './keystroke-buffer'; import { createSuggestionWindow, destroySuggestion, getSuggestionWindow } from './suggestion-window'; -const DEBOUNCE_MS = 600; - -let uIOhook: any = null; -let UiohookKey: any = {}; -let IGNORED_KEYCODES: Set = new Set(); - -let debounceTimer: ReturnType | null = null; -let hookStarted = false; let autocompleteEnabled = true; let savedClipboard = ''; let sourceApp = ''; let pendingSuggestionText = ''; -function loadUiohook(): boolean { - if (uIOhook) return true; - try { - const mod = require('uiohook-napi'); - uIOhook = mod.uIOhook; - UiohookKey = mod.UiohookKey; - IGNORED_KEYCODES = new Set([ - UiohookKey.Shift, UiohookKey.ShiftRight, - UiohookKey.Ctrl, UiohookKey.CtrlRight, - UiohookKey.Alt, UiohookKey.AltRight, - UiohookKey.Meta, UiohookKey.MetaRight, - UiohookKey.CapsLock, UiohookKey.NumLock, UiohookKey.ScrollLock, - UiohookKey.F1, UiohookKey.F2, UiohookKey.F3, UiohookKey.F4, - UiohookKey.F5, UiohookKey.F6, UiohookKey.F7, UiohookKey.F8, - UiohookKey.F9, UiohookKey.F10, UiohookKey.F11, UiohookKey.F12, - UiohookKey.PrintScreen, - ]); - buildKeycodeMap(); - console.log('[autocomplete] uiohook-napi loaded'); - return true; - } catch (err) { - console.error('[autocomplete] Failed to load uiohook-napi:', err); - return false; - } -} - -function clearDebounce(): void { - if (debounceTimer) { - clearTimeout(debounceTimer); - debounceTimer = null; - } -} - function isSurfSenseWindow(): boolean { const app = getFrontmostApp(); return app === 'Electron' || app === 'SurfSense' || app === 'surfsense-desktop'; } -function onKeyDown(event: { - keycode: number; - shiftKey?: boolean; - ctrlKey?: boolean; - metaKey?: boolean; - altKey?: boolean; -}): void { - if (!autocompleteEnabled) return; - - const currentApp = getFrontmostApp(); - if (currentApp !== getLastTrackedApp()) { - resetBuffer(); - setLastTrackedApp(currentApp); - } - - const win = getSuggestionWindow(); - - if (event.keycode === UiohookKey.Tab && win && !win.isDestroyed()) { - if (pendingSuggestionText) { - acceptAndInject(pendingSuggestionText); - } - return; - } - - if (event.keycode === UiohookKey.Escape) { - if (win && !win.isDestroyed()) { - destroySuggestion(); - pendingSuggestionText = ''; - } - clearDebounce(); - return; - } - - if (currentApp === 'Electron' || currentApp === 'SurfSense' || currentApp === 'surfsense-desktop') { - return; - } - - if (event.ctrlKey || event.metaKey || event.altKey) { - resetBuffer(); - clearDebounce(); - return; - } - - if (event.keycode === UiohookKey.Backspace) { - removeLastChar(); - } else if (event.keycode === UiohookKey.Delete) { - // forward delete doesn't affect our trailing buffer - } else if (event.keycode === UiohookKey.Enter) { - appendToBuffer('\n'); - } else if (event.keycode === UiohookKey.Space) { - appendToBuffer(' '); - } else if ( - event.keycode === UiohookKey.ArrowLeft || event.keycode === UiohookKey.ArrowRight || - event.keycode === UiohookKey.ArrowUp || event.keycode === UiohookKey.ArrowDown || - event.keycode === UiohookKey.Home || event.keycode === UiohookKey.End || - event.keycode === UiohookKey.PageUp || event.keycode === UiohookKey.PageDown - ) { - resetBuffer(); - clearDebounce(); - return; - } else if (IGNORED_KEYCODES.has(event.keycode)) { - return; - } else { - const ch = resolveChar(event.keycode, !!event.shiftKey); - if (ch) appendToBuffer(ch); - } - - if (win && !win.isDestroyed()) { - destroySuggestion(); - } - - clearDebounce(); - debounceTimer = setTimeout(() => { - triggerAutocomplete(); - }, DEBOUNCE_MS); -} - -function onMouseClick(): void { - resetBuffer(); -} - async function triggerAutocomplete(): Promise { if (!hasAccessibilityPermission()) return; if (isSurfSenseWindow()) return; - const text = getBufferTrimmed(); - if (!text || text.length < 5) return; - sourceApp = getFrontmostApp(); savedClipboard = clipboard.readText(); @@ -168,8 +39,8 @@ async function triggerAutocomplete(): Promise { setTimeout(() => { if (sw && !sw.isDestroyed()) { sw.webContents.send(IPC_CHANNELS.AUTOCOMPLETE_CONTEXT, { - text: getBuffer(), - cursorPosition: getBuffer().length, + text: '', + cursorPosition: 0, searchSpaceId, }); } @@ -190,7 +61,6 @@ async function acceptAndInject(text: string): Promise { simulatePaste(); await new Promise((r) => setTimeout(r, 100)); clipboard.writeText(savedClipboard); - appendToBuffer(text); } catch { clipboard.writeText(savedClipboard); } @@ -210,7 +80,6 @@ function registerIpcHandlers(): void { ipcMain.handle(IPC_CHANNELS.SET_AUTOCOMPLETE_ENABLED, (_event, enabled: boolean) => { autocompleteEnabled = enabled; if (!enabled) { - clearDebounce(); destroySuggestion(); } }); @@ -220,25 +89,10 @@ function registerIpcHandlers(): void { export function registerAutocomplete(): void { registerIpcHandlers(); - if (!loadUiohook()) { - console.error('[autocomplete] Cannot start: uiohook-napi failed to load'); - return; - } - - uIOhook.on('keydown', onKeyDown); - uIOhook.on('click', onMouseClick); - try { - uIOhook.start(); - hookStarted = true; - } catch (err) { - console.error('[autocomplete] uIOhook.start() failed:', err); - } + // TODO: Phase 2 — replace with vision-based trigger (desktopCapturer + globalShortcut) + console.log('[autocomplete] IPC handlers registered'); } export function unregisterAutocomplete(): void { - clearDebounce(); destroySuggestion(); - if (uIOhook && hookStarted) { - try { uIOhook.stop(); } catch { /* already stopped */ } - } } diff --git a/surfsense_desktop/src/modules/autocomplete/keystroke-buffer.ts b/surfsense_desktop/src/modules/autocomplete/keystroke-buffer.ts deleted file mode 100644 index ca232d307..000000000 --- a/surfsense_desktop/src/modules/autocomplete/keystroke-buffer.ts +++ /dev/null @@ -1,76 +0,0 @@ -const MAX_BUFFER_LENGTH = 4000; -const KEYCODE_TO_CHAR: Record = {}; - -let keystrokeBuffer = ''; -let lastTrackedApp = ''; - -export function buildKeycodeMap(): void { - const letters: [string, number][] = [ - ['q', 16], ['w', 17], ['e', 18], ['r', 19], ['t', 20], - ['y', 21], ['u', 22], ['i', 23], ['o', 24], ['p', 25], - ['a', 30], ['s', 31], ['d', 32], ['f', 33], ['g', 34], - ['h', 35], ['j', 36], ['k', 37], ['l', 38], - ['z', 44], ['x', 45], ['c', 46], ['v', 47], - ['b', 48], ['n', 49], ['m', 50], - ]; - for (const [ch, code] of letters) { - KEYCODE_TO_CHAR[code] = [ch, ch.toUpperCase()]; - } - - const digits: [string, string, number][] = [ - ['1', '!', 2], ['2', '@', 3], ['3', '#', 4], ['4', '$', 5], - ['5', '%', 6], ['6', '^', 7], ['7', '&', 8], ['8', '*', 9], - ['9', '(', 10], ['0', ')', 11], - ]; - for (const [norm, shifted, code] of digits) { - KEYCODE_TO_CHAR[code] = [norm, shifted]; - } - - const punctuation: [string, string, number][] = [ - [';', ':', 39], ['=', '+', 13], [',', '<', 51], ['-', '_', 12], - ['.', '>', 52], ['/', '?', 53], ['`', '~', 41], ['[', '{', 26], - ['\\', '|', 43], [']', '}', 27], ["'", '"', 40], - ]; - for (const [norm, shifted, code] of punctuation) { - KEYCODE_TO_CHAR[code] = [norm, shifted]; - } -} - -export function resetBuffer(): void { - keystrokeBuffer = ''; -} - -export function appendToBuffer(char: string): void { - keystrokeBuffer += char; - if (keystrokeBuffer.length > MAX_BUFFER_LENGTH) { - keystrokeBuffer = keystrokeBuffer.slice(-MAX_BUFFER_LENGTH); - } -} - -export function removeLastChar(): void { - if (keystrokeBuffer.length > 0) { - keystrokeBuffer = keystrokeBuffer.slice(0, -1); - } -} - -export function getBuffer(): string { - return keystrokeBuffer; -} - -export function getBufferTrimmed(): string { - return keystrokeBuffer.trim(); -} - -export function getLastTrackedApp(): string { - return lastTrackedApp; -} - -export function setLastTrackedApp(app: string): void { - lastTrackedApp = app; -} - -export function resolveChar(keycode: number, shift: boolean): string | null { - const mapping = KEYCODE_TO_CHAR[keycode]; - if (!mapping) return null; - return shift ? mapping[1] : mapping[0]; -} diff --git a/surfsense_desktop/src/modules/permissions.ts b/surfsense_desktop/src/modules/permissions.ts index 9a6159c9a..4ac671b7c 100644 --- a/surfsense_desktop/src/modules/permissions.ts +++ b/surfsense_desktop/src/modules/permissions.ts @@ -4,7 +4,6 @@ type PermissionStatus = 'authorized' | 'denied' | 'not determined' | 'restricted export interface PermissionsStatus { accessibility: PermissionStatus; - inputMonitoring: PermissionStatus; } function isMac(): boolean { @@ -17,19 +16,18 @@ function getNodeMacPermissions() { export function getPermissionsStatus(): PermissionsStatus { if (!isMac()) { - return { accessibility: 'authorized', inputMonitoring: 'authorized' }; + return { accessibility: 'authorized' }; } const perms = getNodeMacPermissions(); return { accessibility: perms.getAuthStatus('accessibility'), - inputMonitoring: perms.getAuthStatus('input-monitoring'), }; } export function allPermissionsGranted(): boolean { const status = getPermissionsStatus(); - return status.accessibility === 'authorized' && status.inputMonitoring === 'authorized'; + return status.accessibility === 'authorized'; } export function requestAccessibility(): void { @@ -38,12 +36,6 @@ export function requestAccessibility(): void { perms.askForAccessibilityAccess(); } -export async function requestInputMonitoring(): Promise { - if (!isMac()) return 'authorized'; - const perms = getNodeMacPermissions(); - return perms.askForInputMonitoringAccess('listen'); -} - export function restartApp(): void { app.relaunch(); app.exit(0); diff --git a/surfsense_desktop/src/preload.ts b/surfsense_desktop/src/preload.ts index 956afcc46..157fe216b 100644 --- a/surfsense_desktop/src/preload.ts +++ b/surfsense_desktop/src/preload.ts @@ -24,7 +24,6 @@ contextBridge.exposeInMainWorld('electronAPI', { // Permissions getPermissionsStatus: () => ipcRenderer.invoke(IPC_CHANNELS.GET_PERMISSIONS_STATUS), requestAccessibility: () => ipcRenderer.invoke(IPC_CHANNELS.REQUEST_ACCESSIBILITY), - requestInputMonitoring: () => ipcRenderer.invoke(IPC_CHANNELS.REQUEST_INPUT_MONITORING), restartApp: () => ipcRenderer.invoke(IPC_CHANNELS.RESTART_APP), // Autocomplete onAutocompleteContext: (callback: (data: { text: string; cursorPosition: number; searchSpaceId?: string }) => void) => { diff --git a/surfsense_web/app/desktop/permissions/page.tsx b/surfsense_web/app/desktop/permissions/page.tsx index 8bde63357..e0d3131e0 100644 --- a/surfsense_web/app/desktop/permissions/page.tsx +++ b/surfsense_web/app/desktop/permissions/page.tsx @@ -10,26 +10,8 @@ type PermissionStatus = "authorized" | "denied" | "not determined" | "restricted interface PermissionsStatus { accessibility: PermissionStatus; - inputMonitoring: PermissionStatus; } -const STEPS = [ - { - id: "input-monitoring", - title: "Input Monitoring", - description: "Helps you write faster by enriching your text with suggestions from your knowledge base.", - action: "requestInputMonitoring", - field: "inputMonitoring" as const, - }, - { - id: "accessibility", - title: "Accessibility", - description: "Lets you accept suggestions seamlessly, right where you're typing.", - action: "requestAccessibility", - field: "accessibility" as const, - }, -]; - function StatusBadge({ status }: { status: PermissionStatus }) { if (status === "authorized") { return ( @@ -66,13 +48,11 @@ export default function DesktopPermissionsPage() { let interval: ReturnType | null = null; - const isResolved = (s: string) => s === "authorized" || s === "restricted"; - const poll = async () => { const status = await window.electronAPI!.getPermissionsStatus(); setPermissions(status); - if (isResolved(status.accessibility) && isResolved(status.inputMonitoring)) { + if (status.accessibility === "authorized" || status.accessibility === "restricted") { if (interval) clearInterval(interval); } }; @@ -98,14 +78,10 @@ export default function DesktopPermissionsPage() { ); } - const allGranted = permissions.accessibility === "authorized" && permissions.inputMonitoring === "authorized"; + const allGranted = permissions.accessibility === "authorized"; - const handleRequest = async (action: string) => { - if (action === "requestInputMonitoring") { - await window.electronAPI!.requestInputMonitoring(); - } else if (action === "requestAccessibility") { - await window.electronAPI!.requestAccessibility(); - } + const handleRequest = async () => { + await window.electronAPI!.requestAccessibility(); }; const handleContinue = () => { @@ -127,61 +103,55 @@ export default function DesktopPermissionsPage() {

System Permissions

- SurfSense needs two macOS permissions to provide system-wide autocomplete. + SurfSense needs Accessibility permission to insert suggestions into the active application.

- {/* Steps */} + {/* Permission card */}
- {STEPS.map((step, index) => { - const status = permissions[step.field]; - const isGranted = status === "authorized"; - - return ( -
-
-
- - {isGranted ? "✓" : index + 1} - -
-

{step.title}

-

{step.description}

-
-
- -
- {!isGranted && ( -
- - {status === "denied" && ( -

- Toggle SurfSense on in System Settings to continue. -

- )} +
+
+
+ + {allGranted ? "\u2713" : "1"} + +
+

Accessibility

- If SurfSense doesn't appear in the list, click + and select it from Applications. + Lets SurfSense insert suggestions seamlessly, right where you're typing.

-
- )} +
- ); - })} + +
+ {!allGranted && ( +
+ + {permissions.accessibility === "denied" && ( +

+ Toggle SurfSense on in System Settings to continue. +

+ )} +

+ If SurfSense doesn't appear in the list, click + and select it from Applications. +

+
+ )} +
{/* Footer */} @@ -198,7 +168,7 @@ export default function DesktopPermissionsPage() { ) : ( <>
- {/* Permission card */} + {/* Steps */}
-
-
-
- - {allGranted ? "\u2713" : "1"} - -
-

Accessibility

-

- Lets SurfSense insert suggestions seamlessly, right where you're typing. -

+ {STEPS.map((step, index) => { + const status = permissions[step.field]; + const isGranted = status === "authorized"; + + return ( +
+
+
+ + {isGranted ? "\u2713" : index + 1} + +
+

{step.title}

+

{step.description}

+
+
+
-
- -
- {!allGranted && ( -
- - {permissions.accessibility === "denied" && ( -

- Toggle SurfSense on in System Settings to continue. -

+ {!isGranted && ( +
+ + {status === "denied" && ( +

+ Toggle SurfSense on in System Settings to continue. +

+ )} +

+ If SurfSense doesn't appear in the list, click + and select it from Applications. +

+
)} -

- If SurfSense doesn't appear in the list, click + and select it from Applications. -

- )} -
+ ); + })}
{/* Footer */} @@ -168,7 +198,7 @@ export default function DesktopPermissionsPage() { ) : ( <> +
); diff --git a/surfsense_web/app/desktop/suggestion/suggestion.css b/surfsense_web/app/desktop/suggestion/suggestion.css index 0d3332103..712d12618 100644 --- a/surfsense_web/app/desktop/suggestion/suggestion.css +++ b/surfsense_web/app/desktop/suggestion/suggestion.css @@ -36,32 +36,44 @@ html, body { white-space: pre-wrap; } -.suggestion-hint { - color: #666; - font-size: 11px; +.suggestion-actions { display: flex; - align-items: center; - gap: 6px; + justify-content: flex-end; + gap: 4px; border-top: 1px solid #2a2a2a; padding-top: 6px; } -.suggestion-hint kbd { - background: #2a2a2a; - border: 1px solid #3c3c3c; +.suggestion-btn { + padding: 2px 8px; border-radius: 3px; - padding: 0 4px; + border: 1px solid #3c3c3c; font-family: inherit; font-size: 10px; - font-weight: 600; - color: #999; - line-height: 18px; + font-weight: 500; + cursor: pointer; + line-height: 16px; + transition: background 0.15s, border-color 0.15s; } -.suggestion-separator { - width: 1px; - height: 10px; +.suggestion-btn-accept { + background: #2563eb; + border-color: #3b82f6; + color: #fff; +} + +.suggestion-btn-accept:hover { + background: #1d4ed8; +} + +.suggestion-btn-dismiss { + background: #2a2a2a; + color: #999; +} + +.suggestion-btn-dismiss:hover { background: #333; + color: #ccc; } .suggestion-error { diff --git a/surfsense_web/types/window.d.ts b/surfsense_web/types/window.d.ts index a5b8566f9..dc3a6465e 100644 --- a/surfsense_web/types/window.d.ts +++ b/surfsense_web/types/window.d.ts @@ -17,8 +17,10 @@ interface ElectronAPI { // Permissions getPermissionsStatus: () => Promise<{ accessibility: 'authorized' | 'denied' | 'not determined' | 'restricted' | 'limited'; + screenRecording: 'authorized' | 'denied' | 'not determined' | 'restricted' | 'limited'; }>; requestAccessibility: () => Promise; + requestScreenRecording: () => Promise; restartApp: () => Promise; // Autocomplete onAutocompleteContext: (callback: (data: { screenshot: string; searchSpaceId?: string }) => void) => () => void; From 080acf5e0a18b62953cffcf19806633cc9122c98 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 3 Apr 2026 20:16:23 +0200 Subject: [PATCH 17/21] remove dead code: allPermissionsGranted, simulateCopy, pendingSuggestionText, updateSuggestionText --- surfsense_desktop/src/ipc/channels.ts | 1 - surfsense_desktop/src/modules/autocomplete/index.ts | 7 ------- surfsense_desktop/src/modules/permissions.ts | 5 ----- surfsense_desktop/src/modules/platform.ts | 8 -------- surfsense_desktop/src/preload.ts | 1 - surfsense_web/app/desktop/suggestion/page.tsx | 6 +----- surfsense_web/types/window.d.ts | 1 - 7 files changed, 1 insertion(+), 28 deletions(-) diff --git a/surfsense_desktop/src/ipc/channels.ts b/surfsense_desktop/src/ipc/channels.ts index e41355eaf..d0e2b9ab4 100644 --- a/surfsense_desktop/src/ipc/channels.ts +++ b/surfsense_desktop/src/ipc/channels.ts @@ -15,7 +15,6 @@ export const IPC_CHANNELS = { AUTOCOMPLETE_CONTEXT: 'autocomplete-context', ACCEPT_SUGGESTION: 'accept-suggestion', DISMISS_SUGGESTION: 'dismiss-suggestion', - UPDATE_SUGGESTION_TEXT: 'update-suggestion-text', SET_AUTOCOMPLETE_ENABLED: 'set-autocomplete-enabled', GET_AUTOCOMPLETE_ENABLED: 'get-autocomplete-enabled', } as const; diff --git a/surfsense_desktop/src/modules/autocomplete/index.ts b/surfsense_desktop/src/modules/autocomplete/index.ts index 958886b63..3ed9c4a00 100644 --- a/surfsense_desktop/src/modules/autocomplete/index.ts +++ b/surfsense_desktop/src/modules/autocomplete/index.ts @@ -11,7 +11,6 @@ const SHORTCUT = 'CommandOrControl+Shift+Space'; let autocompleteEnabled = true; let savedClipboard = ''; let sourceApp = ''; -let pendingSuggestionText = ''; function isSurfSenseWindow(): boolean { const app = getFrontmostApp(); @@ -72,7 +71,6 @@ async function acceptAndInject(text: string): Promise { clipboard.writeText(text); destroySuggestion(); - pendingSuggestionText = ''; try { await new Promise((r) => setTimeout(r, 50)); @@ -90,10 +88,6 @@ function registerIpcHandlers(): void { }); ipcMain.handle(IPC_CHANNELS.DISMISS_SUGGESTION, () => { destroySuggestion(); - pendingSuggestionText = ''; - }); - ipcMain.handle(IPC_CHANNELS.UPDATE_SUGGESTION_TEXT, (_event, text: string) => { - pendingSuggestionText = text; }); ipcMain.handle(IPC_CHANNELS.SET_AUTOCOMPLETE_ENABLED, (_event, enabled: boolean) => { autocompleteEnabled = enabled; @@ -111,7 +105,6 @@ export function registerAutocomplete(): void { const sw = getSuggestionWindow(); if (sw && !sw.isDestroyed()) { destroySuggestion(); - pendingSuggestionText = ''; return; } triggerAutocomplete(); diff --git a/surfsense_desktop/src/modules/permissions.ts b/surfsense_desktop/src/modules/permissions.ts index a2f057795..02786113e 100644 --- a/surfsense_desktop/src/modules/permissions.ts +++ b/surfsense_desktop/src/modules/permissions.ts @@ -27,11 +27,6 @@ export function getPermissionsStatus(): PermissionsStatus { }; } -export function allPermissionsGranted(): boolean { - const status = getPermissionsStatus(); - return status.accessibility === 'authorized' && status.screenRecording === 'authorized'; -} - export function requestAccessibility(): void { if (!isMac()) return; const perms = getNodeMacPermissions(); diff --git a/surfsense_desktop/src/modules/platform.ts b/surfsense_desktop/src/modules/platform.ts index 1ab0c38fb..1e6ac74e4 100644 --- a/surfsense_desktop/src/modules/platform.ts +++ b/surfsense_desktop/src/modules/platform.ts @@ -19,14 +19,6 @@ export function getFrontmostApp(): string { return ''; } -export function simulateCopy(): void { - if (process.platform === 'darwin') { - execSync('osascript -e \'tell application "System Events" to keystroke "c" using command down\''); - } else if (process.platform === 'win32') { - execSync('powershell -command "Add-Type -AssemblyName System.Windows.Forms; [System.Windows.Forms.SendKeys]::SendWait(\'^c\')"'); - } -} - export function simulatePaste(): void { if (process.platform === 'darwin') { execSync('osascript -e \'tell application "System Events" to keystroke "v" using command down\''); diff --git a/surfsense_desktop/src/preload.ts b/surfsense_desktop/src/preload.ts index 5c8b64f6f..31c5ca865 100644 --- a/surfsense_desktop/src/preload.ts +++ b/surfsense_desktop/src/preload.ts @@ -36,7 +36,6 @@ contextBridge.exposeInMainWorld('electronAPI', { }, acceptSuggestion: (text: string) => ipcRenderer.invoke(IPC_CHANNELS.ACCEPT_SUGGESTION, text), dismissSuggestion: () => ipcRenderer.invoke(IPC_CHANNELS.DISMISS_SUGGESTION), - updateSuggestionText: (text: string) => ipcRenderer.invoke(IPC_CHANNELS.UPDATE_SUGGESTION_TEXT, text), setAutocompleteEnabled: (enabled: boolean) => ipcRenderer.invoke(IPC_CHANNELS.SET_AUTOCOMPLETE_ENABLED, enabled), getAutocompleteEnabled: () => ipcRenderer.invoke(IPC_CHANNELS.GET_AUTOCOMPLETE_ENABLED), }); diff --git a/surfsense_web/app/desktop/suggestion/page.tsx b/surfsense_web/app/desktop/suggestion/page.tsx index 4de90e03c..b68fe450d 100644 --- a/surfsense_web/app/desktop/suggestion/page.tsx +++ b/surfsense_web/app/desktop/suggestion/page.tsx @@ -116,11 +116,7 @@ export default function SuggestionPage() { try { const parsed: SSEEvent = JSON.parse(data); if (parsed.type === "text-delta") { - setSuggestion((prev) => { - const updated = prev + parsed.delta; - window.electronAPI?.updateSuggestionText?.(updated); - return updated; - }); + setSuggestion((prev) => prev + parsed.delta); } else if (parsed.type === "error") { setError(friendlyError(parsed.errorText)); } diff --git a/surfsense_web/types/window.d.ts b/surfsense_web/types/window.d.ts index dc3a6465e..2fc550306 100644 --- a/surfsense_web/types/window.d.ts +++ b/surfsense_web/types/window.d.ts @@ -26,7 +26,6 @@ interface ElectronAPI { onAutocompleteContext: (callback: (data: { screenshot: string; searchSpaceId?: string }) => void) => () => void; acceptSuggestion: (text: string) => Promise; dismissSuggestion: () => Promise; - updateSuggestionText: (text: string) => Promise; setAutocompleteEnabled: (enabled: boolean) => Promise; getAutocompleteEnabled: () => Promise; } From 960b8fc01273795407398fe8b108ebbd6009ef58 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 3 Apr 2026 21:34:01 +0200 Subject: [PATCH 18/21] add app context and KB grounding to autocomplete suggestions --- .../app/routes/autocomplete_routes.py | 7 +- .../services/vision_autocomplete_service.py | 117 +++++++++++++++++- .../src/modules/autocomplete/index.ts | 5 +- surfsense_desktop/src/modules/platform.ts | 18 +++ surfsense_desktop/src/preload.ts | 4 +- surfsense_web/app/desktop/suggestion/page.tsx | 6 +- surfsense_web/types/window.d.ts | 2 +- 7 files changed, 150 insertions(+), 9 deletions(-) diff --git a/surfsense_backend/app/routes/autocomplete_routes.py b/surfsense_backend/app/routes/autocomplete_routes.py index 329476ca1..a6f66f408 100644 --- a/surfsense_backend/app/routes/autocomplete_routes.py +++ b/surfsense_backend/app/routes/autocomplete_routes.py @@ -14,6 +14,8 @@ router = APIRouter(prefix="/autocomplete", tags=["autocomplete"]) class VisionAutocompleteRequest(BaseModel): screenshot: str search_space_id: int + app_name: str = "" + window_title: str = "" @router.post("/vision/stream") @@ -23,7 +25,10 @@ async def vision_autocomplete_stream( session: AsyncSession = Depends(get_async_session), ): return StreamingResponse( - stream_vision_autocomplete(body.screenshot, body.search_space_id, session), + stream_vision_autocomplete( + body.screenshot, body.search_space_id, session, + app_name=body.app_name, window_title=body.window_title, + ), media_type="text/event-stream", headers={ **VercelStreamingService.get_response_headers(), diff --git a/surfsense_backend/app/services/vision_autocomplete_service.py b/surfsense_backend/app/services/vision_autocomplete_service.py index 526b0d35c..0804df7fb 100644 --- a/surfsense_backend/app/services/vision_autocomplete_service.py +++ b/surfsense_backend/app/services/vision_autocomplete_service.py @@ -4,11 +4,21 @@ from typing import AsyncGenerator from langchain_core.messages import HumanMessage, SystemMessage from sqlalchemy.ext.asyncio import AsyncSession +from app.retriever.chunks_hybrid_search import ChucksHybridSearchRetriever from app.services.llm_service import get_vision_llm from app.services.new_streaming_service import VercelStreamingService logger = logging.getLogger(__name__) +KB_TOP_K = 5 +KB_MAX_CHARS = 4000 + +EXTRACT_QUERY_PROMPT = """Look at this screenshot and describe in 1-2 short sentences what the user is working on and what topic they need to write about. Be specific about the subject matter. Output ONLY the description, nothing else.""" + +EXTRACT_QUERY_PROMPT_WITH_APP = """The user is currently in the application "{app_name}" with the window titled "{window_title}". + +Look at this screenshot and describe in 1-2 short sentences what the user is working on and what topic they need to write about. Be specific about the subject matter. Output ONLY the description, nothing else.""" + VISION_SYSTEM_PROMPT = """You are a smart writing assistant that analyzes the user's screen to draft or complete text. You will receive a screenshot of the user's screen. Your job: @@ -28,13 +38,107 @@ Rules: - Do NOT describe the screenshot or explain your reasoning. - If you cannot determine what to write, output nothing.""" +APP_CONTEXT_BLOCK = """ + +The user is currently working in "{app_name}" (window: "{window_title}"). Use this to understand the type of application and adapt your tone and format accordingly.""" + +KB_CONTEXT_BLOCK = """ + +You also have access to the user's knowledge base documents below. Use them to write more accurate, informed, and contextually relevant text. Do NOT cite or reference the documents explicitly — just let the knowledge inform your writing naturally. + + +{kb_context} +""" + + +def _build_system_prompt(app_name: str, window_title: str, kb_context: str) -> str: + """Assemble the system prompt from optional context blocks.""" + prompt = VISION_SYSTEM_PROMPT + if app_name: + prompt += APP_CONTEXT_BLOCK.format(app_name=app_name, window_title=window_title) + if kb_context: + prompt += KB_CONTEXT_BLOCK.format(kb_context=kb_context) + return prompt + + +async def _extract_query_from_screenshot( + llm, screenshot_data_url: str, + app_name: str = "", window_title: str = "", +) -> str | None: + """Ask the Vision LLM to describe what the user is working on.""" + if app_name: + prompt_text = EXTRACT_QUERY_PROMPT_WITH_APP.format( + app_name=app_name, window_title=window_title, + ) + else: + prompt_text = EXTRACT_QUERY_PROMPT + + try: + response = await llm.ainvoke([ + HumanMessage(content=[ + {"type": "text", "text": prompt_text}, + {"type": "image_url", "image_url": {"url": screenshot_data_url}}, + ]), + ]) + query = response.content.strip() if hasattr(response, "content") else "" + return query if query else None + except Exception as e: + logger.warning(f"Failed to extract query from screenshot: {e}") + return None + + +async def _search_knowledge_base( + session: AsyncSession, search_space_id: int, query: str +) -> str: + """Search the KB and return formatted context string.""" + try: + retriever = ChucksHybridSearchRetriever(session) + results = await retriever.hybrid_search( + query_text=query, + top_k=KB_TOP_K, + search_space_id=search_space_id, + ) + + if not results: + return "" + + parts: list[str] = [] + char_count = 0 + for doc in results: + title = doc.get("document", {}).get("title", "Untitled") + for chunk in doc.get("chunks", []): + content = chunk.get("content", "").strip() + if not content: + continue + entry = f"[{title}]\n{content}" + if char_count + len(entry) > KB_MAX_CHARS: + break + parts.append(entry) + char_count += len(entry) + if char_count >= KB_MAX_CHARS: + break + + return "\n\n---\n\n".join(parts) + except Exception as e: + logger.warning(f"KB search failed, proceeding without context: {e}") + return "" + async def stream_vision_autocomplete( screenshot_data_url: str, search_space_id: int, session: AsyncSession, + *, + app_name: str = "", + window_title: str = "", ) -> AsyncGenerator[str, None]: - """Analyze a screenshot with the vision LLM and stream a text completion.""" + """Analyze a screenshot with the vision LLM and stream a text completion. + + Pipeline: + 1. Extract a search query from the screenshot (non-streaming) + 2. Search the knowledge base for relevant context + 3. Stream the final completion with screenshot + KB + app context + """ streaming = VercelStreamingService() llm = await get_vision_llm(session, search_space_id) @@ -44,8 +148,17 @@ async def stream_vision_autocomplete( yield streaming.format_done() return + kb_context = "" + query = await _extract_query_from_screenshot( + llm, screenshot_data_url, app_name=app_name, window_title=window_title, + ) + if query: + kb_context = await _search_knowledge_base(session, search_space_id, query) + + system_prompt = _build_system_prompt(app_name, window_title, kb_context) + messages = [ - SystemMessage(content=VISION_SYSTEM_PROMPT), + SystemMessage(content=system_prompt), HumanMessage(content=[ { "type": "text", diff --git a/surfsense_desktop/src/modules/autocomplete/index.ts b/surfsense_desktop/src/modules/autocomplete/index.ts index 3ed9c4a00..0d5073de4 100644 --- a/surfsense_desktop/src/modules/autocomplete/index.ts +++ b/surfsense_desktop/src/modules/autocomplete/index.ts @@ -1,6 +1,6 @@ import { clipboard, globalShortcut, ipcMain, screen } from 'electron'; import { IPC_CHANNELS } from '../../ipc/channels'; -import { getFrontmostApp, hasAccessibilityPermission, simulatePaste } from '../platform'; +import { getFrontmostApp, getWindowTitle, hasAccessibilityPermission, simulatePaste } from '../platform'; import { hasScreenRecordingPermission, requestAccessibility, requestScreenRecording } from '../permissions'; import { getMainWindow } from '../window'; import { captureScreen } from './screenshot'; @@ -27,6 +27,7 @@ async function triggerAutocomplete(): Promise { } sourceApp = getFrontmostApp(); + const windowTitle = getWindowTitle(); savedClipboard = clipboard.readText(); const screenshot = await captureScreen(); @@ -55,6 +56,8 @@ async function triggerAutocomplete(): Promise { sw.webContents.send(IPC_CHANNELS.AUTOCOMPLETE_CONTEXT, { screenshot, searchSpaceId, + appName: sourceApp, + windowTitle, }); } }, 300); diff --git a/surfsense_desktop/src/modules/platform.ts b/surfsense_desktop/src/modules/platform.ts index 1e6ac74e4..122e2efed 100644 --- a/surfsense_desktop/src/modules/platform.ts +++ b/surfsense_desktop/src/modules/platform.ts @@ -32,6 +32,24 @@ export function checkAccessibilityPermission(): boolean { return systemPreferences.isTrustedAccessibilityClient(true); } +export function getWindowTitle(): string { + try { + if (process.platform === 'darwin') { + return execSync( + 'osascript -e \'tell application "System Events" to get title of front window of first application process whose frontmost is true\'' + ).toString().trim(); + } + if (process.platform === 'win32') { + return execSync( + 'powershell -command "(Get-Process | Where-Object { $_.MainWindowHandle -eq (Add-Type -MemberDefinition \'[DllImport(\\\"user32.dll\\\")] public static extern IntPtr GetForegroundWindow();\' -Name W -PassThru)::GetForegroundWindow() }).MainWindowTitle"' + ).toString().trim(); + } + } catch { + return ''; + } + return ''; +} + export function hasAccessibilityPermission(): boolean { if (process.platform !== 'darwin') return true; return systemPreferences.isTrustedAccessibilityClient(false); diff --git a/surfsense_desktop/src/preload.ts b/surfsense_desktop/src/preload.ts index 31c5ca865..2bd09f13c 100644 --- a/surfsense_desktop/src/preload.ts +++ b/surfsense_desktop/src/preload.ts @@ -27,8 +27,8 @@ contextBridge.exposeInMainWorld('electronAPI', { requestScreenRecording: () => ipcRenderer.invoke(IPC_CHANNELS.REQUEST_SCREEN_RECORDING), restartApp: () => ipcRenderer.invoke(IPC_CHANNELS.RESTART_APP), // Autocomplete - onAutocompleteContext: (callback: (data: { screenshot: string; searchSpaceId?: string }) => void) => { - const listener = (_event: unknown, data: { screenshot: string; searchSpaceId?: string }) => callback(data); + onAutocompleteContext: (callback: (data: { screenshot: string; searchSpaceId?: string; appName?: string; windowTitle?: string }) => void) => { + const listener = (_event: unknown, data: { screenshot: string; searchSpaceId?: string; appName?: string; windowTitle?: string }) => callback(data); ipcRenderer.on(IPC_CHANNELS.AUTOCOMPLETE_CONTEXT, listener); return () => { ipcRenderer.removeListener(IPC_CHANNELS.AUTOCOMPLETE_CONTEXT, listener); diff --git a/surfsense_web/app/desktop/suggestion/page.tsx b/surfsense_web/app/desktop/suggestion/page.tsx index b68fe450d..b7d9b97bd 100644 --- a/surfsense_web/app/desktop/suggestion/page.tsx +++ b/surfsense_web/app/desktop/suggestion/page.tsx @@ -46,7 +46,7 @@ export default function SuggestionPage() { }, [error]); const fetchSuggestion = useCallback( - async (screenshot: string, searchSpaceId: string) => { + async (screenshot: string, searchSpaceId: string, appName?: string, windowTitle?: string) => { abortRef.current?.abort(); const controller = new AbortController(); abortRef.current = controller; @@ -77,6 +77,8 @@ export default function SuggestionPage() { body: JSON.stringify({ screenshot, search_space_id: parseInt(searchSpaceId, 10), + app_name: appName || "", + window_title: windowTitle || "", }), signal: controller.signal, }, @@ -142,7 +144,7 @@ export default function SuggestionPage() { const cleanup = window.electronAPI.onAutocompleteContext((data) => { const searchSpaceId = data.searchSpaceId || "1"; if (data.screenshot) { - fetchSuggestion(data.screenshot, searchSpaceId); + fetchSuggestion(data.screenshot, searchSpaceId, data.appName, data.windowTitle); } }); diff --git a/surfsense_web/types/window.d.ts b/surfsense_web/types/window.d.ts index 2fc550306..85b6bdf51 100644 --- a/surfsense_web/types/window.d.ts +++ b/surfsense_web/types/window.d.ts @@ -23,7 +23,7 @@ interface ElectronAPI { requestScreenRecording: () => Promise; restartApp: () => Promise; // Autocomplete - onAutocompleteContext: (callback: (data: { screenshot: string; searchSpaceId?: string }) => void) => () => void; + onAutocompleteContext: (callback: (data: { screenshot: string; searchSpaceId?: string; appName?: string; windowTitle?: string }) => void) => () => void; acceptSuggestion: (text: string) => Promise; dismissSuggestion: () => Promise; setAutocompleteEnabled: (enabled: boolean) => Promise; From 46e8134b23ec5bb56dce475811437a283bcaec47 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sat, 4 Apr 2026 08:40:55 +0200 Subject: [PATCH 19/21] catch non-vision model errors with actionable user message --- .../app/services/vision_autocomplete_service.py | 12 ++++++++++-- surfsense_web/app/desktop/suggestion/page.tsx | 2 ++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/surfsense_backend/app/services/vision_autocomplete_service.py b/surfsense_backend/app/services/vision_autocomplete_service.py index 0804df7fb..e172c6522 100644 --- a/surfsense_backend/app/services/vision_autocomplete_service.py +++ b/surfsense_backend/app/services/vision_autocomplete_service.py @@ -186,6 +186,14 @@ async def stream_vision_autocomplete( yield streaming.format_done() except Exception as e: - logger.error(f"Vision autocomplete streaming error: {e}") - yield streaming.format_error(str(e)) + error_str = str(e).lower() + if "content must be a string" in error_str or "does not support image" in error_str: + logger.warning(f"Vision autocomplete: selected model does not support vision: {e}") + yield streaming.format_error( + "The selected model does not support vision. " + "Please set a vision-capable model (e.g. GPT-4o, Gemini) in your search space settings." + ) + else: + logger.error(f"Vision autocomplete streaming error: {e}") + yield streaming.format_error(str(e)) yield streaming.format_done() diff --git a/surfsense_web/app/desktop/suggestion/page.tsx b/surfsense_web/app/desktop/suggestion/page.tsx index b7d9b97bd..7188b73c6 100644 --- a/surfsense_web/app/desktop/suggestion/page.tsx +++ b/surfsense_web/app/desktop/suggestion/page.tsx @@ -24,6 +24,8 @@ function friendlyError(raw: string | number): string { return "Please sign in to use suggestions."; if (lower.includes("no vision llm configured") || lower.includes("no llm configured")) return "No Vision LLM configured. Set one in search space settings."; + if (lower.includes("does not support vision")) + return "Selected model doesn\u2019t support vision. Set a vision-capable model in settings."; if (lower.includes("fetch") || lower.includes("network") || lower.includes("econnrefused")) return "Can\u2019t reach the server. Check your connection."; return "Something went wrong. Try again."; From 18103417bb10751c1b55b745bcd711596191dede Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sat, 4 Apr 2026 08:57:09 +0200 Subject: [PATCH 20/21] fix: harden autocomplete endpoint security and error handling --- .../app/routes/autocomplete_routes.py | 11 +++-- .../services/vision_autocomplete_service.py | 46 +++++++++++++++---- surfsense_web/app/desktop/suggestion/page.tsx | 18 ++++++++ .../app/desktop/suggestion/suggestion.css | 3 +- 4 files changed, 64 insertions(+), 14 deletions(-) diff --git a/surfsense_backend/app/routes/autocomplete_routes.py b/surfsense_backend/app/routes/autocomplete_routes.py index a6f66f408..bb56709cb 100644 --- a/surfsense_backend/app/routes/autocomplete_routes.py +++ b/surfsense_backend/app/routes/autocomplete_routes.py @@ -1,18 +1,21 @@ -from fastapi import APIRouter, Depends +from fastapi import APIRouter, Depends, HTTPException from fastapi.responses import StreamingResponse -from pydantic import BaseModel +from pydantic import BaseModel, Field from sqlalchemy.ext.asyncio import AsyncSession from app.db import User, get_async_session from app.services.new_streaming_service import VercelStreamingService from app.services.vision_autocomplete_service import stream_vision_autocomplete from app.users import current_active_user +from app.utils.rbac import check_search_space_access router = APIRouter(prefix="/autocomplete", tags=["autocomplete"]) +MAX_SCREENSHOT_SIZE = 20 * 1024 * 1024 # 20 MB base64 ceiling + class VisionAutocompleteRequest(BaseModel): - screenshot: str + screenshot: str = Field(..., max_length=MAX_SCREENSHOT_SIZE) search_space_id: int app_name: str = "" window_title: str = "" @@ -24,6 +27,8 @@ async def vision_autocomplete_stream( user: User = Depends(current_active_user), session: AsyncSession = Depends(get_async_session), ): + await check_search_space_access(session, user, body.search_space_id) + return StreamingResponse( stream_vision_autocomplete( body.screenshot, body.search_space_id, session, diff --git a/surfsense_backend/app/services/vision_autocomplete_service.py b/surfsense_backend/app/services/vision_autocomplete_service.py index e172c6522..2ddb56f51 100644 --- a/surfsense_backend/app/services/vision_autocomplete_service.py +++ b/surfsense_backend/app/services/vision_autocomplete_service.py @@ -61,11 +61,21 @@ def _build_system_prompt(app_name: str, window_title: str, kb_context: str) -> s return prompt +def _is_vision_unsupported_error(e: Exception) -> bool: + """Check if an exception indicates the model doesn't support vision/images.""" + msg = str(e).lower() + return "content must be a string" in msg or "does not support image" in msg + + async def _extract_query_from_screenshot( llm, screenshot_data_url: str, app_name: str = "", window_title: str = "", ) -> str | None: - """Ask the Vision LLM to describe what the user is working on.""" + """Ask the Vision LLM to describe what the user is working on. + + Raises vision-unsupported errors so the caller can return a + friendly message immediately instead of retrying with astream. + """ if app_name: prompt_text = EXTRACT_QUERY_PROMPT_WITH_APP.format( app_name=app_name, window_title=window_title, @@ -83,6 +93,8 @@ async def _extract_query_from_screenshot( query = response.content.strip() if hasattr(response, "content") else "" return query if query else None except Exception as e: + if _is_vision_unsupported_error(e): + raise logger.warning(f"Failed to extract query from screenshot: {e}") return None @@ -140,6 +152,10 @@ async def stream_vision_autocomplete( 3. Stream the final completion with screenshot + KB + app context """ streaming = VercelStreamingService() + vision_error_msg = ( + "The selected model does not support vision. " + "Please set a vision-capable model (e.g. GPT-4o, Gemini) in your search space settings." + ) llm = await get_vision_llm(session, search_space_id) if not llm: @@ -149,9 +165,17 @@ async def stream_vision_autocomplete( return kb_context = "" - query = await _extract_query_from_screenshot( - llm, screenshot_data_url, app_name=app_name, window_title=window_title, - ) + try: + query = await _extract_query_from_screenshot( + llm, screenshot_data_url, app_name=app_name, window_title=window_title, + ) + except Exception as e: + logger.warning(f"Vision autocomplete: selected model does not support vision: {e}") + yield streaming.format_message_start() + yield streaming.format_error(vision_error_msg) + yield streaming.format_done() + return + if query: kb_context = await _search_knowledge_base(session, search_space_id, query) @@ -171,10 +195,13 @@ async def stream_vision_autocomplete( ]), ] + text_started = False + text_id = "" try: yield streaming.format_message_start() text_id = streaming.generate_text_id() yield streaming.format_text_start(text_id) + text_started = True async for chunk in llm.astream(messages): token = chunk.content if hasattr(chunk, "content") else str(chunk) @@ -186,13 +213,12 @@ async def stream_vision_autocomplete( yield streaming.format_done() except Exception as e: - error_str = str(e).lower() - if "content must be a string" in error_str or "does not support image" in error_str: + if text_started: + yield streaming.format_text_end(text_id) + + if _is_vision_unsupported_error(e): logger.warning(f"Vision autocomplete: selected model does not support vision: {e}") - yield streaming.format_error( - "The selected model does not support vision. " - "Please set a vision-capable model (e.g. GPT-4o, Gemini) in your search space settings." - ) + yield streaming.format_error(vision_error_msg) else: logger.error(f"Vision autocomplete streaming error: {e}") yield streaming.format_error(str(e)) diff --git a/surfsense_web/app/desktop/suggestion/page.tsx b/surfsense_web/app/desktop/suggestion/page.tsx index 7188b73c6..03944867f 100644 --- a/surfsense_web/app/desktop/suggestion/page.tsx +++ b/surfsense_web/app/desktop/suggestion/page.tsx @@ -36,9 +36,17 @@ const AUTO_DISMISS_MS = 3000; export default function SuggestionPage() { const [suggestion, setSuggestion] = useState(""); const [isLoading, setIsLoading] = useState(true); + const [isDesktop, setIsDesktop] = useState(true); const [error, setError] = useState(null); const abortRef = useRef(null); + useEffect(() => { + if (!window.electronAPI?.onAutocompleteContext) { + setIsDesktop(false); + setIsLoading(false); + } + }, []); + useEffect(() => { if (!error) return; const timer = setTimeout(() => { @@ -153,6 +161,16 @@ export default function SuggestionPage() { return cleanup; }, [fetchSuggestion]); + if (!isDesktop) { + return ( +
+ + This page is only available in the SurfSense desktop app. + +
+ ); + } + if (error) { return (
diff --git a/surfsense_web/app/desktop/suggestion/suggestion.css b/surfsense_web/app/desktop/suggestion/suggestion.css index 712d12618..62f4d2ea7 100644 --- a/surfsense_web/app/desktop/suggestion/suggestion.css +++ b/surfsense_web/app/desktop/suggestion/suggestion.css @@ -1,4 +1,5 @@ -html, body { +html:has(.suggestion-body), +body:has(.suggestion-body) { margin: 0 !important; padding: 0 !important; background: transparent !important; From a180bf5576dc424646ea55a81e37b16c1c84743c Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sat, 4 Apr 2026 09:39:20 +0200 Subject: [PATCH 21/21] fix merge conflicts --- ...=> 119_add_vision_llm_id_to_search_spaces.py} | 10 +++++----- .../app/services/vision_autocomplete_service.py | 4 ++-- .../src/modules/autocomplete/index.ts | 16 +++++++++++----- .../components/editor-panel/editor-panel.tsx | 3 --- .../layout/ui/tabs/DocumentTabContent.tsx | 3 --- 5 files changed, 18 insertions(+), 18 deletions(-) rename surfsense_backend/alembic/versions/{117_add_vision_llm_id_to_search_spaces.py => 119_add_vision_llm_id_to_search_spaces.py} (86%) diff --git a/surfsense_backend/alembic/versions/117_add_vision_llm_id_to_search_spaces.py b/surfsense_backend/alembic/versions/119_add_vision_llm_id_to_search_spaces.py similarity index 86% rename from surfsense_backend/alembic/versions/117_add_vision_llm_id_to_search_spaces.py rename to surfsense_backend/alembic/versions/119_add_vision_llm_id_to_search_spaces.py index 2bec374c6..8e41d5e67 100644 --- a/surfsense_backend/alembic/versions/117_add_vision_llm_id_to_search_spaces.py +++ b/surfsense_backend/alembic/versions/119_add_vision_llm_id_to_search_spaces.py @@ -1,7 +1,7 @@ -"""117_add_vision_llm_id_to_search_spaces +"""119_add_vision_llm_id_to_search_spaces -Revision ID: 117 -Revises: 116 +Revision ID: 119 +Revises: 118 Adds vision_llm_id column to search_spaces for vision/screenshot analysis LLM role assignment. Defaults to 0 (Auto mode), same convention as @@ -16,8 +16,8 @@ import sqlalchemy as sa from alembic import op -revision: str = "117" -down_revision: str | None = "116" +revision: str = "119" +down_revision: str | None = "118" branch_labels: str | Sequence[str] | None = None depends_on: str | Sequence[str] | None = None diff --git a/surfsense_backend/app/services/vision_autocomplete_service.py b/surfsense_backend/app/services/vision_autocomplete_service.py index 2ddb56f51..f24a5c848 100644 --- a/surfsense_backend/app/services/vision_autocomplete_service.py +++ b/surfsense_backend/app/services/vision_autocomplete_service.py @@ -220,6 +220,6 @@ async def stream_vision_autocomplete( logger.warning(f"Vision autocomplete: selected model does not support vision: {e}") yield streaming.format_error(vision_error_msg) else: - logger.error(f"Vision autocomplete streaming error: {e}") - yield streaming.format_error(str(e)) + logger.error(f"Vision autocomplete streaming error: {e}", exc_info=True) + yield streaming.format_error("Autocomplete failed. Please try again.") yield streaming.format_done() diff --git a/surfsense_desktop/src/modules/autocomplete/index.ts b/surfsense_desktop/src/modules/autocomplete/index.ts index 0d5073de4..01a4cf913 100644 --- a/surfsense_desktop/src/modules/autocomplete/index.ts +++ b/surfsense_desktop/src/modules/autocomplete/index.ts @@ -11,6 +11,7 @@ const SHORTCUT = 'CommandOrControl+Shift+Space'; let autocompleteEnabled = true; let savedClipboard = ''; let sourceApp = ''; +let lastSearchSpaceId: string | null = null; function isSurfSenseWindow(): boolean { const app = getFrontmostApp(); @@ -36,19 +37,24 @@ async function triggerAutocomplete(): Promise { return; } - const cursor = screen.getCursorScreenPoint(); - const win = createSuggestionWindow(cursor.x, cursor.y); - - let searchSpaceId = '1'; const mainWin = getMainWindow(); if (mainWin && !mainWin.isDestroyed()) { const mainUrl = mainWin.webContents.getURL(); const match = mainUrl.match(/\/dashboard\/(\d+)/); if (match) { - searchSpaceId = match[1]; + lastSearchSpaceId = match[1]; } } + if (!lastSearchSpaceId) { + console.warn('[autocomplete] No active search space. Open a search space first.'); + return; + } + + const searchSpaceId = lastSearchSpaceId; + const cursor = screen.getCursorScreenPoint(); + const win = createSuggestionWindow(cursor.x, cursor.y); + win.webContents.once('did-finish-load', () => { const sw = getSuggestionWindow(); setTimeout(() => { diff --git a/surfsense_web/components/editor-panel/editor-panel.tsx b/surfsense_web/components/editor-panel/editor-panel.tsx index 248fe68eb..c307b3cea 100644 --- a/surfsense_web/components/editor-panel/editor-panel.tsx +++ b/surfsense_web/components/editor-panel/editor-panel.tsx @@ -96,9 +96,6 @@ export function EditorPanelContent({ } try { - const response = await authenticatedFetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-spaces/${searchSpaceId}/documents/${documentId}/editor-content`, - { method: "GET", signal: controller.signal } const url = new URL( `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-spaces/${searchSpaceId}/documents/${documentId}/editor-content` ); diff --git a/surfsense_web/components/layout/ui/tabs/DocumentTabContent.tsx b/surfsense_web/components/layout/ui/tabs/DocumentTabContent.tsx index d2ce3cc64..97c5b7cd9 100644 --- a/surfsense_web/components/layout/ui/tabs/DocumentTabContent.tsx +++ b/surfsense_web/components/layout/ui/tabs/DocumentTabContent.tsx @@ -81,9 +81,6 @@ export function DocumentTabContent({ documentId, searchSpaceId, title }: Documen } try { - const response = await authenticatedFetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-spaces/${searchSpaceId}/documents/${documentId}/editor-content`, - { method: "GET", signal: controller.signal } const url = new URL( `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-spaces/${searchSpaceId}/documents/${documentId}/editor-content` );