From f4d197f7022b0ceec560925f38fe6c125a77b0fc Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 2 Apr 2026 13:18:20 +0200 Subject: [PATCH 01/42] feat: add native module support for desktop autocomplete --- surfsense_desktop/.npmrc | 1 + surfsense_desktop/electron-builder.yml | 13 +++++ surfsense_desktop/package.json | 8 +++- surfsense_desktop/pnpm-lock.yaml | 50 ++++++++++++++++++++ surfsense_desktop/scripts/build-electron.mjs | 2 +- 5 files changed, 71 insertions(+), 3 deletions(-) create mode 100644 surfsense_desktop/.npmrc diff --git a/surfsense_desktop/.npmrc b/surfsense_desktop/.npmrc new file mode 100644 index 000000000..d67f37488 --- /dev/null +++ b/surfsense_desktop/.npmrc @@ -0,0 +1 @@ +node-linker=hoisted diff --git a/surfsense_desktop/electron-builder.yml b/surfsense_desktop/electron-builder.yml index eaca0f19b..74c69d223 100644 --- a/surfsense_desktop/electron-builder.yml +++ b/surfsense_desktop/electron-builder.yml @@ -9,6 +9,16 @@ directories: files: - dist/**/* - "!node_modules" + - node_modules/uiohook-napi/**/* + - "!node_modules/uiohook-napi/build" + - "!node_modules/uiohook-napi/src" + - "!node_modules/uiohook-napi/libuiohook" + - "!node_modules/uiohook-napi/binding.gyp" + - node_modules/node-gyp-build/**/* + - node_modules/node-mac-permissions/**/* + - "!node_modules/node-mac-permissions/build" + - "!node_modules/node-mac-permissions/src" + - "!node_modules/node-mac-permissions/binding.gyp" - "!src" - "!scripts" - "!release" @@ -29,6 +39,9 @@ extraResources: filter: ["**/*"] asarUnpack: - "**/*.node" + - "node_modules/uiohook-napi/**/*" + - "node_modules/node-gyp-build/**/*" + - "node_modules/node-mac-permissions/**/*" mac: icon: assets/icon.icns category: public.app-category.productivity diff --git a/surfsense_desktop/package.json b/surfsense_desktop/package.json index bd0cc67ab..a2e452b7c 100644 --- a/surfsense_desktop/package.json +++ b/surfsense_desktop/package.json @@ -11,12 +11,14 @@ "dist:mac": "pnpm build && electron-builder --mac --config electron-builder.yml", "dist:win": "pnpm build && electron-builder --win --config electron-builder.yml", "dist:linux": "pnpm build && electron-builder --linux --config electron-builder.yml", - "typecheck": "tsc --noEmit" + "typecheck": "tsc --noEmit", + "postinstall": "electron-rebuild" }, "author": "MODSetter", "license": "MIT", "packageManager": "pnpm@10.24.0", "devDependencies": { + "@electron/rebuild": "^4.0.3", "@types/node": "^25.5.0", "concurrently": "^9.2.1", "dotenv": "^17.3.1", @@ -28,6 +30,8 @@ }, "dependencies": { "electron-updater": "^6.8.3", - "get-port-please": "^3.2.0" + "get-port-please": "^3.2.0", + "node-mac-permissions": "^2.5.0", + "uiohook-napi": "^1.5.5" } } diff --git a/surfsense_desktop/pnpm-lock.yaml b/surfsense_desktop/pnpm-lock.yaml index ea65be0bb..82bad9456 100644 --- a/surfsense_desktop/pnpm-lock.yaml +++ b/surfsense_desktop/pnpm-lock.yaml @@ -14,7 +14,16 @@ importers: get-port-please: specifier: ^3.2.0 version: 3.2.0 + node-mac-permissions: + specifier: ^2.5.0 + version: 2.5.0 + uiohook-napi: + specifier: ^1.5.5 + version: 1.5.5 devDependencies: + '@electron/rebuild': + specifier: ^4.0.3 + version: 4.0.3 '@types/node': specifier: ^25.5.0 version: 25.5.0 @@ -343,6 +352,7 @@ packages: '@xmldom/xmldom@0.8.11': resolution: {integrity: sha512-cQzWCtO6C8TQiYl1ruKNn2U6Ao4o4WBBcbL61yJl84x+j5sOWWFU9X7DpND8XZG3daDppSsigMdfAIl2upQBRw==} engines: {node: '>=10.0.0'} + deprecated: this version has critical issues, please update to the latest version abbrev@3.0.1: resolution: {integrity: sha512-AO2ac6pjRB3SJmGJo+v5/aK6Omggp6fsLrs6wN9bd35ulu4cCwaAU9+7ZhXjeqHVkaHThLuzH0nZr0YpCDhygg==} @@ -424,6 +434,9 @@ packages: base64-js@1.5.1: resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} + bindings@1.5.0: + resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==} + bl@4.1.0: resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} @@ -738,6 +751,9 @@ packages: picomatch: optional: true + file-uri-to-path@1.0.0: + resolution: {integrity: sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==} + filelist@1.0.6: resolution: {integrity: sha512-5giy2PkLYY1cP39p17Ech+2xlpTRL9HLspOfEgm0L6CwBXBTgsK5ou0JtzYuepxkaQ/tvhCFIJ5uXo0OrM2DxA==} @@ -1103,14 +1119,25 @@ packages: node-addon-api@1.7.2: resolution: {integrity: sha512-ibPK3iA+vaY1eEjESkQkM0BbCqFOaZMiXRTtdB0u7b4djtY6JnsjvPdUHVMg6xQt3B8fpTTWHI9A+ADjM9frzg==} + node-addon-api@7.1.1: + resolution: {integrity: sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==} + node-api-version@0.2.1: resolution: {integrity: sha512-2xP/IGGMmmSQpI1+O/k72jF/ykvZ89JeuKX3TLJAYPDVLUalrshrLHkeVcCCZqG/eEa635cr8IBYzgnDvM2O8Q==} + node-gyp-build@4.8.4: + resolution: {integrity: sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==} + hasBin: true + node-gyp@11.5.0: resolution: {integrity: sha512-ra7Kvlhxn5V9Slyus0ygMa2h+UqExPqUIkfk7Pc8QTLT956JLSy51uWFwHtIYy0vI8cB4BDhc/S03+880My/LQ==} engines: {node: ^18.17.0 || >=20.5.0} hasBin: true + node-mac-permissions@2.5.0: + resolution: {integrity: sha512-zR8SVCaN3WqV1xwWd04XVAdzm3UTdjbxciLrZtB0Cc7F2Kd34AJfhPD4hm1HU0YH3oGUZO4X9OBLY5ijSTHsGw==} + os: [darwin] + nopt@8.1.0: resolution: {integrity: sha512-ieGu42u/Qsa4TFktmaKEwM6MQH0pOWnaB3htzh0JRtx84+Mebc0cbZYN5bC+6WTZ4+77xrL9Pn5m7CV6VIkV7A==} engines: {node: ^18.17.0 || >=20.5.0} @@ -1424,6 +1451,10 @@ packages: engines: {node: '>=14.17'} hasBin: true + uiohook-napi@1.5.5: + resolution: {integrity: sha512-oSlTdnECw2GBfsJPTbBQBeE4v/EXP0EZmX6BJq5nzH/JgFaBE8JpFwEA/kLhiEP7HxQw28FViWiYgdIZzWuuJQ==} + engines: {node: '>= 16'} + undici-types@7.16.0: resolution: {integrity: sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==} @@ -1923,6 +1954,10 @@ snapshots: base64-js@1.5.1: {} + bindings@1.5.0: + dependencies: + file-uri-to-path: 1.0.0 + bl@4.1.0: dependencies: buffer: 5.7.1 @@ -2348,6 +2383,8 @@ snapshots: optionalDependencies: picomatch: 4.0.3 + file-uri-to-path@1.0.0: {} + filelist@1.0.6: dependencies: minimatch: 5.1.9 @@ -2739,10 +2776,14 @@ snapshots: node-addon-api@1.7.2: optional: true + node-addon-api@7.1.1: {} + node-api-version@0.2.1: dependencies: semver: 7.7.4 + node-gyp-build@4.8.4: {} + node-gyp@11.5.0: dependencies: env-paths: 2.2.1 @@ -2758,6 +2799,11 @@ snapshots: transitivePeerDependencies: - supports-color + node-mac-permissions@2.5.0: + dependencies: + bindings: 1.5.0 + node-addon-api: 7.1.1 + nopt@8.1.0: dependencies: abbrev: 3.0.1 @@ -3064,6 +3110,10 @@ snapshots: typescript@5.9.3: {} + uiohook-napi@1.5.5: + dependencies: + node-gyp-build: 4.8.4 + undici-types@7.16.0: {} undici-types@7.18.2: {} diff --git a/surfsense_desktop/scripts/build-electron.mjs b/surfsense_desktop/scripts/build-electron.mjs index 923830296..83d941dd2 100644 --- a/surfsense_desktop/scripts/build-electron.mjs +++ b/surfsense_desktop/scripts/build-electron.mjs @@ -104,7 +104,7 @@ async function buildElectron() { bundle: true, platform: 'node', target: 'node18', - external: ['electron'], + external: ['electron', 'uiohook-napi', 'node-mac-permissions'], sourcemap: true, minify: false, define: { From fbd033d0a4ed756a789879b1bba69c08bc71506d Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 2 Apr 2026 13:19:21 +0200 Subject: [PATCH 02/42] feat: add autocomplete streaming endpoint with KB context --- surfsense_backend/app/routes/__init__.py | 2 + .../app/routes/autocomplete_routes.py | 136 ++++++++++++++++++ 2 files changed, 138 insertions(+) create mode 100644 surfsense_backend/app/routes/autocomplete_routes.py diff --git a/surfsense_backend/app/routes/__init__.py b/surfsense_backend/app/routes/__init__.py index 1937f11cb..a063b5976 100644 --- a/surfsense_backend/app/routes/__init__.py +++ b/surfsense_backend/app/routes/__init__.py @@ -3,6 +3,7 @@ from fastapi import APIRouter from .airtable_add_connector_route import ( router as airtable_add_connector_router, ) +from .autocomplete_routes import router as autocomplete_router from .chat_comments_routes import router as chat_comments_router from .circleback_webhook_route import router as circleback_webhook_router from .clickup_add_connector_route import router as clickup_add_connector_router @@ -95,3 +96,4 @@ router.include_router(incentive_tasks_router) # Incentive tasks for earning fre router.include_router(stripe_router) # Stripe checkout for additional page packs router.include_router(youtube_router) # YouTube playlist resolution router.include_router(prompts_router) +router.include_router(autocomplete_router) # Lightweight autocomplete with KB context diff --git a/surfsense_backend/app/routes/autocomplete_routes.py b/surfsense_backend/app/routes/autocomplete_routes.py new file mode 100644 index 000000000..9a285a723 --- /dev/null +++ b/surfsense_backend/app/routes/autocomplete_routes.py @@ -0,0 +1,136 @@ +import logging +from typing import AsyncGenerator + +from fastapi import APIRouter, Depends, Query +from fastapi.responses import StreamingResponse +from langchain_core.messages import HumanMessage, SystemMessage +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db import User, get_async_session +from app.retriever.chunks_hybrid_search import ChucksHybridSearchRetriever +from app.services.llm_service import get_agent_llm +from app.services.new_streaming_service import VercelStreamingService +from app.users import current_active_user + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/autocomplete", tags=["autocomplete"]) + +AUTOCOMPLETE_SYSTEM_PROMPT = """You are an inline text autocomplete engine. Your job is to complete the user's text naturally. + +Rules: +- Output ONLY the continuation text. Do NOT repeat what the user already typed. +- Keep completions concise: 1-3 sentences maximum. +- Match the user's tone, style, and language. +- If knowledge base context is provided, use it to make the completion factually accurate and personalized. +- Do NOT add quotes, explanations, or meta-commentary. +- Do NOT start with a space unless grammatically required. +- If you cannot produce a useful completion, output nothing.""" + +KB_CONTEXT_TEMPLATE = """ +Relevant knowledge base context (use this to personalize the completion): +--- +{kb_context} +--- +""" + + +async def _stream_autocomplete( + text: str, + cursor_position: int, + search_space_id: int, + session: AsyncSession, +) -> AsyncGenerator[str, None]: + """Stream an autocomplete response with KB context.""" + streaming_service = VercelStreamingService() + + try: + # Text before cursor is what we're completing + text_before_cursor = text[:cursor_position] if cursor_position >= 0 else text + + if not text_before_cursor.strip(): + yield streaming_service.format_message_start() + yield streaming_service.format_finish() + yield streaming_service.format_done() + return + + # Fast KB lookup: vector-only search, top 3 chunks, no planner LLM + kb_context = "" + try: + retriever = ChucksHybridSearchRetriever(session) + chunks = await retriever.vector_search( + query_text=text_before_cursor[-200:], # last 200 chars for relevance + top_k=3, + search_space_id=search_space_id, + ) + if chunks: + kb_snippets = [] + for chunk in chunks: + content = getattr(chunk, "content", None) or getattr(chunk, "chunk_text", "") + if content: + kb_snippets.append(content[:300]) + if kb_snippets: + kb_context = KB_CONTEXT_TEMPLATE.format( + kb_context="\n\n".join(kb_snippets) + ) + except Exception as e: + logger.warning(f"KB search failed for autocomplete, proceeding without context: {e}") + + # Get the search space's configured LLM + llm = await get_agent_llm(session, search_space_id) + if not llm: + yield streaming_service.format_message_start() + error_msg = "No LLM configured for this search space" + yield streaming_service.format_error(error_msg) + yield streaming_service.format_done() + return + + system_prompt = AUTOCOMPLETE_SYSTEM_PROMPT + if kb_context: + system_prompt += kb_context + + messages = [ + SystemMessage(content=system_prompt), + HumanMessage(content=f"Complete this text:\n{text_before_cursor}"), + ] + + # Stream the response + yield streaming_service.format_message_start() + text_id = streaming_service.generate_text_id() + yield streaming_service.format_text_start(text_id) + + async for chunk in llm.astream(messages): + token = chunk.content if hasattr(chunk, "content") else str(chunk) + if token: + yield streaming_service.format_text_delta(text_id, token) + + yield streaming_service.format_text_end(text_id) + yield streaming_service.format_finish() + yield streaming_service.format_done() + + except Exception as e: + logger.error(f"Autocomplete streaming error: {e}") + yield streaming_service.format_error(str(e)) + yield streaming_service.format_done() + + +@router.post("/stream") +async def autocomplete_stream( + text: str = Query(..., description="Current text in the input field"), + cursor_position: int = Query(-1, description="Cursor position in the text (-1 for end)"), + search_space_id: int = Query(..., description="Search space ID for KB context and LLM config"), + user: User = Depends(current_active_user), + session: AsyncSession = Depends(get_async_session), +): + """Stream an autocomplete suggestion based on the current text and KB context.""" + if cursor_position < 0: + cursor_position = len(text) + + return StreamingResponse( + _stream_autocomplete(text, cursor_position, search_space_id, session), + media_type="text/event-stream", + headers={ + **VercelStreamingService.get_response_headers(), + "X-Accel-Buffering": "no", + }, + ) From bcc227a4ddc34fbd6af3fb1115e01af0997bac85 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 2 Apr 2026 13:19:59 +0200 Subject: [PATCH 03/42] feat: add suggestion tooltip UI and autocomplete API types --- surfsense_web/app/suggestion/layout.tsx | 13 ++ surfsense_web/app/suggestion/page.tsx | 160 ++++++++++++++++++++ surfsense_web/app/suggestion/suggestion.css | 96 ++++++++++++ surfsense_web/types/window.d.ts | 6 + 4 files changed, 275 insertions(+) create mode 100644 surfsense_web/app/suggestion/layout.tsx create mode 100644 surfsense_web/app/suggestion/page.tsx create mode 100644 surfsense_web/app/suggestion/suggestion.css diff --git a/surfsense_web/app/suggestion/layout.tsx b/surfsense_web/app/suggestion/layout.tsx new file mode 100644 index 000000000..36b7e037b --- /dev/null +++ b/surfsense_web/app/suggestion/layout.tsx @@ -0,0 +1,13 @@ +import "./suggestion.css"; + +export const metadata = { + title: "SurfSense Suggestion", +}; + +export default function SuggestionLayout({ + children, +}: { + children: React.ReactNode; +}) { + return
{children}
; +} diff --git a/surfsense_web/app/suggestion/page.tsx b/surfsense_web/app/suggestion/page.tsx new file mode 100644 index 000000000..14dfab3af --- /dev/null +++ b/surfsense_web/app/suggestion/page.tsx @@ -0,0 +1,160 @@ +"use client"; + +import { useCallback, useEffect, useRef, useState } from "react"; +import { getBearerToken } from "@/lib/auth-utils"; + +type SSEEvent = + | { type: "text-delta"; id: string; delta: string } + | { type: "text-start"; id: string } + | { type: "text-end"; id: string } + | { type: "start"; messageId: string } + | { type: "finish" } + | { type: "error"; errorText: string }; + +export default function SuggestionPage() { + const [suggestion, setSuggestion] = useState(""); + const [isLoading, setIsLoading] = useState(true); + const [error, setError] = useState(null); + const abortRef = useRef(null); + + const fetchSuggestion = useCallback( + async (text: string, cursorPosition: number, searchSpaceId: string) => { + abortRef.current?.abort(); + const controller = new AbortController(); + abortRef.current = controller; + + setIsLoading(true); + setSuggestion(""); + setError(null); + + const token = getBearerToken(); + if (!token) { + setError("Not authenticated"); + setIsLoading(false); + return; + } + + const backendUrl = + process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL || "http://localhost:8000"; + + const params = new URLSearchParams({ + text, + cursor_position: String(cursorPosition), + search_space_id: searchSpaceId, + }); + + try { + const response = await fetch( + `${backendUrl}/api/v1/autocomplete/stream?${params}`, + { + method: "POST", + headers: { + Authorization: `Bearer ${token}`, + "Content-Type": "application/json", + }, + signal: controller.signal, + }, + ); + + if (!response.ok) { + setError(`Error: ${response.status}`); + setIsLoading(false); + return; + } + + if (!response.body) { + setError("No response body"); + setIsLoading(false); + return; + } + + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ""; + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + buffer += decoder.decode(value, { stream: true }); + const events = buffer.split(/\r?\n\r?\n/); + buffer = events.pop() || ""; + + for (const event of events) { + const lines = event.split(/\r?\n/); + for (const line of lines) { + if (!line.startsWith("data: ")) continue; + const data = line.slice(6).trim(); + if (!data || data === "[DONE]") continue; + + try { + const parsed: SSEEvent = JSON.parse(data); + if (parsed.type === "text-delta") { + setSuggestion((prev) => { + const updated = prev + parsed.delta; + window.electronAPI?.updateSuggestionText?.(updated); + return updated; + }); + } else if (parsed.type === "error") { + setError(parsed.errorText); + } + } catch { + continue; + } + } + } + } + } catch (err) { + if (err instanceof DOMException && err.name === "AbortError") return; + setError("Failed to get suggestion"); + } finally { + setIsLoading(false); + } + }, + [], + ); + + useEffect(() => { + if (!window.electronAPI?.onAutocompleteContext) return; + + const cleanup = window.electronAPI.onAutocompleteContext((data) => { + const searchSpaceId = data.searchSpaceId || "1"; + fetchSuggestion(data.text, data.cursorPosition, searchSpaceId); + }); + + return cleanup; + }, [fetchSuggestion]); + + if (error) { + return ( +
+ {error} +
+ ); + } + + if (isLoading && !suggestion) { + return ( +
+
+ + + +
+
+ ); + } + + if (!suggestion) return null; + + return ( +
+

{suggestion}

+
+ Tab accept + · + Esc dismiss +
+
+ ); +} diff --git a/surfsense_web/app/suggestion/suggestion.css b/surfsense_web/app/suggestion/suggestion.css new file mode 100644 index 000000000..e9471e7f8 --- /dev/null +++ b/surfsense_web/app/suggestion/suggestion.css @@ -0,0 +1,96 @@ +.suggestion-body { + margin: 0; + padding: 0; + background: transparent; + overflow: hidden; + font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; + -webkit-font-smoothing: antialiased; + user-select: none; + -webkit-app-region: no-drag; +} + +.suggestion-tooltip { + background: rgba(30, 30, 30, 0.95); + backdrop-filter: blur(12px); + -webkit-backdrop-filter: blur(12px); + border: 1px solid rgba(255, 255, 255, 0.1); + border-radius: 10px; + padding: 10px 14px; + margin: 4px; + max-width: 400px; + box-shadow: 0 8px 32px rgba(0, 0, 0, 0.4), + 0 2px 8px rgba(0, 0, 0, 0.2); +} + +.suggestion-text { + color: rgba(255, 255, 255, 0.9); + font-size: 13px; + line-height: 1.5; + margin: 0 0 8px 0; + word-wrap: break-word; + white-space: pre-wrap; +} + +.suggestion-hint { + color: rgba(255, 255, 255, 0.4); + font-size: 11px; + display: flex; + align-items: center; + gap: 4px; +} + +.suggestion-key { + background: rgba(255, 255, 255, 0.1); + border: 1px solid rgba(255, 255, 255, 0.15); + border-radius: 3px; + padding: 1px 5px; + font-size: 10px; + font-weight: 500; + color: rgba(255, 255, 255, 0.6); +} + +.suggestion-separator { + margin: 0 2px; +} + +.suggestion-error { + border-color: rgba(255, 80, 80, 0.3); +} + +.suggestion-error-text { + color: rgba(255, 120, 120, 0.9); + font-size: 12px; +} + +.suggestion-loading { + display: flex; + gap: 4px; + padding: 4px 0; +} + +.suggestion-dot { + width: 5px; + height: 5px; + border-radius: 50%; + background: rgba(255, 255, 255, 0.4); + animation: suggestion-pulse 1.2s infinite ease-in-out; +} + +.suggestion-dot:nth-child(2) { + animation-delay: 0.15s; +} + +.suggestion-dot:nth-child(3) { + animation-delay: 0.3s; +} + +@keyframes suggestion-pulse { + 0%, 80%, 100% { + opacity: 0.3; + transform: scale(0.8); + } + 40% { + opacity: 1; + transform: scale(1); + } +} diff --git a/surfsense_web/types/window.d.ts b/surfsense_web/types/window.d.ts index 9cf1aa596..a30358527 100644 --- a/surfsense_web/types/window.d.ts +++ b/surfsense_web/types/window.d.ts @@ -14,6 +14,12 @@ interface ElectronAPI { setQuickAskMode: (mode: string) => Promise; getQuickAskMode: () => Promise; replaceText: (text: string) => Promise; + onAutocompleteContext: (callback: (data: { text: string; cursorPosition: number; searchSpaceId?: string }) => void) => () => void; + acceptSuggestion: (text: string) => Promise; + dismissSuggestion: () => Promise; + updateSuggestionText: (text: string) => Promise; + setAutocompleteEnabled: (enabled: boolean) => Promise; + getAutocompleteEnabled: () => Promise; } declare global { From ec2b7851b6393147b09a9f24273fe84d4314371f Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 2 Apr 2026 13:26:32 +0200 Subject: [PATCH 04/42] feat: add macOS permission infrastructure for autocomplete --- surfsense_desktop/src/ipc/channels.ts | 5 ++ surfsense_desktop/src/ipc/handlers.ts | 22 +++++++++ surfsense_desktop/src/modules/permissions.ts | 50 ++++++++++++++++++++ surfsense_desktop/src/preload.ts | 5 ++ surfsense_web/types/window.d.ts | 9 ++++ 5 files changed, 91 insertions(+) create mode 100644 surfsense_desktop/src/modules/permissions.ts diff --git a/surfsense_desktop/src/ipc/channels.ts b/surfsense_desktop/src/ipc/channels.ts index 25ec1bc0e..a5209dcf3 100644 --- a/surfsense_desktop/src/ipc/channels.ts +++ b/surfsense_desktop/src/ipc/channels.ts @@ -6,4 +6,9 @@ export const IPC_CHANNELS = { SET_QUICK_ASK_MODE: 'set-quick-ask-mode', GET_QUICK_ASK_MODE: 'get-quick-ask-mode', REPLACE_TEXT: 'replace-text', + // Permissions + GET_PERMISSIONS_STATUS: 'get-permissions-status', + REQUEST_ACCESSIBILITY: 'request-accessibility', + REQUEST_INPUT_MONITORING: 'request-input-monitoring', + RESTART_APP: 'restart-app', } as const; diff --git a/surfsense_desktop/src/ipc/handlers.ts b/surfsense_desktop/src/ipc/handlers.ts index 18e343719..fc31329f1 100644 --- a/surfsense_desktop/src/ipc/handlers.ts +++ b/surfsense_desktop/src/ipc/handlers.ts @@ -1,5 +1,11 @@ import { app, ipcMain, shell } from 'electron'; import { IPC_CHANNELS } from './channels'; +import { + getPermissionsStatus, + requestAccessibility, + requestInputMonitoring, + restartApp, +} from '../modules/permissions'; export function registerIpcHandlers(): void { ipcMain.on(IPC_CHANNELS.OPEN_EXTERNAL, (_event, url: string) => { @@ -16,4 +22,20 @@ export function registerIpcHandlers(): void { ipcMain.handle(IPC_CHANNELS.GET_APP_VERSION, () => { return app.getVersion(); }); + + ipcMain.handle(IPC_CHANNELS.GET_PERMISSIONS_STATUS, () => { + return getPermissionsStatus(); + }); + + ipcMain.handle(IPC_CHANNELS.REQUEST_ACCESSIBILITY, () => { + requestAccessibility(); + }); + + ipcMain.handle(IPC_CHANNELS.REQUEST_INPUT_MONITORING, () => { + requestInputMonitoring(); + }); + + ipcMain.handle(IPC_CHANNELS.RESTART_APP, () => { + restartApp(); + }); } diff --git a/surfsense_desktop/src/modules/permissions.ts b/surfsense_desktop/src/modules/permissions.ts new file mode 100644 index 000000000..9a6159c9a --- /dev/null +++ b/surfsense_desktop/src/modules/permissions.ts @@ -0,0 +1,50 @@ +import { app } from 'electron'; + +type PermissionStatus = 'authorized' | 'denied' | 'not determined' | 'restricted' | 'limited'; + +export interface PermissionsStatus { + accessibility: PermissionStatus; + inputMonitoring: PermissionStatus; +} + +function isMac(): boolean { + return process.platform === 'darwin'; +} + +function getNodeMacPermissions() { + return require('node-mac-permissions'); +} + +export function getPermissionsStatus(): PermissionsStatus { + if (!isMac()) { + return { accessibility: 'authorized', inputMonitoring: 'authorized' }; + } + + const perms = getNodeMacPermissions(); + return { + accessibility: perms.getAuthStatus('accessibility'), + inputMonitoring: perms.getAuthStatus('input-monitoring'), + }; +} + +export function allPermissionsGranted(): boolean { + const status = getPermissionsStatus(); + return status.accessibility === 'authorized' && status.inputMonitoring === 'authorized'; +} + +export function requestAccessibility(): void { + if (!isMac()) return; + const perms = getNodeMacPermissions(); + perms.askForAccessibilityAccess(); +} + +export async function requestInputMonitoring(): Promise { + if (!isMac()) return 'authorized'; + const perms = getNodeMacPermissions(); + return perms.askForInputMonitoringAccess('listen'); +} + +export function restartApp(): void { + app.relaunch(); + app.exit(0); +} diff --git a/surfsense_desktop/src/preload.ts b/surfsense_desktop/src/preload.ts index 264ec25b3..069276489 100644 --- a/surfsense_desktop/src/preload.ts +++ b/surfsense_desktop/src/preload.ts @@ -21,4 +21,9 @@ contextBridge.exposeInMainWorld('electronAPI', { setQuickAskMode: (mode: string) => ipcRenderer.invoke(IPC_CHANNELS.SET_QUICK_ASK_MODE, mode), getQuickAskMode: () => ipcRenderer.invoke(IPC_CHANNELS.GET_QUICK_ASK_MODE), replaceText: (text: string) => ipcRenderer.invoke(IPC_CHANNELS.REPLACE_TEXT, text), + // Permissions + getPermissionsStatus: () => ipcRenderer.invoke(IPC_CHANNELS.GET_PERMISSIONS_STATUS), + requestAccessibility: () => ipcRenderer.invoke(IPC_CHANNELS.REQUEST_ACCESSIBILITY), + requestInputMonitoring: () => ipcRenderer.invoke(IPC_CHANNELS.REQUEST_INPUT_MONITORING), + restartApp: () => ipcRenderer.invoke(IPC_CHANNELS.RESTART_APP), }); diff --git a/surfsense_web/types/window.d.ts b/surfsense_web/types/window.d.ts index a30358527..8cf331b42 100644 --- a/surfsense_web/types/window.d.ts +++ b/surfsense_web/types/window.d.ts @@ -14,6 +14,15 @@ interface ElectronAPI { setQuickAskMode: (mode: string) => Promise; getQuickAskMode: () => Promise; replaceText: (text: string) => Promise; + // Permissions + getPermissionsStatus: () => Promise<{ + accessibility: 'authorized' | 'denied' | 'not determined' | 'restricted' | 'limited'; + inputMonitoring: 'authorized' | 'denied' | 'not determined' | 'restricted' | 'limited'; + }>; + requestAccessibility: () => Promise; + requestInputMonitoring: () => Promise; + restartApp: () => Promise; + // Autocomplete onAutocompleteContext: (callback: (data: { text: string; cursorPosition: number; searchSpaceId?: string }) => void) => () => void; acceptSuggestion: (text: string) => Promise; dismissSuggestion: () => Promise; From eaabad38fcd2bfc8d6bef87f8ea60ea4d8192d78 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 2 Apr 2026 13:44:57 +0200 Subject: [PATCH 05/42] feat: add permission onboarding page and startup routing for macOS --- surfsense_desktop/src/ipc/handlers.ts | 4 +- surfsense_desktop/src/main.ts | 15 +- surfsense_desktop/src/modules/window.ts | 4 +- .../app/desktop/permissions/page.tsx | 212 ++++++++++++++++++ .../app/{ => desktop}/suggestion/layout.tsx | 0 .../app/{ => desktop}/suggestion/page.tsx | 0 .../{ => desktop}/suggestion/suggestion.css | 0 7 files changed, 228 insertions(+), 7 deletions(-) create mode 100644 surfsense_web/app/desktop/permissions/page.tsx rename surfsense_web/app/{ => desktop}/suggestion/layout.tsx (100%) rename surfsense_web/app/{ => desktop}/suggestion/page.tsx (100%) rename surfsense_web/app/{ => desktop}/suggestion/suggestion.css (100%) diff --git a/surfsense_desktop/src/ipc/handlers.ts b/surfsense_desktop/src/ipc/handlers.ts index fc31329f1..a6d82be4b 100644 --- a/surfsense_desktop/src/ipc/handlers.ts +++ b/surfsense_desktop/src/ipc/handlers.ts @@ -31,8 +31,8 @@ export function registerIpcHandlers(): void { requestAccessibility(); }); - ipcMain.handle(IPC_CHANNELS.REQUEST_INPUT_MONITORING, () => { - requestInputMonitoring(); + ipcMain.handle(IPC_CHANNELS.REQUEST_INPUT_MONITORING, async () => { + return await requestInputMonitoring(); }); ipcMain.handle(IPC_CHANNELS.RESTART_APP, () => { diff --git a/surfsense_desktop/src/main.ts b/surfsense_desktop/src/main.ts index 3ab41073b..bc164758b 100644 --- a/surfsense_desktop/src/main.ts +++ b/surfsense_desktop/src/main.ts @@ -7,6 +7,7 @@ import { setupAutoUpdater } from './modules/auto-updater'; import { setupMenu } from './modules/menu'; import { registerQuickAsk, unregisterQuickAsk } from './modules/quick-ask'; import { registerIpcHandlers } from './ipc/handlers'; +import { allPermissionsGranted } from './modules/permissions'; registerGlobalErrorHandlers(); @@ -16,7 +17,13 @@ if (!setupDeepLinks()) { registerIpcHandlers(); -// App lifecycle +function getInitialPath(): string { + if (process.platform === 'darwin' && !allPermissionsGranted()) { + return '/desktop/permissions'; + } + return '/dashboard'; +} + app.whenReady().then(async () => { setupMenu(); try { @@ -26,7 +33,9 @@ app.whenReady().then(async () => { setTimeout(() => app.quit(), 0); return; } - createMainWindow(); + + const initialPath = getInitialPath(); + createMainWindow(initialPath); registerQuickAsk(); setupAutoUpdater(); @@ -34,7 +43,7 @@ app.whenReady().then(async () => { app.on('activate', () => { if (BrowserWindow.getAllWindows().length === 0) { - createMainWindow(); + createMainWindow(getInitialPath()); } }); }); diff --git a/surfsense_desktop/src/modules/window.ts b/surfsense_desktop/src/modules/window.ts index 245814cad..7a77773d8 100644 --- a/surfsense_desktop/src/modules/window.ts +++ b/surfsense_desktop/src/modules/window.ts @@ -12,7 +12,7 @@ export function getMainWindow(): BrowserWindow | null { return mainWindow; } -export function createMainWindow(): BrowserWindow { +export function createMainWindow(initialPath = '/dashboard'): BrowserWindow { mainWindow = new BrowserWindow({ width: 1280, height: 800, @@ -33,7 +33,7 @@ export function createMainWindow(): BrowserWindow { mainWindow?.show(); }); - mainWindow.loadURL(`http://localhost:${getServerPort()}/dashboard`); + mainWindow.loadURL(`http://localhost:${getServerPort()}${initialPath}`); mainWindow.webContents.setWindowOpenHandler(({ url }) => { if (url.startsWith('http://localhost')) { diff --git a/surfsense_web/app/desktop/permissions/page.tsx b/surfsense_web/app/desktop/permissions/page.tsx new file mode 100644 index 000000000..2bcdc42df --- /dev/null +++ b/surfsense_web/app/desktop/permissions/page.tsx @@ -0,0 +1,212 @@ +"use client"; + +import { useEffect, useState } from "react"; +import { useRouter } from "next/navigation"; +import { Logo } from "@/components/Logo"; +import { Button } from "@/components/ui/button"; +import { Spinner } from "@/components/ui/spinner"; + +type PermissionStatus = "authorized" | "denied" | "not determined" | "restricted" | "limited"; + +interface PermissionsStatus { + accessibility: PermissionStatus; + inputMonitoring: PermissionStatus; +} + +const STEPS = [ + { + id: "input-monitoring", + title: "Input Monitoring", + description: "Helps you write faster by enriching your text with suggestions from your knowledge base.", + action: "requestInputMonitoring", + field: "inputMonitoring" as const, + }, + { + id: "accessibility", + title: "Accessibility", + description: "Lets you accept suggestions seamlessly, right where you're typing.", + action: "requestAccessibility", + field: "accessibility" as const, + }, +]; + +function StatusBadge({ status }: { status: PermissionStatus }) { + if (status === "authorized") { + return ( + + + Granted + + ); + } + if (status === "denied") { + return ( + + + Denied + + ); + } + return ( + + + Pending + + ); +} + +export default function DesktopPermissionsPage() { + const router = useRouter(); + const [permissions, setPermissions] = useState(null); + const [isElectron, setIsElectron] = useState(false); + + useEffect(() => { + if (!window.electronAPI) return; + setIsElectron(true); + + let interval: ReturnType | null = null; + + const isResolved = (s: string) => s === "authorized" || s === "restricted"; + + const poll = async () => { + const status = await window.electronAPI!.getPermissionsStatus(); + setPermissions(status); + + if (isResolved(status.accessibility) && isResolved(status.inputMonitoring)) { + if (interval) clearInterval(interval); + } + }; + + poll(); + interval = setInterval(poll, 2000); + return () => { if (interval) clearInterval(interval); }; + }, []); + + if (!isElectron) { + return ( +
+

This page is only available in the desktop app.

+
+ ); + } + + if (!permissions) { + return ( +
+ +
+ ); + } + + const allGranted = permissions.accessibility === "authorized" && permissions.inputMonitoring === "authorized"; + + const handleRequest = async (action: string) => { + if (action === "requestInputMonitoring") { + await window.electronAPI!.requestInputMonitoring(); + } else if (action === "requestAccessibility") { + await window.electronAPI!.requestAccessibility(); + } + }; + + const handleContinue = () => { + if (allGranted) { + window.electronAPI!.restartApp(); + } + }; + + const handleSkip = () => { + router.push("/dashboard"); + }; + + return ( +
+
+ {/* Header */} +
+ +
+

System Permissions

+

+ SurfSense needs two macOS permissions to provide system-wide autocomplete. +

+
+
+ + {/* Steps */} +
+ {STEPS.map((step, index) => { + const status = permissions[step.field]; + const isGranted = status === "authorized"; + + return ( +
+
+
+ + {isGranted ? "✓" : index + 1} + +
+

{step.title}

+

{step.description}

+
+
+ +
+ {!isGranted && ( +
+ + {status === "denied" && ( +

+ Toggle SurfSense on in System Settings to continue. +

+ )} +
+ )} +
+ ); + })} +
+ + {/* Footer */} +
+ {allGranted ? ( + <> + +

+ A restart is needed for permissions to take effect. +

+ + ) : ( + <> + + + + )} +
+
+
+ ); +} diff --git a/surfsense_web/app/suggestion/layout.tsx b/surfsense_web/app/desktop/suggestion/layout.tsx similarity index 100% rename from surfsense_web/app/suggestion/layout.tsx rename to surfsense_web/app/desktop/suggestion/layout.tsx diff --git a/surfsense_web/app/suggestion/page.tsx b/surfsense_web/app/desktop/suggestion/page.tsx similarity index 100% rename from surfsense_web/app/suggestion/page.tsx rename to surfsense_web/app/desktop/suggestion/page.tsx diff --git a/surfsense_web/app/suggestion/suggestion.css b/surfsense_web/app/desktop/suggestion/suggestion.css similarity index 100% rename from surfsense_web/app/suggestion/suggestion.css rename to surfsense_web/app/desktop/suggestion/suggestion.css From b2706b00a1bf793e8d1aa63235a0c53d5cc6766c Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 2 Apr 2026 14:29:12 +0200 Subject: [PATCH 06/42] feat: add autocomplete module with keystroke monitoring and IPC wiring --- surfsense_desktop/src/ipc/channels.ts | 7 + surfsense_desktop/src/main.ts | 3 + surfsense_desktop/src/modules/autocomplete.ts | 267 ++++++++++++++++++ surfsense_desktop/src/modules/platform.ts | 40 +++ surfsense_desktop/src/preload.ts | 13 + 5 files changed, 330 insertions(+) create mode 100644 surfsense_desktop/src/modules/autocomplete.ts diff --git a/surfsense_desktop/src/ipc/channels.ts b/surfsense_desktop/src/ipc/channels.ts index a5209dcf3..2965f516f 100644 --- a/surfsense_desktop/src/ipc/channels.ts +++ b/surfsense_desktop/src/ipc/channels.ts @@ -11,4 +11,11 @@ export const IPC_CHANNELS = { REQUEST_ACCESSIBILITY: 'request-accessibility', REQUEST_INPUT_MONITORING: 'request-input-monitoring', RESTART_APP: 'restart-app', + // Autocomplete + AUTOCOMPLETE_CONTEXT: 'autocomplete-context', + ACCEPT_SUGGESTION: 'accept-suggestion', + DISMISS_SUGGESTION: 'dismiss-suggestion', + UPDATE_SUGGESTION_TEXT: 'update-suggestion-text', + SET_AUTOCOMPLETE_ENABLED: 'set-autocomplete-enabled', + GET_AUTOCOMPLETE_ENABLED: 'get-autocomplete-enabled', } as const; diff --git a/surfsense_desktop/src/main.ts b/surfsense_desktop/src/main.ts index bc164758b..9623be82e 100644 --- a/surfsense_desktop/src/main.ts +++ b/surfsense_desktop/src/main.ts @@ -6,6 +6,7 @@ import { setupDeepLinks, handlePendingDeepLink } from './modules/deep-links'; import { setupAutoUpdater } from './modules/auto-updater'; import { setupMenu } from './modules/menu'; import { registerQuickAsk, unregisterQuickAsk } from './modules/quick-ask'; +import { registerAutocomplete, unregisterAutocomplete } from './modules/autocomplete'; import { registerIpcHandlers } from './ipc/handlers'; import { allPermissionsGranted } from './modules/permissions'; @@ -37,6 +38,7 @@ app.whenReady().then(async () => { const initialPath = getInitialPath(); createMainWindow(initialPath); registerQuickAsk(); + registerAutocomplete(); setupAutoUpdater(); handlePendingDeepLink(); @@ -56,4 +58,5 @@ app.on('window-all-closed', () => { app.on('will-quit', () => { unregisterQuickAsk(); + unregisterAutocomplete(); }); diff --git a/surfsense_desktop/src/modules/autocomplete.ts b/surfsense_desktop/src/modules/autocomplete.ts new file mode 100644 index 000000000..2b877723f --- /dev/null +++ b/surfsense_desktop/src/modules/autocomplete.ts @@ -0,0 +1,267 @@ +import { BrowserWindow, clipboard, ipcMain, screen, shell } from 'electron'; +import path from 'path'; +import { IPC_CHANNELS } from '../ipc/channels'; +import { allPermissionsGranted } from './permissions'; +import { getFieldContent, getFrontmostApp, hasAccessibilityPermission, simulatePaste } from './platform'; +import { getServerPort } from './server'; +import { getMainWindow } from './window'; + +const DEBOUNCE_MS = 600; +const TOOLTIP_WIDTH = 420; +const TOOLTIP_HEIGHT = 140; + +let uIOhook: any = null; +let UiohookKey: any = {}; +let IGNORED_KEYCODES: Set = new Set(); + +let suggestionWindow: BrowserWindow | null = null; +let debounceTimer: ReturnType | null = null; +let hookStarted = false; +let autocompleteEnabled = true; +let savedClipboard = ''; +let sourceApp = ''; +let pendingSuggestionText = ''; + +function loadUiohook(): boolean { + if (uIOhook) return true; + try { + const mod = require('uiohook-napi'); + uIOhook = mod.uIOhook; + UiohookKey = mod.UiohookKey; + IGNORED_KEYCODES = new Set([ + UiohookKey.Shift, UiohookKey.ShiftRight, + UiohookKey.Ctrl, UiohookKey.CtrlRight, + UiohookKey.Alt, UiohookKey.AltRight, + UiohookKey.Meta, UiohookKey.MetaRight, + UiohookKey.CapsLock, UiohookKey.NumLock, UiohookKey.ScrollLock, + UiohookKey.F1, UiohookKey.F2, UiohookKey.F3, UiohookKey.F4, + UiohookKey.F5, UiohookKey.F6, UiohookKey.F7, UiohookKey.F8, + UiohookKey.F9, UiohookKey.F10, UiohookKey.F11, UiohookKey.F12, + UiohookKey.PrintScreen, + UiohookKey.Insert, UiohookKey.Delete, + UiohookKey.Home, UiohookKey.End, + UiohookKey.PageUp, UiohookKey.PageDown, + UiohookKey.ArrowUp, UiohookKey.ArrowDown, + UiohookKey.ArrowLeft, UiohookKey.ArrowRight, + ]); + console.log('[autocomplete] uiohook-napi loaded'); + return true; + } catch (err) { + console.error('[autocomplete] Failed to load uiohook-napi:', err); + return false; + } +} + +function destroySuggestion(): void { + if (suggestionWindow && !suggestionWindow.isDestroyed()) { + suggestionWindow.close(); + } + suggestionWindow = null; +} + +function clampToScreen(x: number, y: number, w: number, h: number): { x: number; y: number } { + const display = screen.getDisplayNearestPoint({ x, y }); + const { x: dx, y: dy, width: dw, height: dh } = display.workArea; + return { + x: Math.max(dx, Math.min(x, dx + dw - w)), + y: Math.max(dy, Math.min(y, dy + dh - h)), + }; +} + +function createSuggestionWindow(x: number, y: number): BrowserWindow { + destroySuggestion(); + + const pos = clampToScreen(x, y + 20, TOOLTIP_WIDTH, TOOLTIP_HEIGHT); + + suggestionWindow = new BrowserWindow({ + width: TOOLTIP_WIDTH, + height: TOOLTIP_HEIGHT, + x: pos.x, + y: pos.y, + frame: false, + transparent: true, + focusable: false, + alwaysOnTop: true, + skipTaskbar: true, + resizable: false, + hasShadow: true, + type: 'panel', + webPreferences: { + preload: path.join(__dirname, 'preload.js'), + contextIsolation: true, + nodeIntegration: false, + sandbox: true, + }, + show: false, + }); + + suggestionWindow.loadURL(`http://localhost:${getServerPort()}/desktop/suggestion?t=${Date.now()}`); + + suggestionWindow.once('ready-to-show', () => { + suggestionWindow?.showInactive(); + }); + + suggestionWindow.webContents.setWindowOpenHandler(({ url }) => { + if (url.startsWith('http://localhost')) { + return { action: 'allow' }; + } + shell.openExternal(url); + return { action: 'deny' }; + }); + + suggestionWindow.on('closed', () => { + suggestionWindow = null; + }); + + return suggestionWindow; +} + +function clearDebounce(): void { + if (debounceTimer) { + clearTimeout(debounceTimer); + debounceTimer = null; + } +} + +function isSurfSenseWindow(): boolean { + const app = getFrontmostApp(); + return app === 'Electron' || app === 'SurfSense' || app === 'surfsense-desktop'; +} + +function onKeyDown(event: { keycode: number; ctrlKey?: boolean; metaKey?: boolean; altKey?: boolean }): void { + if (!autocompleteEnabled) return; + + if (event.keycode === UiohookKey.Tab && suggestionWindow && !suggestionWindow.isDestroyed()) { + if (pendingSuggestionText) { + acceptAndInject(pendingSuggestionText); + } + return; + } + + if (event.keycode === UiohookKey.Escape) { + if (suggestionWindow && !suggestionWindow.isDestroyed()) { + destroySuggestion(); + pendingSuggestionText = ''; + } + clearDebounce(); + return; + } + + if (IGNORED_KEYCODES.has(event.keycode)) return; + if (event.ctrlKey || event.metaKey || event.altKey) return; + if (isSurfSenseWindow()) return; + + if (suggestionWindow && !suggestionWindow.isDestroyed()) { + destroySuggestion(); + } + + clearDebounce(); + debounceTimer = setTimeout(() => { + triggerAutocomplete(); + }, DEBOUNCE_MS); +} + +async function triggerAutocomplete(): Promise { + if (!hasAccessibilityPermission()) return; + if (isSurfSenseWindow()) return; + + const fieldContent = getFieldContent(); + if (!fieldContent || !fieldContent.text.trim()) return; + if (fieldContent.text.trim().length < 5) return; + + sourceApp = getFrontmostApp(); + savedClipboard = clipboard.readText(); + + const cursor = screen.getCursorScreenPoint(); + const win = createSuggestionWindow(cursor.x, cursor.y); + + let searchSpaceId = '1'; + const mainWin = getMainWindow(); + if (mainWin && !mainWin.isDestroyed()) { + const mainUrl = mainWin.webContents.getURL(); + const match = mainUrl.match(/\/dashboard\/(\d+)/); + if (match) { + searchSpaceId = match[1]; + } + } + + win.webContents.once('did-finish-load', () => { + if (suggestionWindow && !suggestionWindow.isDestroyed()) { + suggestionWindow.webContents.send(IPC_CHANNELS.AUTOCOMPLETE_CONTEXT, { + text: fieldContent.text, + cursorPosition: fieldContent.cursorPosition, + searchSpaceId, + }); + } + }); +} + +async function acceptAndInject(text: string): Promise { + if (!sourceApp) return; + if (!hasAccessibilityPermission()) return; + + clipboard.writeText(text); + destroySuggestion(); + pendingSuggestionText = ''; + + try { + await new Promise((r) => setTimeout(r, 50)); + simulatePaste(); + await new Promise((r) => setTimeout(r, 100)); + clipboard.writeText(savedClipboard); + } catch { + clipboard.writeText(savedClipboard); + } +} + +function registerIpcHandlers(): void { + ipcMain.handle(IPC_CHANNELS.ACCEPT_SUGGESTION, async (_event, text: string) => { + await acceptAndInject(text); + }); + ipcMain.handle(IPC_CHANNELS.DISMISS_SUGGESTION, () => { + destroySuggestion(); + pendingSuggestionText = ''; + }); + ipcMain.handle(IPC_CHANNELS.UPDATE_SUGGESTION_TEXT, (_event, text: string) => { + pendingSuggestionText = text; + }); + ipcMain.handle(IPC_CHANNELS.SET_AUTOCOMPLETE_ENABLED, (_event, enabled: boolean) => { + autocompleteEnabled = enabled; + if (!enabled) { + clearDebounce(); + destroySuggestion(); + } + }); + ipcMain.handle(IPC_CHANNELS.GET_AUTOCOMPLETE_ENABLED, () => autocompleteEnabled); +} + +export function registerAutocomplete(): void { + registerIpcHandlers(); + + if (!allPermissionsGranted()) { + console.log('[autocomplete] Permissions not granted — hook not started'); + return; + } + + if (!loadUiohook()) { + console.error('[autocomplete] Cannot start: uiohook-napi failed to load'); + return; + } + + uIOhook.on('keydown', onKeyDown); + try { + uIOhook.start(); + hookStarted = true; + console.log('[autocomplete] uIOhook started'); + } catch (err) { + console.error('[autocomplete] uIOhook.start() failed:', err); + } +} + +export function unregisterAutocomplete(): void { + clearDebounce(); + destroySuggestion(); + if (uIOhook && hookStarted) { + try { uIOhook.stop(); } catch { /* already stopped */ } + } +} diff --git a/surfsense_desktop/src/modules/platform.ts b/surfsense_desktop/src/modules/platform.ts index 37e126799..262866d07 100644 --- a/surfsense_desktop/src/modules/platform.ts +++ b/surfsense_desktop/src/modules/platform.ts @@ -53,3 +53,43 @@ export function checkAccessibilityPermission(): boolean { if (process.platform !== 'darwin') return true; return systemPreferences.isTrustedAccessibilityClient(true); } + +export function hasAccessibilityPermission(): boolean { + if (process.platform !== 'darwin') return true; + return systemPreferences.isTrustedAccessibilityClient(false); +} + +export interface FieldContent { + text: string; + cursorPosition: number; +} + +export function getFieldContent(): FieldContent | null { + if (process.platform !== 'darwin') return null; + + try { + const text = execSync( + 'osascript -e \'tell application "System Events" to get value of attribute "AXValue" of focused UI element of first application process whose frontmost is true\'', + { timeout: 500 } + ).toString().trim(); + + let cursorPosition = text.length; + try { + const rangeStr = execSync( + 'osascript -e \'tell application "System Events" to get value of attribute "AXSelectedTextRange" of focused UI element of first application process whose frontmost is true\'', + { timeout: 500 } + ).toString().trim(); + + const locationMatch = rangeStr.match(/location[:\s]*(\d+)/i); + if (locationMatch) { + cursorPosition = parseInt(locationMatch[1], 10); + } + } catch { + // Fall back to end of text + } + + return { text, cursorPosition }; + } catch { + return null; + } +} diff --git a/surfsense_desktop/src/preload.ts b/surfsense_desktop/src/preload.ts index 069276489..956afcc46 100644 --- a/surfsense_desktop/src/preload.ts +++ b/surfsense_desktop/src/preload.ts @@ -26,4 +26,17 @@ contextBridge.exposeInMainWorld('electronAPI', { requestAccessibility: () => ipcRenderer.invoke(IPC_CHANNELS.REQUEST_ACCESSIBILITY), requestInputMonitoring: () => ipcRenderer.invoke(IPC_CHANNELS.REQUEST_INPUT_MONITORING), restartApp: () => ipcRenderer.invoke(IPC_CHANNELS.RESTART_APP), + // Autocomplete + onAutocompleteContext: (callback: (data: { text: string; cursorPosition: number; searchSpaceId?: string }) => void) => { + const listener = (_event: unknown, data: { text: string; cursorPosition: number; searchSpaceId?: string }) => callback(data); + ipcRenderer.on(IPC_CHANNELS.AUTOCOMPLETE_CONTEXT, listener); + return () => { + ipcRenderer.removeListener(IPC_CHANNELS.AUTOCOMPLETE_CONTEXT, listener); + }; + }, + acceptSuggestion: (text: string) => ipcRenderer.invoke(IPC_CHANNELS.ACCEPT_SUGGESTION, text), + dismissSuggestion: () => ipcRenderer.invoke(IPC_CHANNELS.DISMISS_SUGGESTION), + updateSuggestionText: (text: string) => ipcRenderer.invoke(IPC_CHANNELS.UPDATE_SUGGESTION_TEXT, text), + setAutocompleteEnabled: (enabled: boolean) => ipcRenderer.invoke(IPC_CHANNELS.SET_AUTOCOMPLETE_ENABLED, enabled), + getAutocompleteEnabled: () => ipcRenderer.invoke(IPC_CHANNELS.GET_AUTOCOMPLETE_ENABLED), }); From 6899134a20605b619e372564666a25eba5bb76fa Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 2 Apr 2026 14:37:26 +0200 Subject: [PATCH 07/42] feat: add autocomplete toggle in desktop settings --- .../components/DesktopContent.tsx | 79 +++++++++++++++++++ .../settings/user-settings-dialog.tsx | 7 +- 2 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 surfsense_web/app/dashboard/[search_space_id]/user-settings/components/DesktopContent.tsx diff --git a/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/DesktopContent.tsx b/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/DesktopContent.tsx new file mode 100644 index 000000000..1522e153f --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/DesktopContent.tsx @@ -0,0 +1,79 @@ +"use client"; + +import { useEffect, useState } from "react"; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; +import { Label } from "@/components/ui/label"; +import { Switch } from "@/components/ui/switch"; +import { Spinner } from "@/components/ui/spinner"; + +export function DesktopContent() { + const [isElectron, setIsElectron] = useState(false); + const [loading, setLoading] = useState(true); + const [enabled, setEnabled] = useState(true); + + useEffect(() => { + if (!window.electronAPI) { + setLoading(false); + return; + } + setIsElectron(true); + + window.electronAPI.getAutocompleteEnabled().then((val) => { + setEnabled(val); + setLoading(false); + }); + }, []); + + if (!isElectron) { + return ( +
+

+ Desktop settings are only available in the SurfSense desktop app. +

+
+ ); + } + + if (loading) { + return ( +
+ +
+ ); + } + + const handleToggle = async (checked: boolean) => { + setEnabled(checked); + await window.electronAPI!.setAutocompleteEnabled(checked); + }; + + return ( +
+ + + Autocomplete + + Get inline writing suggestions powered by your knowledge base as you type in any app. + + + +
+
+ +

+ Show suggestions while typing in other applications. +

+
+ +
+ + +
+ ); +} diff --git a/surfsense_web/components/settings/user-settings-dialog.tsx b/surfsense_web/components/settings/user-settings-dialog.tsx index 389ebc5fd..b74ff973b 100644 --- a/surfsense_web/components/settings/user-settings-dialog.tsx +++ b/surfsense_web/components/settings/user-settings-dialog.tsx @@ -1,13 +1,14 @@ "use client"; import { useAtom } from "jotai"; -import { Globe, KeyRound, Receipt, Sparkles, User } from "lucide-react"; +import { Globe, KeyRound, Monitor, Receipt, Sparkles, User } from "lucide-react"; import { useTranslations } from "next-intl"; import { ApiKeyContent } from "@/app/dashboard/[search_space_id]/user-settings/components/ApiKeyContent"; import { CommunityPromptsContent } from "@/app/dashboard/[search_space_id]/user-settings/components/CommunityPromptsContent"; import { ProfileContent } from "@/app/dashboard/[search_space_id]/user-settings/components/ProfileContent"; import { PromptsContent } from "@/app/dashboard/[search_space_id]/user-settings/components/PromptsContent"; import { PurchaseHistoryContent } from "@/app/dashboard/[search_space_id]/user-settings/components/PurchaseHistoryContent"; +import { DesktopContent } from "@/app/dashboard/[search_space_id]/user-settings/components/DesktopContent"; import { userSettingsDialogAtom } from "@/atoms/settings/settings-dialog.atoms"; import { SettingsDialog } from "@/components/settings/settings-dialog"; @@ -37,6 +38,9 @@ export function UserSettingsDialog() { label: "Purchase History", icon: , }, + ...(typeof window !== "undefined" && window.electronAPI + ? [{ value: "desktop", label: "Desktop", icon: }] + : []), ]; return ( @@ -54,6 +58,7 @@ export function UserSettingsDialog() { {state.initialTab === "prompts" && } {state.initialTab === "community-prompts" && } {state.initialTab === "purchases" && } + {state.initialTab === "desktop" && } ); From 9c1d9357c4e3b0fe5eb25f737069d6494cea2188 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 2 Apr 2026 20:19:16 +0200 Subject: [PATCH 08/42] refactor: fix dynamic tooltip resizing and split autocomplete into SPR modules --- surfsense_desktop/electron-builder.yml | 12 +- surfsense_desktop/package.json | 1 + surfsense_desktop/pnpm-lock.yaml | 3 + surfsense_desktop/scripts/build-electron.mjs | 2 +- surfsense_desktop/src/main.ts | 3 +- .../index.ts} | 181 ++++++++---------- .../modules/autocomplete/keystroke-buffer.ts | 76 ++++++++ .../modules/autocomplete/suggestion-window.ts | 103 ++++++++++ surfsense_desktop/src/modules/platform.ts | 49 ----- .../app/desktop/permissions/page.tsx | 13 +- surfsense_web/app/desktop/suggestion/page.tsx | 6 +- .../app/desktop/suggestion/suggestion.css | 70 ++++--- 12 files changed, 326 insertions(+), 193 deletions(-) rename surfsense_desktop/src/modules/{autocomplete.ts => autocomplete/index.ts} (55%) create mode 100644 surfsense_desktop/src/modules/autocomplete/keystroke-buffer.ts create mode 100644 surfsense_desktop/src/modules/autocomplete/suggestion-window.ts diff --git a/surfsense_desktop/electron-builder.yml b/surfsense_desktop/electron-builder.yml index 74c69d223..115b69c8e 100644 --- a/surfsense_desktop/electron-builder.yml +++ b/surfsense_desktop/electron-builder.yml @@ -10,13 +10,13 @@ files: - dist/**/* - "!node_modules" - node_modules/uiohook-napi/**/* - - "!node_modules/uiohook-napi/build" - "!node_modules/uiohook-napi/src" - "!node_modules/uiohook-napi/libuiohook" - "!node_modules/uiohook-napi/binding.gyp" - node_modules/node-gyp-build/**/* + - node_modules/bindings/**/* + - node_modules/file-uri-to-path/**/* - node_modules/node-mac-permissions/**/* - - "!node_modules/node-mac-permissions/build" - "!node_modules/node-mac-permissions/src" - "!node_modules/node-mac-permissions/binding.gyp" - "!src" @@ -41,13 +41,19 @@ asarUnpack: - "**/*.node" - "node_modules/uiohook-napi/**/*" - "node_modules/node-gyp-build/**/*" + - "node_modules/bindings/**/*" + - "node_modules/file-uri-to-path/**/*" - "node_modules/node-mac-permissions/**/*" mac: icon: assets/icon.icns category: public.app-category.productivity artifactName: "${productName}-${version}-${arch}.${ext}" - hardenedRuntime: true + hardenedRuntime: false gatekeeperAssess: false + extendInfo: + NSInputMonitoringUsageDescription: "SurfSense uses input monitoring to provide system-wide autocomplete suggestions as you type." + NSAccessibilityUsageDescription: "SurfSense uses accessibility features to read text fields and insert suggestions." + NSAppleEventsUsageDescription: "SurfSense uses Apple Events to read text from the active application and insert autocomplete suggestions." target: - target: dmg arch: [x64, arm64] diff --git a/surfsense_desktop/package.json b/surfsense_desktop/package.json index a2e452b7c..01a63b265 100644 --- a/surfsense_desktop/package.json +++ b/surfsense_desktop/package.json @@ -29,6 +29,7 @@ "wait-on": "^9.0.4" }, "dependencies": { + "bindings": "^1.5.0", "electron-updater": "^6.8.3", "get-port-please": "^3.2.0", "node-mac-permissions": "^2.5.0", diff --git a/surfsense_desktop/pnpm-lock.yaml b/surfsense_desktop/pnpm-lock.yaml index 82bad9456..d0b453d31 100644 --- a/surfsense_desktop/pnpm-lock.yaml +++ b/surfsense_desktop/pnpm-lock.yaml @@ -8,6 +8,9 @@ importers: .: dependencies: + bindings: + specifier: ^1.5.0 + version: 1.5.0 electron-updater: specifier: ^6.8.3 version: 6.8.3 diff --git a/surfsense_desktop/scripts/build-electron.mjs b/surfsense_desktop/scripts/build-electron.mjs index 83d941dd2..c2869ec46 100644 --- a/surfsense_desktop/scripts/build-electron.mjs +++ b/surfsense_desktop/scripts/build-electron.mjs @@ -104,7 +104,7 @@ async function buildElectron() { bundle: true, platform: 'node', target: 'node18', - external: ['electron', 'uiohook-napi', 'node-mac-permissions'], + external: ['electron', 'uiohook-napi', 'node-mac-permissions', 'bindings', 'file-uri-to-path'], sourcemap: true, minify: false, define: { diff --git a/surfsense_desktop/src/main.ts b/surfsense_desktop/src/main.ts index 9623be82e..c96453c6d 100644 --- a/surfsense_desktop/src/main.ts +++ b/surfsense_desktop/src/main.ts @@ -19,7 +19,8 @@ if (!setupDeepLinks()) { registerIpcHandlers(); function getInitialPath(): string { - if (process.platform === 'darwin' && !allPermissionsGranted()) { + const granted = allPermissionsGranted(); + if (process.platform === 'darwin' && !granted) { return '/desktop/permissions'; } return '/dashboard'; diff --git a/surfsense_desktop/src/modules/autocomplete.ts b/surfsense_desktop/src/modules/autocomplete/index.ts similarity index 55% rename from surfsense_desktop/src/modules/autocomplete.ts rename to surfsense_desktop/src/modules/autocomplete/index.ts index 2b877723f..2ea37d051 100644 --- a/surfsense_desktop/src/modules/autocomplete.ts +++ b/surfsense_desktop/src/modules/autocomplete/index.ts @@ -1,20 +1,19 @@ -import { BrowserWindow, clipboard, ipcMain, screen, shell } from 'electron'; -import path from 'path'; -import { IPC_CHANNELS } from '../ipc/channels'; -import { allPermissionsGranted } from './permissions'; -import { getFieldContent, getFrontmostApp, hasAccessibilityPermission, simulatePaste } from './platform'; -import { getServerPort } from './server'; -import { getMainWindow } from './window'; +import { clipboard, ipcMain, screen } from 'electron'; +import { IPC_CHANNELS } from '../../ipc/channels'; +import { getFrontmostApp, hasAccessibilityPermission, simulatePaste } from '../platform'; +import { getMainWindow } from '../window'; +import { + appendToBuffer, buildKeycodeMap, getBuffer, getBufferTrimmed, + getLastTrackedApp, removeLastChar, resetBuffer, resolveChar, setLastTrackedApp, +} from './keystroke-buffer'; +import { createSuggestionWindow, destroySuggestion, getSuggestionWindow } from './suggestion-window'; const DEBOUNCE_MS = 600; -const TOOLTIP_WIDTH = 420; -const TOOLTIP_HEIGHT = 140; let uIOhook: any = null; let UiohookKey: any = {}; let IGNORED_KEYCODES: Set = new Set(); -let suggestionWindow: BrowserWindow | null = null; let debounceTimer: ReturnType | null = null; let hookStarted = false; let autocompleteEnabled = true; @@ -38,12 +37,8 @@ function loadUiohook(): boolean { UiohookKey.F5, UiohookKey.F6, UiohookKey.F7, UiohookKey.F8, UiohookKey.F9, UiohookKey.F10, UiohookKey.F11, UiohookKey.F12, UiohookKey.PrintScreen, - UiohookKey.Insert, UiohookKey.Delete, - UiohookKey.Home, UiohookKey.End, - UiohookKey.PageUp, UiohookKey.PageDown, - UiohookKey.ArrowUp, UiohookKey.ArrowDown, - UiohookKey.ArrowLeft, UiohookKey.ArrowRight, ]); + buildKeycodeMap(); console.log('[autocomplete] uiohook-napi loaded'); return true; } catch (err) { @@ -52,70 +47,6 @@ function loadUiohook(): boolean { } } -function destroySuggestion(): void { - if (suggestionWindow && !suggestionWindow.isDestroyed()) { - suggestionWindow.close(); - } - suggestionWindow = null; -} - -function clampToScreen(x: number, y: number, w: number, h: number): { x: number; y: number } { - const display = screen.getDisplayNearestPoint({ x, y }); - const { x: dx, y: dy, width: dw, height: dh } = display.workArea; - return { - x: Math.max(dx, Math.min(x, dx + dw - w)), - y: Math.max(dy, Math.min(y, dy + dh - h)), - }; -} - -function createSuggestionWindow(x: number, y: number): BrowserWindow { - destroySuggestion(); - - const pos = clampToScreen(x, y + 20, TOOLTIP_WIDTH, TOOLTIP_HEIGHT); - - suggestionWindow = new BrowserWindow({ - width: TOOLTIP_WIDTH, - height: TOOLTIP_HEIGHT, - x: pos.x, - y: pos.y, - frame: false, - transparent: true, - focusable: false, - alwaysOnTop: true, - skipTaskbar: true, - resizable: false, - hasShadow: true, - type: 'panel', - webPreferences: { - preload: path.join(__dirname, 'preload.js'), - contextIsolation: true, - nodeIntegration: false, - sandbox: true, - }, - show: false, - }); - - suggestionWindow.loadURL(`http://localhost:${getServerPort()}/desktop/suggestion?t=${Date.now()}`); - - suggestionWindow.once('ready-to-show', () => { - suggestionWindow?.showInactive(); - }); - - suggestionWindow.webContents.setWindowOpenHandler(({ url }) => { - if (url.startsWith('http://localhost')) { - return { action: 'allow' }; - } - shell.openExternal(url); - return { action: 'deny' }; - }); - - suggestionWindow.on('closed', () => { - suggestionWindow = null; - }); - - return suggestionWindow; -} - function clearDebounce(): void { if (debounceTimer) { clearTimeout(debounceTimer); @@ -128,10 +59,24 @@ function isSurfSenseWindow(): boolean { return app === 'Electron' || app === 'SurfSense' || app === 'surfsense-desktop'; } -function onKeyDown(event: { keycode: number; ctrlKey?: boolean; metaKey?: boolean; altKey?: boolean }): void { +function onKeyDown(event: { + keycode: number; + shiftKey?: boolean; + ctrlKey?: boolean; + metaKey?: boolean; + altKey?: boolean; +}): void { if (!autocompleteEnabled) return; - if (event.keycode === UiohookKey.Tab && suggestionWindow && !suggestionWindow.isDestroyed()) { + const currentApp = getFrontmostApp(); + if (currentApp !== getLastTrackedApp()) { + resetBuffer(); + setLastTrackedApp(currentApp); + } + + const win = getSuggestionWindow(); + + if (event.keycode === UiohookKey.Tab && win && !win.isDestroyed()) { if (pendingSuggestionText) { acceptAndInject(pendingSuggestionText); } @@ -139,7 +84,7 @@ function onKeyDown(event: { keycode: number; ctrlKey?: boolean; metaKey?: boolea } if (event.keycode === UiohookKey.Escape) { - if (suggestionWindow && !suggestionWindow.isDestroyed()) { + if (win && !win.isDestroyed()) { destroySuggestion(); pendingSuggestionText = ''; } @@ -147,11 +92,41 @@ function onKeyDown(event: { keycode: number; ctrlKey?: boolean; metaKey?: boolea return; } - if (IGNORED_KEYCODES.has(event.keycode)) return; - if (event.ctrlKey || event.metaKey || event.altKey) return; - if (isSurfSenseWindow()) return; + if (currentApp === 'Electron' || currentApp === 'SurfSense' || currentApp === 'surfsense-desktop') { + return; + } - if (suggestionWindow && !suggestionWindow.isDestroyed()) { + if (event.ctrlKey || event.metaKey || event.altKey) { + resetBuffer(); + clearDebounce(); + return; + } + + if (event.keycode === UiohookKey.Backspace) { + removeLastChar(); + } else if (event.keycode === UiohookKey.Delete) { + // forward delete doesn't affect our trailing buffer + } else if (event.keycode === UiohookKey.Enter) { + appendToBuffer('\n'); + } else if (event.keycode === UiohookKey.Space) { + appendToBuffer(' '); + } else if ( + event.keycode === UiohookKey.ArrowLeft || event.keycode === UiohookKey.ArrowRight || + event.keycode === UiohookKey.ArrowUp || event.keycode === UiohookKey.ArrowDown || + event.keycode === UiohookKey.Home || event.keycode === UiohookKey.End || + event.keycode === UiohookKey.PageUp || event.keycode === UiohookKey.PageDown + ) { + resetBuffer(); + clearDebounce(); + return; + } else if (IGNORED_KEYCODES.has(event.keycode)) { + return; + } else { + const ch = resolveChar(event.keycode, !!event.shiftKey); + if (ch) appendToBuffer(ch); + } + + if (win && !win.isDestroyed()) { destroySuggestion(); } @@ -161,13 +136,16 @@ function onKeyDown(event: { keycode: number; ctrlKey?: boolean; metaKey?: boolea }, DEBOUNCE_MS); } +function onMouseClick(): void { + resetBuffer(); +} + async function triggerAutocomplete(): Promise { if (!hasAccessibilityPermission()) return; if (isSurfSenseWindow()) return; - const fieldContent = getFieldContent(); - if (!fieldContent || !fieldContent.text.trim()) return; - if (fieldContent.text.trim().length < 5) return; + const text = getBufferTrimmed(); + if (!text || text.length < 5) return; sourceApp = getFrontmostApp(); savedClipboard = clipboard.readText(); @@ -186,13 +164,16 @@ async function triggerAutocomplete(): Promise { } win.webContents.once('did-finish-load', () => { - if (suggestionWindow && !suggestionWindow.isDestroyed()) { - suggestionWindow.webContents.send(IPC_CHANNELS.AUTOCOMPLETE_CONTEXT, { - text: fieldContent.text, - cursorPosition: fieldContent.cursorPosition, - searchSpaceId, - }); - } + const sw = getSuggestionWindow(); + setTimeout(() => { + if (sw && !sw.isDestroyed()) { + sw.webContents.send(IPC_CHANNELS.AUTOCOMPLETE_CONTEXT, { + text: getBuffer(), + cursorPosition: getBuffer().length, + searchSpaceId, + }); + } + }, 300); }); } @@ -209,6 +190,7 @@ async function acceptAndInject(text: string): Promise { simulatePaste(); await new Promise((r) => setTimeout(r, 100)); clipboard.writeText(savedClipboard); + appendToBuffer(text); } catch { clipboard.writeText(savedClipboard); } @@ -238,21 +220,16 @@ function registerIpcHandlers(): void { export function registerAutocomplete(): void { registerIpcHandlers(); - if (!allPermissionsGranted()) { - console.log('[autocomplete] Permissions not granted — hook not started'); - return; - } - if (!loadUiohook()) { console.error('[autocomplete] Cannot start: uiohook-napi failed to load'); return; } uIOhook.on('keydown', onKeyDown); + uIOhook.on('click', onMouseClick); try { uIOhook.start(); hookStarted = true; - console.log('[autocomplete] uIOhook started'); } catch (err) { console.error('[autocomplete] uIOhook.start() failed:', err); } diff --git a/surfsense_desktop/src/modules/autocomplete/keystroke-buffer.ts b/surfsense_desktop/src/modules/autocomplete/keystroke-buffer.ts new file mode 100644 index 000000000..ca232d307 --- /dev/null +++ b/surfsense_desktop/src/modules/autocomplete/keystroke-buffer.ts @@ -0,0 +1,76 @@ +const MAX_BUFFER_LENGTH = 4000; +const KEYCODE_TO_CHAR: Record = {}; + +let keystrokeBuffer = ''; +let lastTrackedApp = ''; + +export function buildKeycodeMap(): void { + const letters: [string, number][] = [ + ['q', 16], ['w', 17], ['e', 18], ['r', 19], ['t', 20], + ['y', 21], ['u', 22], ['i', 23], ['o', 24], ['p', 25], + ['a', 30], ['s', 31], ['d', 32], ['f', 33], ['g', 34], + ['h', 35], ['j', 36], ['k', 37], ['l', 38], + ['z', 44], ['x', 45], ['c', 46], ['v', 47], + ['b', 48], ['n', 49], ['m', 50], + ]; + for (const [ch, code] of letters) { + KEYCODE_TO_CHAR[code] = [ch, ch.toUpperCase()]; + } + + const digits: [string, string, number][] = [ + ['1', '!', 2], ['2', '@', 3], ['3', '#', 4], ['4', '$', 5], + ['5', '%', 6], ['6', '^', 7], ['7', '&', 8], ['8', '*', 9], + ['9', '(', 10], ['0', ')', 11], + ]; + for (const [norm, shifted, code] of digits) { + KEYCODE_TO_CHAR[code] = [norm, shifted]; + } + + const punctuation: [string, string, number][] = [ + [';', ':', 39], ['=', '+', 13], [',', '<', 51], ['-', '_', 12], + ['.', '>', 52], ['/', '?', 53], ['`', '~', 41], ['[', '{', 26], + ['\\', '|', 43], [']', '}', 27], ["'", '"', 40], + ]; + for (const [norm, shifted, code] of punctuation) { + KEYCODE_TO_CHAR[code] = [norm, shifted]; + } +} + +export function resetBuffer(): void { + keystrokeBuffer = ''; +} + +export function appendToBuffer(char: string): void { + keystrokeBuffer += char; + if (keystrokeBuffer.length > MAX_BUFFER_LENGTH) { + keystrokeBuffer = keystrokeBuffer.slice(-MAX_BUFFER_LENGTH); + } +} + +export function removeLastChar(): void { + if (keystrokeBuffer.length > 0) { + keystrokeBuffer = keystrokeBuffer.slice(0, -1); + } +} + +export function getBuffer(): string { + return keystrokeBuffer; +} + +export function getBufferTrimmed(): string { + return keystrokeBuffer.trim(); +} + +export function getLastTrackedApp(): string { + return lastTrackedApp; +} + +export function setLastTrackedApp(app: string): void { + lastTrackedApp = app; +} + +export function resolveChar(keycode: number, shift: boolean): string | null { + const mapping = KEYCODE_TO_CHAR[keycode]; + if (!mapping) return null; + return shift ? mapping[1] : mapping[0]; +} diff --git a/surfsense_desktop/src/modules/autocomplete/suggestion-window.ts b/surfsense_desktop/src/modules/autocomplete/suggestion-window.ts new file mode 100644 index 000000000..f03930cf6 --- /dev/null +++ b/surfsense_desktop/src/modules/autocomplete/suggestion-window.ts @@ -0,0 +1,103 @@ +import { BrowserWindow, screen, shell } from 'electron'; +import path from 'path'; +import { getServerPort } from '../server'; + +const TOOLTIP_WIDTH = 420; +const TOOLTIP_HEIGHT = 38; +const MAX_HEIGHT = 400; + +let suggestionWindow: BrowserWindow | null = null; +let resizeTimer: ReturnType | null = null; + +function clampToScreen(x: number, y: number, w: number, h: number): { x: number; y: number } { + const display = screen.getDisplayNearestPoint({ x, y }); + const { x: dx, y: dy, width: dw, height: dh } = display.workArea; + return { + x: Math.max(dx, Math.min(x, dx + dw - w)), + y: Math.max(dy, Math.min(y, dy + dh - h)), + }; +} + +function stopResizePolling(): void { + if (resizeTimer) { clearInterval(resizeTimer); resizeTimer = null; } +} + +function startResizePolling(win: BrowserWindow): void { + stopResizePolling(); + let lastH = 0; + resizeTimer = setInterval(async () => { + if (!win || win.isDestroyed()) { stopResizePolling(); return; } + try { + const h: number = await win.webContents.executeJavaScript( + `document.body.scrollHeight` + ); + if (h > 0 && h !== lastH) { + lastH = h; + const clamped = Math.min(h, MAX_HEIGHT); + const bounds = win.getBounds(); + win.setBounds({ x: bounds.x, y: bounds.y, width: TOOLTIP_WIDTH, height: clamped }); + } + } catch {} + }, 150); +} + +export function getSuggestionWindow(): BrowserWindow | null { + return suggestionWindow; +} + +export function destroySuggestion(): void { + stopResizePolling(); + if (suggestionWindow && !suggestionWindow.isDestroyed()) { + suggestionWindow.close(); + } + suggestionWindow = null; +} + +export function createSuggestionWindow(x: number, y: number): BrowserWindow { + destroySuggestion(); + + const pos = clampToScreen(x, y + 20, TOOLTIP_WIDTH, TOOLTIP_HEIGHT); + + suggestionWindow = new BrowserWindow({ + width: TOOLTIP_WIDTH, + height: TOOLTIP_HEIGHT, + x: pos.x, + y: pos.y, + frame: false, + transparent: true, + focusable: false, + alwaysOnTop: true, + skipTaskbar: true, + hasShadow: true, + type: 'panel', + webPreferences: { + preload: path.join(__dirname, '..', 'preload.js'), + contextIsolation: true, + nodeIntegration: false, + sandbox: true, + }, + show: false, + }); + + suggestionWindow.loadURL(`http://localhost:${getServerPort()}/desktop/suggestion?t=${Date.now()}`); + + suggestionWindow.once('ready-to-show', () => { + suggestionWindow?.showInactive(); + if (suggestionWindow) startResizePolling(suggestionWindow); + }); + + suggestionWindow.webContents.setWindowOpenHandler(({ url }) => { + if (url.startsWith('http://localhost')) { + return { action: 'allow' }; + } + shell.openExternal(url); + return { action: 'deny' }; + }); + + suggestionWindow.on('closed', () => { + stopResizePolling(); + suggestionWindow = null; + }); + + return suggestionWindow; +} diff --git a/surfsense_desktop/src/modules/platform.ts b/surfsense_desktop/src/modules/platform.ts index 262866d07..1ab0c38fb 100644 --- a/surfsense_desktop/src/modules/platform.ts +++ b/surfsense_desktop/src/modules/platform.ts @@ -19,20 +19,6 @@ export function getFrontmostApp(): string { return ''; } -export function getSelectedText(): string { - try { - if (process.platform === 'darwin') { - return execSync( - 'osascript -e \'tell application "System Events" to get value of attribute "AXSelectedText" of focused UI element of first application process whose frontmost is true\'' - ).toString().trim(); - } - // Windows: no reliable accessibility API for selected text across apps - } catch { - return ''; - } - return ''; -} - export function simulateCopy(): void { if (process.platform === 'darwin') { execSync('osascript -e \'tell application "System Events" to keystroke "c" using command down\''); @@ -58,38 +44,3 @@ export function hasAccessibilityPermission(): boolean { if (process.platform !== 'darwin') return true; return systemPreferences.isTrustedAccessibilityClient(false); } - -export interface FieldContent { - text: string; - cursorPosition: number; -} - -export function getFieldContent(): FieldContent | null { - if (process.platform !== 'darwin') return null; - - try { - const text = execSync( - 'osascript -e \'tell application "System Events" to get value of attribute "AXValue" of focused UI element of first application process whose frontmost is true\'', - { timeout: 500 } - ).toString().trim(); - - let cursorPosition = text.length; - try { - const rangeStr = execSync( - 'osascript -e \'tell application "System Events" to get value of attribute "AXSelectedTextRange" of focused UI element of first application process whose frontmost is true\'', - { timeout: 500 } - ).toString().trim(); - - const locationMatch = rangeStr.match(/location[:\s]*(\d+)/i); - if (locationMatch) { - cursorPosition = parseInt(locationMatch[1], 10); - } - } catch { - // Fall back to end of text - } - - return { text, cursorPosition }; - } catch { - return null; - } -} diff --git a/surfsense_web/app/desktop/permissions/page.tsx b/surfsense_web/app/desktop/permissions/page.tsx index 2bcdc42df..8bde63357 100644 --- a/surfsense_web/app/desktop/permissions/page.tsx +++ b/surfsense_web/app/desktop/permissions/page.tsx @@ -169,11 +169,14 @@ export default function DesktopPermissionsPage() { > Open System Settings - {status === "denied" && ( -

- Toggle SurfSense on in System Settings to continue. -

- )} + {status === "denied" && ( +

+ Toggle SurfSense on in System Settings to continue. +

+ )} +

+ If SurfSense doesn't appear in the list, click + and select it from Applications. +

)} diff --git a/surfsense_web/app/desktop/suggestion/page.tsx b/surfsense_web/app/desktop/suggestion/page.tsx index 14dfab3af..69a19e3f1 100644 --- a/surfsense_web/app/desktop/suggestion/page.tsx +++ b/surfsense_web/app/desktop/suggestion/page.tsx @@ -151,9 +151,9 @@ export default function SuggestionPage() {

{suggestion}

- Tab accept - · - Esc dismiss + Tab accept + + Esc dismiss
); diff --git a/surfsense_web/app/desktop/suggestion/suggestion.css b/surfsense_web/app/desktop/suggestion/suggestion.css index e9471e7f8..0d3332103 100644 --- a/surfsense_web/app/desktop/suggestion/suggestion.css +++ b/surfsense_web/app/desktop/suggestion/suggestion.css @@ -1,8 +1,16 @@ +html, body { + margin: 0 !important; + padding: 0 !important; + background: transparent !important; + overflow: hidden !important; + height: auto !important; + width: 100% !important; +} + .suggestion-body { margin: 0; padding: 0; background: transparent; - overflow: hidden; font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; -webkit-font-smoothing: antialiased; user-select: none; @@ -10,69 +18,73 @@ } .suggestion-tooltip { - background: rgba(30, 30, 30, 0.95); - backdrop-filter: blur(12px); - -webkit-backdrop-filter: blur(12px); - border: 1px solid rgba(255, 255, 255, 0.1); - border-radius: 10px; - padding: 10px 14px; + background: #1e1e1e; + border: 1px solid #3c3c3c; + border-radius: 8px; + padding: 8px 12px; margin: 4px; max-width: 400px; - box-shadow: 0 8px 32px rgba(0, 0, 0, 0.4), - 0 2px 8px rgba(0, 0, 0, 0.2); + box-shadow: 0 4px 16px rgba(0, 0, 0, 0.5); } .suggestion-text { - color: rgba(255, 255, 255, 0.9); + color: #d4d4d4; font-size: 13px; - line-height: 1.5; - margin: 0 0 8px 0; + line-height: 1.45; + margin: 0 0 6px 0; word-wrap: break-word; white-space: pre-wrap; } .suggestion-hint { - color: rgba(255, 255, 255, 0.4); + color: #666; font-size: 11px; display: flex; align-items: center; - gap: 4px; + gap: 6px; + border-top: 1px solid #2a2a2a; + padding-top: 6px; } -.suggestion-key { - background: rgba(255, 255, 255, 0.1); - border: 1px solid rgba(255, 255, 255, 0.15); +.suggestion-hint kbd { + background: #2a2a2a; + border: 1px solid #3c3c3c; border-radius: 3px; - padding: 1px 5px; + padding: 0 4px; + font-family: inherit; font-size: 10px; - font-weight: 500; - color: rgba(255, 255, 255, 0.6); + font-weight: 600; + color: #999; + line-height: 18px; } .suggestion-separator { - margin: 0 2px; + width: 1px; + height: 10px; + background: #333; } .suggestion-error { - border-color: rgba(255, 80, 80, 0.3); + border-color: #5c2626; } .suggestion-error-text { - color: rgba(255, 120, 120, 0.9); + color: #f48771; font-size: 12px; } .suggestion-loading { display: flex; - gap: 4px; - padding: 4px 0; + gap: 5px; + padding: 2px 0; + justify-content: center; } .suggestion-dot { - width: 5px; - height: 5px; + width: 4px; + height: 4px; border-radius: 50%; - background: rgba(255, 255, 255, 0.4); + background: #666; animation: suggestion-pulse 1.2s infinite ease-in-out; } @@ -91,6 +103,6 @@ } 40% { opacity: 1; - transform: scale(1); + transform: scale(1.1); } } From 3e68d4aa3ed04c87f155a4da08a6610251755f74 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 2 Apr 2026 20:38:09 +0200 Subject: [PATCH 09/42] refactor: extract autocomplete service and fix tooltip screen-edge positioning --- .../app/routes/autocomplete_routes.py | 109 +---------------- .../app/services/autocomplete_service.py | 110 ++++++++++++++++++ .../modules/autocomplete/suggestion-window.ts | 27 +++-- 3 files changed, 130 insertions(+), 116 deletions(-) create mode 100644 surfsense_backend/app/services/autocomplete_service.py diff --git a/surfsense_backend/app/routes/autocomplete_routes.py b/surfsense_backend/app/routes/autocomplete_routes.py index 9a285a723..68c56d0e0 100644 --- a/surfsense_backend/app/routes/autocomplete_routes.py +++ b/surfsense_backend/app/routes/autocomplete_routes.py @@ -1,118 +1,14 @@ -import logging -from typing import AsyncGenerator - from fastapi import APIRouter, Depends, Query from fastapi.responses import StreamingResponse -from langchain_core.messages import HumanMessage, SystemMessage from sqlalchemy.ext.asyncio import AsyncSession from app.db import User, get_async_session -from app.retriever.chunks_hybrid_search import ChucksHybridSearchRetriever -from app.services.llm_service import get_agent_llm +from app.services.autocomplete_service import stream_autocomplete from app.services.new_streaming_service import VercelStreamingService from app.users import current_active_user -logger = logging.getLogger(__name__) - router = APIRouter(prefix="/autocomplete", tags=["autocomplete"]) -AUTOCOMPLETE_SYSTEM_PROMPT = """You are an inline text autocomplete engine. Your job is to complete the user's text naturally. - -Rules: -- Output ONLY the continuation text. Do NOT repeat what the user already typed. -- Keep completions concise: 1-3 sentences maximum. -- Match the user's tone, style, and language. -- If knowledge base context is provided, use it to make the completion factually accurate and personalized. -- Do NOT add quotes, explanations, or meta-commentary. -- Do NOT start with a space unless grammatically required. -- If you cannot produce a useful completion, output nothing.""" - -KB_CONTEXT_TEMPLATE = """ -Relevant knowledge base context (use this to personalize the completion): ---- -{kb_context} ---- -""" - - -async def _stream_autocomplete( - text: str, - cursor_position: int, - search_space_id: int, - session: AsyncSession, -) -> AsyncGenerator[str, None]: - """Stream an autocomplete response with KB context.""" - streaming_service = VercelStreamingService() - - try: - # Text before cursor is what we're completing - text_before_cursor = text[:cursor_position] if cursor_position >= 0 else text - - if not text_before_cursor.strip(): - yield streaming_service.format_message_start() - yield streaming_service.format_finish() - yield streaming_service.format_done() - return - - # Fast KB lookup: vector-only search, top 3 chunks, no planner LLM - kb_context = "" - try: - retriever = ChucksHybridSearchRetriever(session) - chunks = await retriever.vector_search( - query_text=text_before_cursor[-200:], # last 200 chars for relevance - top_k=3, - search_space_id=search_space_id, - ) - if chunks: - kb_snippets = [] - for chunk in chunks: - content = getattr(chunk, "content", None) or getattr(chunk, "chunk_text", "") - if content: - kb_snippets.append(content[:300]) - if kb_snippets: - kb_context = KB_CONTEXT_TEMPLATE.format( - kb_context="\n\n".join(kb_snippets) - ) - except Exception as e: - logger.warning(f"KB search failed for autocomplete, proceeding without context: {e}") - - # Get the search space's configured LLM - llm = await get_agent_llm(session, search_space_id) - if not llm: - yield streaming_service.format_message_start() - error_msg = "No LLM configured for this search space" - yield streaming_service.format_error(error_msg) - yield streaming_service.format_done() - return - - system_prompt = AUTOCOMPLETE_SYSTEM_PROMPT - if kb_context: - system_prompt += kb_context - - messages = [ - SystemMessage(content=system_prompt), - HumanMessage(content=f"Complete this text:\n{text_before_cursor}"), - ] - - # Stream the response - yield streaming_service.format_message_start() - text_id = streaming_service.generate_text_id() - yield streaming_service.format_text_start(text_id) - - async for chunk in llm.astream(messages): - token = chunk.content if hasattr(chunk, "content") else str(chunk) - if token: - yield streaming_service.format_text_delta(text_id, token) - - yield streaming_service.format_text_end(text_id) - yield streaming_service.format_finish() - yield streaming_service.format_done() - - except Exception as e: - logger.error(f"Autocomplete streaming error: {e}") - yield streaming_service.format_error(str(e)) - yield streaming_service.format_done() - @router.post("/stream") async def autocomplete_stream( @@ -122,12 +18,11 @@ async def autocomplete_stream( user: User = Depends(current_active_user), session: AsyncSession = Depends(get_async_session), ): - """Stream an autocomplete suggestion based on the current text and KB context.""" if cursor_position < 0: cursor_position = len(text) return StreamingResponse( - _stream_autocomplete(text, cursor_position, search_space_id, session), + stream_autocomplete(text, cursor_position, search_space_id, session), media_type="text/event-stream", headers={ **VercelStreamingService.get_response_headers(), diff --git a/surfsense_backend/app/services/autocomplete_service.py b/surfsense_backend/app/services/autocomplete_service.py new file mode 100644 index 000000000..7c172275d --- /dev/null +++ b/surfsense_backend/app/services/autocomplete_service.py @@ -0,0 +1,110 @@ +import logging +from typing import AsyncGenerator + +from langchain_core.messages import HumanMessage, SystemMessage +from sqlalchemy.ext.asyncio import AsyncSession + +from app.retriever.chunks_hybrid_search import ChucksHybridSearchRetriever +from app.services.llm_service import get_agent_llm +from app.services.new_streaming_service import VercelStreamingService + +logger = logging.getLogger(__name__) + +SYSTEM_PROMPT = """You are an inline text autocomplete engine. Your job is to complete the user's text naturally. + +Rules: +- Output ONLY the continuation text. Do NOT repeat what the user already typed. +- Keep completions concise: 1-3 sentences maximum. +- Match the user's tone, style, and language. +- If knowledge base context is provided, use it to make the completion factually accurate and personalized. +- Do NOT add quotes, explanations, or meta-commentary. +- Do NOT start with a space unless grammatically required. +- If you cannot produce a useful completion, output nothing.""" + +KB_CONTEXT_TEMPLATE = """ +Relevant knowledge base context (use this to personalize the completion): +--- +{kb_context} +--- +""" + + +async def _retrieve_kb_context( + session: AsyncSession, + text: str, + search_space_id: int, +) -> str: + try: + retriever = ChucksHybridSearchRetriever(session) + chunks = await retriever.vector_search( + query_text=text[-200:], + top_k=3, + search_space_id=search_space_id, + ) + if not chunks: + return "" + snippets = [] + for chunk in chunks: + content = getattr(chunk, "content", None) or getattr(chunk, "chunk_text", "") + if content: + snippets.append(content[:300]) + if not snippets: + return "" + return KB_CONTEXT_TEMPLATE.format(kb_context="\n\n".join(snippets)) + except Exception as e: + logger.warning(f"KB search failed for autocomplete, proceeding without context: {e}") + return "" + + +async def stream_autocomplete( + text: str, + cursor_position: int, + search_space_id: int, + session: AsyncSession, +) -> AsyncGenerator[str, None]: + """Build context, call the LLM, and yield SSE-formatted tokens.""" + streaming = VercelStreamingService() + text_before_cursor = text[:cursor_position] if cursor_position >= 0 else text + + if not text_before_cursor.strip(): + yield streaming.format_message_start() + yield streaming.format_finish() + yield streaming.format_done() + return + + kb_context = await _retrieve_kb_context(session, text_before_cursor, search_space_id) + + llm = await get_agent_llm(session, search_space_id) + if not llm: + yield streaming.format_message_start() + yield streaming.format_error("No LLM configured for this search space") + yield streaming.format_done() + return + + system_prompt = SYSTEM_PROMPT + if kb_context: + system_prompt += kb_context + + messages = [ + SystemMessage(content=system_prompt), + HumanMessage(content=f"Complete this text:\n{text_before_cursor}"), + ] + + try: + yield streaming.format_message_start() + text_id = streaming.generate_text_id() + yield streaming.format_text_start(text_id) + + async for chunk in llm.astream(messages): + token = chunk.content if hasattr(chunk, "content") else str(chunk) + if token: + yield streaming.format_text_delta(text_id, token) + + yield streaming.format_text_end(text_id) + yield streaming.format_finish() + yield streaming.format_done() + + except Exception as e: + logger.error(f"Autocomplete streaming error: {e}") + yield streaming.format_error(str(e)) + yield streaming.format_done() diff --git a/surfsense_desktop/src/modules/autocomplete/suggestion-window.ts b/surfsense_desktop/src/modules/autocomplete/suggestion-window.ts index f03930cf6..e8a2f3a91 100644 --- a/surfsense_desktop/src/modules/autocomplete/suggestion-window.ts +++ b/surfsense_desktop/src/modules/autocomplete/suggestion-window.ts @@ -8,14 +8,22 @@ const MAX_HEIGHT = 400; let suggestionWindow: BrowserWindow | null = null; let resizeTimer: ReturnType | null = null; +let cursorOrigin = { x: 0, y: 0 }; -function clampToScreen(x: number, y: number, w: number, h: number): { x: number; y: number } { - const display = screen.getDisplayNearestPoint({ x, y }); +const CURSOR_GAP = 20; + +function positionOnScreen(cursorX: number, cursorY: number, w: number, h: number): { x: number; y: number } { + const display = screen.getDisplayNearestPoint({ x: cursorX, y: cursorY }); const { x: dx, y: dy, width: dw, height: dh } = display.workArea; - return { - x: Math.max(dx, Math.min(x, dx + dw - w)), - y: Math.max(dy, Math.min(y, dy + dh - h)), - }; + + const x = Math.max(dx, Math.min(cursorX, dx + dw - w)); + + const spaceBelow = (dy + dh) - (cursorY + CURSOR_GAP); + const y = spaceBelow >= h + ? cursorY + CURSOR_GAP + : cursorY - h - CURSOR_GAP; + + return { x, y: Math.max(dy, y) }; } function stopResizePolling(): void { @@ -34,8 +42,8 @@ function startResizePolling(win: BrowserWindow): void { if (h > 0 && h !== lastH) { lastH = h; const clamped = Math.min(h, MAX_HEIGHT); - const bounds = win.getBounds(); - win.setBounds({ x: bounds.x, y: bounds.y, width: TOOLTIP_WIDTH, height: clamped }); + const pos = positionOnScreen(cursorOrigin.x, cursorOrigin.y, TOOLTIP_WIDTH, clamped); + win.setBounds({ x: pos.x, y: pos.y, width: TOOLTIP_WIDTH, height: clamped }); } } catch {} }, 150); @@ -55,8 +63,9 @@ export function destroySuggestion(): void { export function createSuggestionWindow(x: number, y: number): BrowserWindow { destroySuggestion(); + cursorOrigin = { x, y }; - const pos = clampToScreen(x, y + 20, TOOLTIP_WIDTH, TOOLTIP_HEIGHT); + const pos = positionOnScreen(x, y, TOOLTIP_WIDTH, TOOLTIP_HEIGHT); suggestionWindow = new BrowserWindow({ width: TOOLTIP_WIDTH, From a99d999a3658c09ce133940cc0dd15a3353d6cd7 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 2 Apr 2026 21:29:05 +0200 Subject: [PATCH 10/42] fix: correct preload.js path after autocomplete module restructure --- surfsense_desktop/src/modules/autocomplete/suggestion-window.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/surfsense_desktop/src/modules/autocomplete/suggestion-window.ts b/surfsense_desktop/src/modules/autocomplete/suggestion-window.ts index e8a2f3a91..8f61b2901 100644 --- a/surfsense_desktop/src/modules/autocomplete/suggestion-window.ts +++ b/surfsense_desktop/src/modules/autocomplete/suggestion-window.ts @@ -80,7 +80,7 @@ export function createSuggestionWindow(x: number, y: number): BrowserWindow { hasShadow: true, type: 'panel', webPreferences: { - preload: path.join(__dirname, '..', 'preload.js'), + preload: path.join(__dirname, 'preload.js'), contextIsolation: true, nodeIntegration: false, sandbox: true, From 3621951f2aeceebaff70328ffe8a6c91e9bad83d Mon Sep 17 00:00:00 2001 From: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> Date: Thu, 2 Apr 2026 23:21:57 -0700 Subject: [PATCH 11/42] perf: throttle scroll handlers with requestAnimationFrame Wrap scroll handlers in thread.tsx, InboxSidebar.tsx, and DocumentsTableShell.tsx with requestAnimationFrame batching so scroll position state updates fire at most once per animation frame instead of on every scroll event (up to 60/sec at 60fps). Add cleanup useEffect to cancel pending frames on unmount. Fixes #1103 --- .../(manage)/components/DocumentsTableShell.tsx | 12 +++++++++--- surfsense_web/components/assistant-ui/thread.tsx | 12 +++++++++--- .../components/layout/ui/sidebar/InboxSidebar.tsx | 12 +++++++++--- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsTableShell.tsx b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsTableShell.tsx index ceef9f2e1..dc8966571 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsTableShell.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsTableShell.tsx @@ -267,12 +267,18 @@ export function DocumentsTableShell({ const [metadataJson, setMetadataJson] = useState | null>(null); const [metadataLoading, setMetadataLoading] = useState(false); const [previewScrollPos, setPreviewScrollPos] = useState<"top" | "middle" | "bottom">("top"); + const previewRafRef = useRef(); const handlePreviewScroll = useCallback((e: React.UIEvent) => { const el = e.currentTarget; - const atTop = el.scrollTop <= 2; - const atBottom = el.scrollHeight - el.scrollTop - el.clientHeight <= 2; - setPreviewScrollPos(atTop ? "top" : atBottom ? "bottom" : "middle"); + if (previewRafRef.current) return; + previewRafRef.current = requestAnimationFrame(() => { + const atTop = el.scrollTop <= 2; + const atBottom = el.scrollHeight - el.scrollTop - el.clientHeight <= 2; + setPreviewScrollPos(atTop ? "top" : atBottom ? "bottom" : "middle"); + previewRafRef.current = undefined; + }); }, []); + useEffect(() => () => { if (previewRafRef.current) cancelAnimationFrame(previewRafRef.current); }, []); const [deleteDoc, setDeleteDoc] = useState(null); const [isDeleting, setIsDeleting] = useState(false); diff --git a/surfsense_web/components/assistant-ui/thread.tsx b/surfsense_web/components/assistant-ui/thread.tsx index 0d0163d8a..0f230cec3 100644 --- a/surfsense_web/components/assistant-ui/thread.tsx +++ b/surfsense_web/components/assistant-ui/thread.tsx @@ -816,12 +816,18 @@ const ComposerAction: FC = ({ isBlockedByOtherUser = false const isDesktop = useMediaQuery("(min-width: 640px)"); const { openDialog: openUploadDialog } = useDocumentUploadDialog(); const [toolsScrollPos, setToolsScrollPos] = useState<"top" | "middle" | "bottom">("top"); + const toolsRafRef = useRef(); const handleToolsScroll = useCallback((e: React.UIEvent) => { const el = e.currentTarget; - const atTop = el.scrollTop <= 2; - const atBottom = el.scrollHeight - el.scrollTop - el.clientHeight <= 2; - setToolsScrollPos(atTop ? "top" : atBottom ? "bottom" : "middle"); + if (toolsRafRef.current) return; + toolsRafRef.current = requestAnimationFrame(() => { + const atTop = el.scrollTop <= 2; + const atBottom = el.scrollHeight - el.scrollTop - el.clientHeight <= 2; + setToolsScrollPos(atTop ? "top" : atBottom ? "bottom" : "middle"); + toolsRafRef.current = undefined; + }); }, []); + useEffect(() => () => { if (toolsRafRef.current) cancelAnimationFrame(toolsRafRef.current); }, []); const isComposerTextEmpty = useAuiState(({ composer }) => { const text = composer.text?.trim() || ""; return text.length === 0; diff --git a/surfsense_web/components/layout/ui/sidebar/InboxSidebar.tsx b/surfsense_web/components/layout/ui/sidebar/InboxSidebar.tsx index 72400a589..4aa8d4c60 100644 --- a/surfsense_web/components/layout/ui/sidebar/InboxSidebar.tsx +++ b/surfsense_web/components/layout/ui/sidebar/InboxSidebar.tsx @@ -178,12 +178,18 @@ export function InboxSidebarContent({ const [mounted, setMounted] = useState(false); const [openDropdown, setOpenDropdown] = useState<"filter" | null>(null); const [connectorScrollPos, setConnectorScrollPos] = useState<"top" | "middle" | "bottom">("top"); + const connectorRafRef = useRef(); const handleConnectorScroll = useCallback((e: React.UIEvent) => { const el = e.currentTarget; - const atTop = el.scrollTop <= 2; - const atBottom = el.scrollHeight - el.scrollTop - el.clientHeight <= 2; - setConnectorScrollPos(atTop ? "top" : atBottom ? "bottom" : "middle"); + if (connectorRafRef.current) return; + connectorRafRef.current = requestAnimationFrame(() => { + const atTop = el.scrollTop <= 2; + const atBottom = el.scrollHeight - el.scrollTop - el.clientHeight <= 2; + setConnectorScrollPos(atTop ? "top" : atBottom ? "bottom" : "middle"); + connectorRafRef.current = undefined; + }); }, []); + useEffect(() => () => { if (connectorRafRef.current) cancelAnimationFrame(connectorRafRef.current); }, []); const [filterDrawerOpen, setFilterDrawerOpen] = useState(false); const [markingAsReadId, setMarkingAsReadId] = useState(null); From e38a0ff7c345cb83121f7983eec838cfdf579f66 Mon Sep 17 00:00:00 2001 From: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> Date: Thu, 2 Apr 2026 23:43:19 -0700 Subject: [PATCH 12/42] style: format useEffect cleanup to satisfy biome --- .../documents/(manage)/components/DocumentsTableShell.tsx | 7 ++++++- surfsense_web/components/assistant-ui/thread.tsx | 7 ++++++- .../components/layout/ui/sidebar/InboxSidebar.tsx | 7 ++++++- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsTableShell.tsx b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsTableShell.tsx index dc8966571..748fb1911 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsTableShell.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsTableShell.tsx @@ -278,7 +278,12 @@ export function DocumentsTableShell({ previewRafRef.current = undefined; }); }, []); - useEffect(() => () => { if (previewRafRef.current) cancelAnimationFrame(previewRafRef.current); }, []); + useEffect( + () => () => { + if (previewRafRef.current) cancelAnimationFrame(previewRafRef.current); + }, + [] + ); const [deleteDoc, setDeleteDoc] = useState(null); const [isDeleting, setIsDeleting] = useState(false); diff --git a/surfsense_web/components/assistant-ui/thread.tsx b/surfsense_web/components/assistant-ui/thread.tsx index 0f230cec3..718bf3961 100644 --- a/surfsense_web/components/assistant-ui/thread.tsx +++ b/surfsense_web/components/assistant-ui/thread.tsx @@ -827,7 +827,12 @@ const ComposerAction: FC = ({ isBlockedByOtherUser = false toolsRafRef.current = undefined; }); }, []); - useEffect(() => () => { if (toolsRafRef.current) cancelAnimationFrame(toolsRafRef.current); }, []); + useEffect( + () => () => { + if (toolsRafRef.current) cancelAnimationFrame(toolsRafRef.current); + }, + [] + ); const isComposerTextEmpty = useAuiState(({ composer }) => { const text = composer.text?.trim() || ""; return text.length === 0; diff --git a/surfsense_web/components/layout/ui/sidebar/InboxSidebar.tsx b/surfsense_web/components/layout/ui/sidebar/InboxSidebar.tsx index 4aa8d4c60..525b7cf74 100644 --- a/surfsense_web/components/layout/ui/sidebar/InboxSidebar.tsx +++ b/surfsense_web/components/layout/ui/sidebar/InboxSidebar.tsx @@ -189,7 +189,12 @@ export function InboxSidebarContent({ connectorRafRef.current = undefined; }); }, []); - useEffect(() => () => { if (connectorRafRef.current) cancelAnimationFrame(connectorRafRef.current); }, []); + useEffect( + () => () => { + if (connectorRafRef.current) cancelAnimationFrame(connectorRafRef.current); + }, + [] + ); const [filterDrawerOpen, setFilterDrawerOpen] = useState(false); const [markingAsReadId, setMarkingAsReadId] = useState(null); From b9b2bac16f89203e16b637ab12e3edb5ef3b4589 Mon Sep 17 00:00:00 2001 From: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> Date: Thu, 2 Apr 2026 23:59:15 -0700 Subject: [PATCH 13/42] fix: clean up onboarding tour timer leaks Fix two timer cleanup bugs in onboarding-tour.tsx: 1. Remove cleanup return from useCallback (only works in useEffect). Clear retryTimerRef at the start of updateTarget and in a dedicated useEffect cleanup instead. 2. Track recursive setTimeout calls via startCheckTimerRef so they are properly cancelled on unmount instead of leaking. Fixes #1091 --- surfsense_web/components/onboarding-tour.tsx | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/surfsense_web/components/onboarding-tour.tsx b/surfsense_web/components/onboarding-tour.tsx index 1c52169cb..d762d9c15 100644 --- a/surfsense_web/components/onboarding-tour.tsx +++ b/surfsense_web/components/onboarding-tour.tsx @@ -429,6 +429,7 @@ export function OnboardingTour() { const pathname = usePathname(); const retryCountRef = useRef(0); const retryTimerRef = useRef | null>(null); + const startCheckTimerRef = useRef | null>(null); const maxRetries = 10; // Track previous user ID to detect user changes const previousUserIdRef = useRef(null); @@ -460,6 +461,7 @@ export function OnboardingTour() { // Find and track target element with retry logic const updateTarget = useCallback(() => { + if (retryTimerRef.current) clearTimeout(retryTimerRef.current); if (!currentStep) return; const el = document.querySelector(currentStep.target); @@ -480,11 +482,13 @@ export function OnboardingTour() { } }, 200); } + }, [currentStep]); + useEffect(() => { return () => { if (retryTimerRef.current) clearTimeout(retryTimerRef.current); }; - }, [currentStep]); + }, []); // Check if tour should run: localStorage + data validation with user ID tracking useEffect(() => { @@ -573,15 +577,15 @@ export function OnboardingTour() { setPosition(calculatePosition(connectorEl, TOUR_STEPS[0].placement)); } else { // Retry after delay - setTimeout(checkAndStartTour, 200); + startCheckTimerRef.current = setTimeout(checkAndStartTour, 200); } }; // Start checking after initial delay - const timer = setTimeout(checkAndStartTour, 500); + startCheckTimerRef.current = setTimeout(checkAndStartTour, 500); return () => { cancelled = true; - clearTimeout(timer); + if (startCheckTimerRef.current) clearTimeout(startCheckTimerRef.current); }; }, [mounted, user?.id, searchSpaceId, pathname, threadsData, documentTypeCounts, connectors]); From 134beec3920c9c2cc2c84e3588828b8294d856c9 Mon Sep 17 00:00:00 2001 From: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> Date: Fri, 3 Apr 2026 00:05:06 -0700 Subject: [PATCH 14/42] fix: clear upload progress interval on unmount Store the progress setInterval ID in a ref and clear it in a useEffect cleanup. Previously the interval was stored in a local variable and only cleared in onSuccess/onError callbacks, leaking if the component unmounted mid-upload. Fixes #1090 --- .../components/sources/DocumentUploadTab.tsx | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/surfsense_web/components/sources/DocumentUploadTab.tsx b/surfsense_web/components/sources/DocumentUploadTab.tsx index 723a3ad36..5c8ec83a5 100644 --- a/surfsense_web/components/sources/DocumentUploadTab.tsx +++ b/surfsense_web/components/sources/DocumentUploadTab.tsx @@ -4,7 +4,7 @@ import { useAtom } from "jotai"; import { CheckCircle2, FileType, FolderOpen, Info, Upload, X } from "lucide-react"; import { useTranslations } from "next-intl"; -import { type ChangeEvent, useCallback, useMemo, useRef, useState } from "react"; +import { type ChangeEvent, useCallback, useEffect, useMemo, useRef, useState } from "react"; import { useDropzone } from "react-dropzone"; import { toast } from "sonner"; import { uploadDocumentMutationAtom } from "@/atoms/documents/document-mutation.atoms"; @@ -132,6 +132,15 @@ export function DocumentUploadTab({ const { mutate: uploadDocuments, isPending: isUploading } = uploadDocumentMutation; const fileInputRef = useRef(null); const folderInputRef = useRef(null); + const progressIntervalRef = useRef | null>(null); + + useEffect(() => { + return () => { + if (progressIntervalRef.current) { + clearInterval(progressIntervalRef.current); + } + }; + }, []); const acceptedFileTypes = useMemo(() => { const etlService = process.env.NEXT_PUBLIC_ETL_SERVICE; @@ -236,7 +245,7 @@ export function DocumentUploadTab({ setUploadProgress(0); trackDocumentUploadStarted(Number(searchSpaceId), files.length, totalFileSize); - const progressInterval = setInterval(() => { + progressIntervalRef.current = setInterval(() => { setUploadProgress((prev) => (prev >= 90 ? prev : prev + Math.random() * 10)); }, 200); @@ -249,14 +258,14 @@ export function DocumentUploadTab({ }, { onSuccess: () => { - clearInterval(progressInterval); + if (progressIntervalRef.current) clearInterval(progressIntervalRef.current); setUploadProgress(100); trackDocumentUploadSuccess(Number(searchSpaceId), files.length); toast(t("upload_initiated"), { description: t("upload_initiated_desc") }); onSuccess?.(); }, onError: (error: unknown) => { - clearInterval(progressInterval); + if (progressIntervalRef.current) clearInterval(progressIntervalRef.current); setUploadProgress(0); const message = error instanceof Error ? error.message : "Upload failed"; trackDocumentUploadFailure(Number(searchSpaceId), message); From fc84dcffb05f794ca19641253ad8cab68b85d43f Mon Sep 17 00:00:00 2001 From: okxint Date: Fri, 3 Apr 2026 13:59:14 +0530 Subject: [PATCH 15/42] fix: memoize formatRelativeTime in thread list to prevent unnecessary re-renders --- surfsense_web/components/assistant-ui/thread-list.tsx | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/surfsense_web/components/assistant-ui/thread-list.tsx b/surfsense_web/components/assistant-ui/thread-list.tsx index f1d10ca16..e8b8db6fe 100644 --- a/surfsense_web/components/assistant-ui/thread-list.tsx +++ b/surfsense_web/components/assistant-ui/thread-list.tsx @@ -9,7 +9,7 @@ import { TrashIcon, } from "lucide-react"; import { useRouter } from "next/navigation"; -import { memo, useCallback, useEffect, useState } from "react"; +import { memo, useCallback, useEffect, useMemo, useState } from "react"; import { Button } from "@/components/ui/button"; import { DropdownMenu, @@ -224,6 +224,11 @@ const ThreadListItemComponent = memo(function ThreadListItemComponent({ onUnarchive, onDelete, }: ThreadListItemComponentProps) { + const relativeTime = useMemo( + () => formatRelativeTime(new Date(thread.updatedAt)), + [thread.updatedAt] + ); + return ( , document.body )} diff --git a/surfsense_web/components/homepage/use-cases-grid.tsx b/surfsense_web/components/homepage/use-cases-grid.tsx index 2f8c2d537..f9d315b49 100644 --- a/surfsense_web/components/homepage/use-cases-grid.tsx +++ b/surfsense_web/components/homepage/use-cases-grid.tsx @@ -1,4 +1,5 @@ "use client"; +import Image from 'next/image'; import { AnimatePresence, motion } from "motion/react"; import { ExpandedGifOverlay, useExpandedGif } from "@/components/ui/expanded-gif-overlay"; @@ -81,6 +82,15 @@ function UseCaseCard({ alt={title} className="w-full rounded-xl object-cover transition-transform duration-500 group-hover:scale-[1.02]" /> +
+ {title} +

{title}

diff --git a/surfsense_web/components/markdown-viewer.tsx b/surfsense_web/components/markdown-viewer.tsx index a568bd698..1c39f03a0 100644 --- a/surfsense_web/components/markdown-viewer.tsx +++ b/surfsense_web/components/markdown-viewer.tsx @@ -3,6 +3,8 @@ import { createMathPlugin } from "@streamdown/math"; import { Streamdown, type StreamdownProps } from "streamdown"; import "katex/dist/katex.min.css"; import { cn } from "@/lib/utils"; +import Image from 'next/image'; +import { is } from "drizzle-orm"; const code = createCodePlugin({ themes: ["nord", "nord"], @@ -127,16 +129,31 @@ export function MarkdownViewer({ content, className, maxLength }: MarkdownViewer
), hr: ({ ...props }) =>
, - img: ({ src, alt, width: _w, height: _h, ...props }) => ( - // eslint-disable-next-line @next/next/no-img-element - {alt - ), + img: ({ src, alt, width: _w, height: _h, ...props }) => { + const isDataOrUnknownUrl = typeof src === "string" && (src.startsWith("data:") || !src.startsWith("http")); + + return isDataOrUnknownUrl ? ( + // eslint-disable-next-line @next/next/no-img-element + {alt + ) : ( + {alt + ); +}, table: ({ ...props }) => (
diff --git a/surfsense_web/components/tool-ui/citation/citation-list.tsx b/surfsense_web/components/tool-ui/citation/citation-list.tsx index 3151917b6..75b02bf3d 100644 --- a/surfsense_web/components/tool-ui/citation/citation-list.tsx +++ b/surfsense_web/components/tool-ui/citation/citation-list.tsx @@ -7,6 +7,8 @@ import { openSafeNavigationHref, resolveSafeNavigationHref } from "../shared/med import { cn, Popover, PopoverContent, PopoverTrigger } from "./_adapter"; import { Citation } from "./citation"; import type { CitationType, CitationVariant, SerializableCitation } from "./schema"; +import NextImage from 'next/image'; + const TYPE_ICONS: Record = { webpage: Globe, @@ -253,18 +255,18 @@ function OverflowItem({ citation, onClick }: OverflowItemProps) { className="group hover:bg-muted focus-visible:bg-muted flex w-full cursor-pointer items-center gap-2.5 rounded-md px-2 py-2 text-left transition-colors focus-visible:outline-none" > {citation.favicon ? ( - // biome-ignore lint/performance/noImgElement: external favicon from arbitrary domain — next/image requires remotePatterns config - - ) : ( -