diff --git a/README.es.md b/README.es.md
index 299c6e95c..dea86a793 100644
--- a/README.es.md
+++ b/README.es.md
@@ -41,7 +41,7 @@ NotebookLM es una de las mejores y más útiles plataformas de IA que existen, p
- **Sin Dependencia de Proveedores** - Configura cualquier modelo LLM, de imagen, TTS y STT.
- **25+ Fuentes de Datos Externas** - Agrega tus fuentes desde Google Drive, OneDrive, Dropbox, Notion y muchos otros servicios externos.
- **Soporte Multijugador en Tiempo Real** - Trabaja fácilmente con los miembros de tu equipo en un notebook compartido.
-- **Aplicación de Escritorio** - Obtén asistencia de IA en cualquier aplicación con Quick Assist, General Assist, Extreme Assist y sincronización de carpetas locales.
+- **Aplicación de Escritorio** - Obtén asistencia de IA en cualquier aplicación con Quick Assist, General Assist, Screenshot Assist y sincronización de carpetas locales.
...y más por venir.
@@ -84,9 +84,9 @@ https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7
- - Aplicación de Escritorio — Extreme Assist
+ - Aplicación de Escritorio — Screenshot Assist
-
+
- Aplicación de Escritorio — Watch Local Folder
@@ -150,7 +150,7 @@ La aplicación de escritorio incluye estas potentes funciones:
- **General Assist** — Lanza SurfSense al instante desde cualquier aplicación con un atajo global.
- **Quick Assist** — Selecciona texto en cualquier lugar, luego pide a la IA que lo explique, reescriba o actúe sobre él.
-- **Extreme Assist** — Obtén sugerencias de escritura en línea impulsadas por tu base de conocimiento mientras escribes en cualquier aplicación.
+- **Screenshot Assist** — Selecciona una región de tu pantalla y adjúntala al chat para que las respuestas se basen en tu base de conocimiento.
- **Watch Local Folder** — Vigila una carpeta local y sincroniza automáticamente los cambios de archivos con tu base de conocimiento. **Pro tip:** Apúntalo a tu bóveda de Obsidian para mantener tus notas buscables en SurfSense.
Todas las funciones operan contra tu espacio de búsqueda elegido, por lo que tus respuestas siempre están basadas en tus propios datos.
@@ -199,14 +199,14 @@ Todas las funciones operan contra tu espacio de búsqueda elegido, por lo que tu
| **Generación de Videos** | Resúmenes en video cinemáticos vía Veo 3 (solo Ultra) | Disponible (NotebookLM es mejor aquí, mejorando activamente) |
| **Generación de Presentaciones** | Diapositivas más atractivas pero no editables | Crea presentaciones editables basadas en diapositivas |
| **Generación de Podcasts** | Resúmenes de audio con hosts e idiomas personalizables | Disponible con múltiples proveedores TTS (NotebookLM es mejor aquí, mejorando activamente) |
-| **Aplicación de Escritorio** | No | Aplicación nativa con General Assist, Quick Assist, Extreme Assist y sincronización de carpetas locales |
+| **Aplicación de Escritorio** | No | Aplicación nativa con General Assist, Quick Assist, Screenshot Assist y sincronización de carpetas locales |
| **Extensión de Navegador** | No | Extensión multi-navegador para guardar cualquier página web, incluyendo páginas protegidas por autenticación |
Lista completa de Fuentes Externas
-Motores de Búsqueda (Tavily, LinkUp) · SearxNG · Google Drive · OneDrive · Dropbox · Slack · Microsoft Teams · Linear · Jira · ClickUp · Confluence · BookStack · Notion · Gmail · Videos de YouTube · GitHub · Discord · Airtable · Google Calendar · Luma · Circleback · Elasticsearch · Obsidian, y más por venir.
+Motores de Búsqueda (SearXNG, Tavily, LinkUp, Baidu Search) · Google Drive · OneDrive · Dropbox · Slack · Microsoft Teams · Linear · Jira · ClickUp · Confluence · BookStack · Notion · Gmail · Videos de YouTube · GitHub · Discord · Airtable · Google Calendar · Luma · Circleback · Elasticsearch · Obsidian, y más por venir.
diff --git a/README.hi.md b/README.hi.md
index 11a25ee0d..43e24c3ee 100644
--- a/README.hi.md
+++ b/README.hi.md
@@ -41,7 +41,7 @@ NotebookLM वहाँ उपलब्ध सबसे अच्छे और
- **कोई विक्रेता लॉक-इन नहीं** - किसी भी LLM, इमेज, TTS और STT मॉडल को कॉन्फ़िगर करें।
- **25+ बाहरी डेटा स्रोत** - Google Drive, OneDrive, Dropbox, Notion और कई अन्य बाहरी सेवाओं से अपने स्रोत जोड़ें।
- **रीयल-टाइम मल्टीप्लेयर सपोर्ट** - एक साझा notebook में अपनी टीम के सदस्यों के साथ आसानी से काम करें।
-- **डेस्कटॉप ऐप** - Quick Assist, General Assist, Extreme Assist और लोकल फ़ोल्डर सिंक के साथ किसी भी एप्लिकेशन में AI सहायता प्राप्त करें।
+- **डेस्कटॉप ऐप** - Quick Assist, General Assist, Screenshot Assist और लोकल फ़ोल्डर सिंक के साथ किसी भी एप्लिकेशन में AI सहायता प्राप्त करें।
...और भी बहुत कुछ आने वाला है।
@@ -84,9 +84,9 @@ https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7
- डेस्कटॉप ऐप — Watch Local Folder
@@ -150,7 +150,7 @@ SurfSense एक डेस्कटॉप ऐप भी प्रदान क
- **General Assist** — एक ग्लोबल शॉर्टकट से किसी भी एप्लिकेशन से तुरंत SurfSense लॉन्च करें।
- **Quick Assist** — कहीं भी टेक्स्ट चुनें, फिर AI से समझाने, फिर से लिखने या उस पर कार्रवाई करने को कहें।
-- **Extreme Assist** — किसी भी ऐप में टाइप करते समय अपनी नॉलेज बेस से संचालित इनलाइन लेखन सुझाव प्राप्त करें।
+- **Screenshot Assist** — स्क्रीन पर एक क्षेत्र चुनें और उसे चैट में जोड़ें, ताकि उत्तर आपकी नॉलेज बेस पर आधारित रहें।
- **Watch Local Folder** — एक लोकल फ़ोल्डर को वॉच करें और फ़ाइल परिवर्तनों को स्वचालित रूप से अपनी नॉलेज बेस में सिंक करें। **Pro tip:** इसे अपने Obsidian vault पर पॉइंट करें ताकि आपके नोट्स SurfSense में सर्च करने योग्य रहें।
सभी सुविधाएं आपके चुने हुए सर्च स्पेस पर काम करती हैं, ताकि आपके उत्तर हमेशा आपके अपने डेटा पर आधारित हों।
@@ -199,14 +199,14 @@ SurfSense एक डेस्कटॉप ऐप भी प्रदान क
| **वीडियो जनरेशन** | Veo 3 के माध्यम से सिनेमैटिक वीडियो ओवरव्यू (केवल Ultra) | उपलब्ध (NotebookLM यहाँ बेहतर है, सक्रिय रूप से सुधार हो रहा है) |
| **प्रेजेंटेशन जनरेशन** | बेहतर दिखने वाली स्लाइड्स लेकिन संपादन योग्य नहीं | संपादन योग्य, स्लाइड आधारित प्रेजेंटेशन बनाएं |
| **पॉडकास्ट जनरेशन** | कस्टमाइज़ेबल होस्ट और भाषाओं के साथ ऑडियो ओवरव्यू | कई TTS प्रदाताओं के साथ उपलब्ध (NotebookLM यहाँ बेहतर है, सक्रिय रूप से सुधार हो रहा है) |
-| **डेस्कटॉप ऐप** | नहीं | General Assist, Quick Assist, Extreme Assist और लोकल फ़ोल्डर सिंक के साथ नेटिव ऐप |
+| **डेस्कटॉप ऐप** | नहीं | General Assist, Quick Assist, Screenshot Assist और लोकल फ़ोल्डर सिंक के साथ नेटिव ऐप |
| **ब्राउज़र एक्सटेंशन** | नहीं | किसी भी वेबपेज को सहेजने के लिए क्रॉस-ब्राउज़र एक्सटेंशन, प्रमाणीकरण सुरक्षित पेज सहित |
बाहरी स्रोतों की पूरी सूची
-सर्च इंजन (Tavily, LinkUp) · SearxNG · Google Drive · OneDrive · Dropbox · Slack · Microsoft Teams · Linear · Jira · ClickUp · Confluence · BookStack · Notion · Gmail · YouTube वीडियो · GitHub · Discord · Airtable · Google Calendar · Luma · Circleback · Elasticsearch · Obsidian, और भी बहुत कुछ आने वाला है।
+सर्च इंजन (SearXNG, Tavily, LinkUp, Baidu Search) · Google Drive · OneDrive · Dropbox · Slack · Microsoft Teams · Linear · Jira · ClickUp · Confluence · BookStack · Notion · Gmail · YouTube वीडियो · GitHub · Discord · Airtable · Google Calendar · Luma · Circleback · Elasticsearch · Obsidian, और भी बहुत कुछ आने वाला है।
diff --git a/README.md b/README.md
index 9714b9e65..ab9f9e221 100644
--- a/README.md
+++ b/README.md
@@ -42,7 +42,7 @@ NotebookLM is one of the best and most useful AI platforms out there, but once y
- **25+ External Data Sources** - Add your sources from Google Drive, OneDrive, Dropbox, Notion, and many other external services.
- **Real-Time Multiplayer Support** - Work easily with your team members in a shared notebook.
- **AI File Sorting** - Automatically organize your documents into a smart folder hierarchy using AI-powered categorization by source, date, and topic.
-- **Desktop App** - Get AI assistance in any application with Quick Assist, General Assist, Extreme Assist, and local folder sync.
+- **Desktop App** - Get AI assistance in any application with Quick Assist, General Assist, Screenshot Assist, and local folder sync.
...and more to come.
@@ -85,9 +85,9 @@ https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7
- Desktop App — Watch Local Folder
@@ -151,7 +151,7 @@ The desktop app includes these powerful features:
- **General Assist** — Launch SurfSense instantly from any application with a global shortcut.
- **Quick Assist** — Select text anywhere, then ask AI to explain, rewrite, or act on it.
-- **Extreme Assist** — Get inline writing suggestions powered by your knowledge base as you type in any app.
+- **Screenshot Assist** — Select a region on your screen and attach it to chat so answers stay grounded in your knowledge base.
- **Watch Local Folder** — Watch a local folder and automatically sync file changes to your knowledge base. **Pro tip:** Point it at your Obsidian vault to keep your notes searchable in SurfSense.
All features operate against your chosen search space, so your answers are always grounded in your own data.
@@ -201,14 +201,14 @@ All features operate against your chosen search space, so your answers are alway
| **Presentation Generation** | Better looking slides but not editable | Create editable, slide-based presentations |
| **Podcast Generation** | Audio Overviews with customizable hosts and languages | Available with multiple TTS providers (NotebookLM is better here, actively improving) |
| **AI File Sorting** | No | LLM-powered auto-categorization into source, date, category, and subcategory folders |
-| **Desktop App** | No | Native app with General Assist, Quick Assist, Extreme Assist, and local folder sync |
+| **Desktop App** | No | Native app with General Assist, Quick Assist, Screenshot Assist, and local folder sync |
| **Browser Extension** | No | Cross-browser extension to save any webpage, including auth-protected pages |
Full list of External Sources
-Search Engines (Tavily, LinkUp) · SearxNG · Google Drive · OneDrive · Dropbox · Slack · Microsoft Teams · Linear · Jira · ClickUp · Confluence · BookStack · Notion · Gmail · YouTube Videos · GitHub · Discord · Airtable · Google Calendar · Luma · Circleback · Elasticsearch · Obsidian, and more to come.
+Search Engines (SearXNG, Tavily, LinkUp, Baidu Search) · Google Drive · OneDrive · Dropbox · Slack · Microsoft Teams · Linear · Jira · ClickUp · Confluence · BookStack · Notion · Gmail · YouTube Videos · GitHub · Discord · Airtable · Google Calendar · Luma · Circleback · Elasticsearch · Obsidian, and more to come.
diff --git a/README.pt-BR.md b/README.pt-BR.md
index 9323b2bce..fcb004cd6 100644
--- a/README.pt-BR.md
+++ b/README.pt-BR.md
@@ -41,7 +41,7 @@ O NotebookLM é uma das melhores e mais úteis plataformas de IA disponíveis, m
- **Sem Dependência de Fornecedor** - Configure qualquer modelo LLM, de imagem, TTS e STT.
- **25+ Fontes de Dados Externas** - Adicione suas fontes do Google Drive, OneDrive, Dropbox, Notion e muitos outros serviços externos.
- **Suporte Multiplayer em Tempo Real** - Trabalhe facilmente com os membros da sua equipe em um notebook compartilhado.
-- **Aplicativo Desktop** - Obtenha assistência de IA em qualquer aplicativo com Quick Assist, General Assist, Extreme Assist e sincronização de pastas locais.
+- **Aplicativo Desktop** - Obtenha assistência de IA em qualquer aplicativo com Quick Assist, General Assist, Screenshot Assist e sincronização de pastas locais.
...e mais por vir.
@@ -84,9 +84,9 @@ https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7
- Aplicativo Desktop — Watch Local Folder
@@ -150,7 +150,7 @@ O aplicativo desktop inclui estes recursos poderosos:
- **General Assist** — Abra o SurfSense instantaneamente de qualquer aplicativo com um atalho global.
- **Quick Assist** — Selecione texto em qualquer lugar, depois peça à IA para explicar, reescrever ou agir sobre ele.
-- **Extreme Assist** — Receba sugestões de escrita em linha alimentadas pela sua base de conhecimento enquanto digita em qualquer aplicativo.
+- **Screenshot Assist** — Selecione uma região da tela e anexe ao chat para respostas fundamentadas na sua base de conhecimento.
- **Watch Local Folder** — Monitore uma pasta local e sincronize automaticamente as alterações de arquivos com sua base de conhecimento. **Pro tip:** Aponte para seu cofre do Obsidian para manter suas notas pesquisáveis no SurfSense.
Todos os recursos operam no espaço de busca escolhido, para que suas respostas sejam sempre baseadas nos seus próprios dados.
@@ -199,14 +199,14 @@ Todos os recursos operam no espaço de busca escolhido, para que suas respostas
| **Geração de Vídeos** | Visões gerais cinemáticas via Veo 3 (apenas Ultra) | Disponível (NotebookLM é melhor aqui, melhorando ativamente) |
| **Geração de Apresentações** | Slides mais bonitos mas não editáveis | Cria apresentações editáveis baseadas em slides |
| **Geração de Podcasts** | Visões gerais em áudio com hosts e idiomas personalizáveis | Disponível com múltiplos provedores TTS (NotebookLM é melhor aqui, melhorando ativamente) |
-| **Aplicativo Desktop** | Não | Aplicativo nativo com General Assist, Quick Assist, Extreme Assist e sincronização de pastas locais |
+| **Aplicativo Desktop** | Não | Aplicativo nativo com General Assist, Quick Assist, Screenshot Assist e sincronização de pastas locais |
| **Extensão de Navegador** | Não | Extensão multi-navegador para salvar qualquer página web, incluindo páginas protegidas por autenticação |
Lista completa de Fontes Externas
-Mecanismos de Busca (Tavily, LinkUp) · SearxNG · Google Drive · OneDrive · Dropbox · Slack · Microsoft Teams · Linear · Jira · ClickUp · Confluence · BookStack · Notion · Gmail · Vídeos do YouTube · GitHub · Discord · Airtable · Google Calendar · Luma · Circleback · Elasticsearch · Obsidian, e mais por vir.
+Mecanismos de Busca (SearXNG, Tavily, LinkUp, Baidu Search) · Google Drive · OneDrive · Dropbox · Slack · Microsoft Teams · Linear · Jira · ClickUp · Confluence · BookStack · Notion · Gmail · Vídeos do YouTube · GitHub · Discord · Airtable · Google Calendar · Luma · Circleback · Elasticsearch · Obsidian, e mais por vir.
diff --git a/README.zh-CN.md b/README.zh-CN.md
index 29200243b..a07f4afdc 100644
--- a/README.zh-CN.md
+++ b/README.zh-CN.md
@@ -41,7 +41,7 @@ NotebookLM 是目前最好、最实用的 AI 平台之一，但当你开始经
- **无供应商锁定** - 配置任何 LLM、图像、TTS 和 STT 模型。
- **25+ 外部数据源** - 从 Google Drive、OneDrive、Dropbox、Notion 和许多其他外部服务添加你的来源。
- **实时多人协作支持** - 在共享笔记本中轻松与团队成员协作。
-- **桌面应用** - 通过 Quick Assist、General Assist、Extreme Assist 和本地文件夹同步在任何应用程序中获得 AI 助手。
+- **桌面应用** - 通过 Quick Assist、General Assist、Screenshot Assist 和本地文件夹同步在任何应用程序中获得 AI 助手。
...更多功能即将推出。
@@ -84,9 +84,9 @@ https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7
- 桌面应用 — Watch Local Folder
@@ -150,7 +150,7 @@ SurfSense 还提供桌面应用，将 AI 助手带到您计算机上的每个应
- **General Assist** — 通过全局快捷键从任何应用程序即时启动 SurfSense。
- **Quick Assist** — 在任何位置选中文本，然后让 AI 解释、改写或对其执行操作。
-- **Extreme Assist** — 在任何应用中输入时，获得基于您知识库的内联写作建议。
+- **Screenshot Assist** — 在屏幕上框选区域并附加到聊天，让回复基于您的知识库。
- **Watch Local Folder** — 监视本地文件夹，自动将文件更改同步到您的知识库。**Pro tip：** 将其指向您的 Obsidian vault，让笔记在 SurfSense 中随时可搜索。
所有功能均基于您选择的搜索空间运行，确保回答始终以您自己的数据为依据。
@@ -199,14 +199,14 @@ SurfSense 还提供桌面应用，将 AI 助手带到您计算机上的每个应
| **视频生成** | 通过 Veo 3 的电影级视频概览（仅 Ultra） | 可用（NotebookLM 在此方面更好，正在积极改进） |
| **演示文稿生成** | 更美观的幻灯片但不可编辑 | 创建可编辑的幻灯片式演示文稿 |
| **播客生成** | 可自定义主持人和语言的音频概览 | 可用，支持多种 TTS 提供商（NotebookLM 在此方面更好，正在积极改进） |
-| **桌面应用** | 否 | 原生应用，包含 General Assist、Quick Assist、Extreme Assist 和本地文件夹同步 |
+| **桌面应用** | 否 | 原生应用，包含 General Assist、Quick Assist、Screenshot Assist 和本地文件夹同步 |
| **浏览器扩展** | 否 | 跨浏览器扩展，保存任何网页，包括需要身份验证的页面 |
外部数据源完整列表
-搜索引擎（Tavily、LinkUp）· SearxNG · Google Drive · OneDrive · Dropbox · Slack · Microsoft Teams · Linear · Jira · ClickUp · Confluence · BookStack · Notion · Gmail · YouTube 视频 · GitHub · Discord · Airtable · Google Calendar · Luma · Circleback · Elasticsearch · Obsidian，更多即将推出。
+搜索引擎（SearXNG、Tavily、LinkUp、Baidu Search）· Google Drive · OneDrive · Dropbox · Slack · Microsoft Teams · Linear · Jira · ClickUp · Confluence · BookStack · Notion · Gmail · YouTube 视频 · GitHub · Discord · Airtable · Google Calendar · Luma · Circleback · Elasticsearch · Obsidian，更多即将推出。
diff --git a/docker/docker-compose.deps-only.yml b/docker/docker-compose.deps-only.yml
new file mode 100644
index 000000000..ee09a4d5b
--- /dev/null
+++ b/docker/docker-compose.deps-only.yml
@@ -0,0 +1,123 @@
+# =============================================================================
+# SurfSense — Dependencies only (no backend / frontend / Celery images)
+# =============================================================================
+# Postgres, Redis, SearXNG, pgAdmin, Zero — run API + Next + Celery on the host.
+# Celery is not Dockerized here: use `uv run` from surfsense_backend/ (no extra
+# backend image build just for workers).
+#
+# From repo root (SurfSense/):
+# docker compose -f docker/docker-compose.deps-only.yml up -d
+#
+# Compose variable substitution uses `docker/.env` (copy from .env.example).
+# Bind mounts use ./postgresql.conf and ./searxng in this directory.
+#
+# Local Celery (from surfsense_backend/, after Redis is up):
+# uv run celery -A celery_worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo --queues=surfsense,surfsense.connectors
+# uv run celery -A celery_worker.celery_app beat --loglevel=info
+#
+# Host setup:
+# - Backend .env: DATABASE_URL=postgresql+asyncpg://postgres:postgres@localhost:5432/surfsense
+# - Backend .env: SEARXNG_DEFAULT_HOST=http://localhost:${SEARXNG_PORT:-8888}
+# - Backend .env: CELERY_BROKER_URL / REDIS_APP_URL → redis://localhost:6379/0
+# - Web .env: NEXT_PUBLIC_ZERO_CACHE_URL=http://localhost:${ZERO_CACHE_PORT:-4848}
+# =============================================================================
+
+name: surfsense-deps
+
+services:
+ db:
+ image: pgvector/pgvector:pg17
+ ports:
+ - "${POSTGRES_PORT:-5432}:5432"
+ volumes:
+ - postgres_data:/var/lib/postgresql/data
+ - ./postgresql.conf:/etc/postgresql/postgresql.conf:ro
+ environment:
+ - POSTGRES_USER=${DB_USER:-postgres}
+ - POSTGRES_PASSWORD=${DB_PASSWORD:-postgres}
+ - POSTGRES_DB=${DB_NAME:-surfsense}
+ command: postgres -c config_file=/etc/postgresql/postgresql.conf
+ healthcheck:
+ test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-postgres} -d ${DB_NAME:-surfsense}"]
+ interval: 10s
+ timeout: 5s
+ retries: 5
+
+ pgadmin:
+ image: dpage/pgadmin4
+ ports:
+ - "${PGADMIN_PORT:-5050}:80"
+ environment:
+ - PGADMIN_DEFAULT_EMAIL=${PGADMIN_DEFAULT_EMAIL:-admin@surfsense.com}
+ - PGADMIN_DEFAULT_PASSWORD=${PGADMIN_DEFAULT_PASSWORD:-surfsense}
+ volumes:
+ - pgadmin_data:/var/lib/pgadmin
+ depends_on:
+ - db
+
+ redis:
+ image: redis:8-alpine
+ ports:
+ - "${REDIS_PORT:-6379}:6379"
+ volumes:
+ - redis_data:/data
+ command: redis-server --appendonly yes
+ healthcheck:
+ test: ["CMD", "redis-cli", "ping"]
+ interval: 10s
+ timeout: 5s
+ retries: 5
+
+ searxng:
+ image: searxng/searxng:2026.3.13-3c1f68c59
+ ports:
+ - "${SEARXNG_PORT:-8888}:8080"
+ volumes:
+ - ./searxng:/etc/searxng
+ environment:
+ - SEARXNG_SECRET=${SEARXNG_SECRET:-surfsense-searxng-secret}
+ healthcheck:
+ test: ["CMD", "wget", "--spider", "-q", "http://localhost:8080/healthz"]
+ interval: 10s
+ timeout: 5s
+ retries: 5
+
+ zero-cache:
+ image: rocicorp/zero:0.26.2
+ ports:
+ - "${ZERO_CACHE_PORT:-4848}:4848"
+ extra_hosts:
+ - "host.docker.internal:host-gateway"
+ depends_on:
+ db:
+ condition: service_healthy
+ environment:
+ - ZERO_UPSTREAM_DB=postgresql://${DB_USER:-postgres}:${DB_PASSWORD:-postgres}@db:5432/${DB_NAME:-surfsense}?sslmode=${DB_SSLMODE:-disable}
+ - ZERO_CVR_DB=postgresql://${DB_USER:-postgres}:${DB_PASSWORD:-postgres}@db:5432/${DB_NAME:-surfsense}?sslmode=${DB_SSLMODE:-disable}
+ - ZERO_CHANGE_DB=postgresql://${DB_USER:-postgres}:${DB_PASSWORD:-postgres}@db:5432/${DB_NAME:-surfsense}?sslmode=${DB_SSLMODE:-disable}
+ - ZERO_REPLICA_FILE=/data/zero.db
+ - ZERO_ADMIN_PASSWORD=${ZERO_ADMIN_PASSWORD:-surfsense-zero-admin}
+ - ZERO_APP_PUBLICATIONS=${ZERO_APP_PUBLICATIONS:-zero_publication}
+ - ZERO_NUM_SYNC_WORKERS=${ZERO_NUM_SYNC_WORKERS:-4}
+ - ZERO_UPSTREAM_MAX_CONNS=${ZERO_UPSTREAM_MAX_CONNS:-20}
+ - ZERO_CVR_MAX_CONNS=${ZERO_CVR_MAX_CONNS:-30}
+ - ZERO_QUERY_URL=${ZERO_QUERY_URL:-http://host.docker.internal:3000/api/zero/query}
+ - ZERO_MUTATE_URL=${ZERO_MUTATE_URL:-http://host.docker.internal:3000/api/zero/mutate}
+ volumes:
+ - zero_cache_data:/data
+ restart: unless-stopped
+ healthcheck:
+ test: ["CMD", "curl", "-f", "http://localhost:4848/keepalive"]
+ interval: 10s
+ timeout: 5s
+ retries: 5
+
+volumes:
+ postgres_data:
+ name: surfsense-deps-postgres
+ pgadmin_data:
+ name: surfsense-deps-pgadmin
+ redis_data:
+ name: surfsense-deps-redis
+ zero_cache_data:
+ name: surfsense-deps-zero-cache
diff --git a/surfsense_backend/alembic/versions/117_optimize_zero_publication_column_lists.py b/surfsense_backend/alembic/versions/117_optimize_zero_publication_column_lists.py
index 78a26a381..3ad5a043b 100644
--- a/surfsense_backend/alembic/versions/117_optimize_zero_publication_column_lists.py
+++ b/surfsense_backend/alembic/versions/117_optimize_zero_publication_column_lists.py
@@ -79,40 +79,44 @@ def _terminate_blocked_pids(conn, table: str) -> None:
def upgrade() -> None:
conn = op.get_bind()
+ # asyncpg requires LOCK TABLE inside a transaction block. Alembic already
+ # opened one via context.begin_transaction(), but the driver still errors
+ # unless we use an explicit SAVEPOINT (nested transaction) for this block.
+ tx = conn.begin_nested() if conn.in_transaction() else conn.begin()
+ with tx:
+ conn.execute(sa.text("SET lock_timeout = '10s'"))
- conn.execute(sa.text("SET lock_timeout = '10s'"))
+ for tbl in sorted(TABLES_WITH_FULL_IDENTITY):
+ _terminate_blocked_pids(conn, tbl)
+ conn.execute(sa.text(f'LOCK TABLE "{tbl}" IN ACCESS EXCLUSIVE MODE'))
- for tbl in sorted(TABLES_WITH_FULL_IDENTITY):
- _terminate_blocked_pids(conn, tbl)
- conn.execute(sa.text(f'LOCK TABLE "{tbl}" IN ACCESS EXCLUSIVE MODE'))
+ for tbl in TABLES_WITH_FULL_IDENTITY:
+ conn.execute(sa.text(f'ALTER TABLE "{tbl}" REPLICA IDENTITY DEFAULT'))
- for tbl in TABLES_WITH_FULL_IDENTITY:
- conn.execute(sa.text(f'ALTER TABLE "{tbl}" REPLICA IDENTITY DEFAULT'))
+ conn.execute(sa.text(f"DROP PUBLICATION IF EXISTS {PUBLICATION_NAME}"))
- conn.execute(sa.text(f"DROP PUBLICATION IF EXISTS {PUBLICATION_NAME}"))
+ has_zero_ver = conn.execute(
+ sa.text(
+ "SELECT 1 FROM information_schema.columns "
+ "WHERE table_name = 'documents' AND column_name = '_0_version'"
+ )
+ ).fetchone()
- has_zero_ver = conn.execute(
- sa.text(
- "SELECT 1 FROM information_schema.columns "
- "WHERE table_name = 'documents' AND column_name = '_0_version'"
+ cols = DOCUMENT_COLS + (['"_0_version"'] if has_zero_ver else [])
+ col_list = ", ".join(cols)
+
+ conn.execute(
+ sa.text(
+ f"CREATE PUBLICATION {PUBLICATION_NAME} FOR TABLE "
+ f"notifications, "
+ f"documents ({col_list}), "
+ f"folders, "
+ f"search_source_connectors, "
+ f"new_chat_messages, "
+ f"chat_comments, "
+ f"chat_session_state"
+ )
)
- ).fetchone()
-
- cols = DOCUMENT_COLS + (['"_0_version"'] if has_zero_ver else [])
- col_list = ", ".join(cols)
-
- conn.execute(
- sa.text(
- f"CREATE PUBLICATION {PUBLICATION_NAME} FOR TABLE "
- f"notifications, "
- f"documents ({col_list}), "
- f"folders, "
- f"search_source_connectors, "
- f"new_chat_messages, "
- f"chat_comments, "
- f"chat_session_state"
- )
- )
def downgrade() -> None:
diff --git a/surfsense_backend/alembic/versions/121_add_memory_md_columns.py b/surfsense_backend/alembic/versions/121_add_memory_md_columns.py
index d5ff967fd..ac248dfca 100644
--- a/surfsense_backend/alembic/versions/121_add_memory_md_columns.py
+++ b/surfsense_backend/alembic/versions/121_add_memory_md_columns.py
@@ -12,8 +12,6 @@ from __future__ import annotations
from collections.abc import Sequence
-import sqlalchemy as sa
-
from alembic import op
revision: str = "121"
@@ -23,16 +21,30 @@ depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
- op.add_column(
- "user",
- sa.Column("memory_md", sa.Text(), nullable=True, server_default=""),
- )
- op.add_column(
- "searchspaces",
- sa.Column("shared_memory_md", sa.Text(), nullable=True, server_default=""),
+ # Idempotent: column(s) may already exist after a failed run or manual DDL.
+ op.execute(
+ """
+ DO $$
+ BEGIN
+ IF NOT EXISTS (
+ SELECT 1 FROM information_schema.columns
+ WHERE table_schema = 'public' AND table_name = 'user'
+ AND column_name = 'memory_md'
+ ) THEN
+ ALTER TABLE "user" ADD COLUMN memory_md TEXT DEFAULT '';
+ END IF;
+ IF NOT EXISTS (
+ SELECT 1 FROM information_schema.columns
+ WHERE table_schema = 'public' AND table_name = 'searchspaces'
+ AND column_name = 'shared_memory_md'
+ ) THEN
+ ALTER TABLE searchspaces ADD COLUMN shared_memory_md TEXT DEFAULT '';
+ END IF;
+ END$$;
+ """
)
def downgrade() -> None:
- op.drop_column("searchspaces", "shared_memory_md")
- op.drop_column("user", "memory_md")
+ op.execute("ALTER TABLE searchspaces DROP COLUMN IF EXISTS shared_memory_md")
+ op.execute('ALTER TABLE "user" DROP COLUMN IF EXISTS memory_md')
diff --git a/surfsense_backend/app/agents/autocomplete/__init__.py b/surfsense_backend/app/agents/autocomplete/__init__.py
deleted file mode 100644
index 55d7a692d..000000000
--- a/surfsense_backend/app/agents/autocomplete/__init__.py
+++ /dev/null
@@ -1,11 +0,0 @@
-"""Agent-based vision autocomplete with scoped filesystem exploration."""
-
-from app.agents.autocomplete.autocomplete_agent import (
- create_autocomplete_agent,
- stream_autocomplete_agent,
-)
-
-__all__ = [
- "create_autocomplete_agent",
- "stream_autocomplete_agent",
-]
diff --git a/surfsense_backend/app/agents/autocomplete/autocomplete_agent.py b/surfsense_backend/app/agents/autocomplete/autocomplete_agent.py
deleted file mode 100644
index 2d8f05fd3..000000000
--- a/surfsense_backend/app/agents/autocomplete/autocomplete_agent.py
+++ /dev/null
@@ -1,495 +0,0 @@
-"""Vision autocomplete agent with scoped filesystem exploration.
-
-Converts the stateless single-shot vision autocomplete into an agent that
-seeds a virtual filesystem from KB search results and lets the vision LLM
-explore documents via ``ls``, ``read_file``, ``glob``, ``grep``, etc.
-before generating the final completion.
-
-Performance: KB search and agent graph compilation run in parallel so
-the only sequential latency is KB-search (or agent compile, whichever is
-slower) + the agent's LLM turns. There is no separate "query extraction"
-LLM call — the window title is used directly as the KB search query.
-"""
-
-from __future__ import annotations
-
-import asyncio
-import json
-import logging
-import re
-import uuid
-from collections.abc import AsyncGenerator
-from typing import Any
-
-from deepagents.graph import BASE_AGENT_PROMPT
-from deepagents.middleware.patch_tool_calls import PatchToolCallsMiddleware
-from langchain.agents import create_agent
-from langchain_anthropic.middleware import AnthropicPromptCachingMiddleware
-from langchain_core.language_models import BaseChatModel
-from langchain_core.messages import AIMessage, ToolMessage
-
-from app.agents.new_chat.middleware.filesystem import SurfSenseFilesystemMiddleware
-from app.agents.new_chat.middleware.knowledge_search import (
- build_scoped_filesystem,
- search_knowledge_base,
-)
-from app.services.new_streaming_service import VercelStreamingService
-
-logger = logging.getLogger(__name__)
-
-KB_TOP_K = 10
-
-# ---------------------------------------------------------------------------
-# System prompt
-# ---------------------------------------------------------------------------
-
-AUTOCOMPLETE_SYSTEM_PROMPT = """You are a smart writing assistant that analyzes the user's screen to draft or complete text.
-
-You will receive a screenshot of the user's screen. Your PRIMARY source of truth is the screenshot itself — the visual context determines what to write.
-
-Your job:
-1. Analyze the ENTIRE screenshot to understand what the user is working on (email thread, chat conversation, document, code editor, form, etc.).
-2. Identify the text area where the user will type.
-3. Generate the text the user most likely wants to write based on the visual context.
-
-You also have access to the user's knowledge base documents via filesystem tools. However:
-- ONLY consult the knowledge base if the screenshot clearly involves a topic where your KB documents are DIRECTLY relevant (e.g., the user is writing about a specific project/topic that matches a document title).
-- Do NOT explore documents just because they exist. Most autocomplete requests can be answered purely from the screenshot.
-- If you do read a document, only incorporate information that is 100% relevant to what the user is typing RIGHT NOW. Do not add extra details, background, or tangential information from the KB.
-- Keep your output SHORT — autocomplete should feel like a natural continuation, not an essay.
-
-Key behavior:
-- If the text area is EMPTY, draft a concise response or message based on what you see on screen (e.g., reply to an email, respond to a chat message, continue a document).
-- If the text area already has text, continue it naturally — typically just a sentence or two.
-
-Rules:
-- Be CONCISE. Prefer a single paragraph or a few sentences. Autocomplete is a quick assist, not a full draft.
-- Match the tone and formality of the surrounding context.
-- If the screen shows code, write code. If it shows a casual chat, be casual. If it shows a formal email, be formal.
-- Do NOT describe the screenshot or explain your reasoning.
-- Do NOT cite or reference documents explicitly — just let the knowledge inform your writing naturally.
-- If you cannot determine what to write, output an empty JSON array: []
-
-## Output Format
-
-You MUST provide exactly 3 different suggestion options. Each should be a distinct, plausible completion — vary the tone, detail level, or angle.
-
-Return your suggestions as a JSON array of exactly 3 strings. Output ONLY the JSON array, nothing else — no markdown fences, no explanation, no commentary.
-
-Example format:
-["First suggestion text here.", "Second suggestion — a different take.", "Third option with another approach."]
-
-## Filesystem Tools `ls`, `read_file`, `write_file`, `edit_file`, `glob`, `grep`
-
-All file paths must start with a `/`.
-- ls: list files and directories at a given path.
-- read_file: read a file from the filesystem.
-- write_file: create a temporary file in the session (not persisted).
-- edit_file: edit a file in the session (not persisted for /documents/ files).
-- glob: find files matching a pattern (e.g., "**/*.xml").
-- grep: search for text within files.
-
-## When to Use Filesystem Tools
-
-BEFORE reaching for any tool, ask yourself: "Can I write a good completion purely from the screenshot?" If yes, just write it — do NOT explore the KB.
-
-Only use tools when:
-- The user is clearly writing about a specific topic that likely has detailed information in their KB.
-- You need a specific fact, name, number, or reference that the screenshot doesn't provide.
-
-When you do use tools, be surgical:
-- Check the `ls` output first. If no document title looks relevant, stop — do not read files just to see what's there.
-- If a title looks relevant, read only the `` (first ~20 lines) and jump to matched chunks. Do not read entire documents.
-- Extract only the specific information you need and move on to generating the completion.
-
-## Reading Documents Efficiently
-
-Documents are formatted as XML. Each document contains:
-- `` — title, type, URL, etc.
-- `` — a table of every chunk with its **line range** and a
- `matched="true"` flag for chunks that matched the search query.
-- `` — the actual chunks in original document order.
-
-**Workflow**: read the first ~20 lines to see the ``, identify
-chunks marked `matched="true"`, then use `read_file(path, offset=,
-limit=)` to jump directly to those sections."""
-
-APP_CONTEXT_BLOCK = """
-
-The user is currently working in "{app_name}" (window: "{window_title}"). Use this to understand the type of application and adapt your tone and format accordingly."""
-
-
-def _build_autocomplete_system_prompt(app_name: str, window_title: str) -> str:
- prompt = AUTOCOMPLETE_SYSTEM_PROMPT
- if app_name:
- prompt += APP_CONTEXT_BLOCK.format(app_name=app_name, window_title=window_title)
- return prompt
-
-
-# ---------------------------------------------------------------------------
-# Pre-compute KB filesystem (runs in parallel with agent compilation)
-# ---------------------------------------------------------------------------
-
-
-class _KBResult:
- """Container for pre-computed KB filesystem results."""
-
- __slots__ = ("files", "ls_ai_msg", "ls_tool_msg")
-
- def __init__(
- self,
- files: dict[str, Any] | None = None,
- ls_ai_msg: AIMessage | None = None,
- ls_tool_msg: ToolMessage | None = None,
- ) -> None:
- self.files = files
- self.ls_ai_msg = ls_ai_msg
- self.ls_tool_msg = ls_tool_msg
-
- @property
- def has_documents(self) -> bool:
- return bool(self.files)
-
-
-async def precompute_kb_filesystem(
- search_space_id: int,
- query: str,
- top_k: int = KB_TOP_K,
-) -> _KBResult:
- """Search the KB and build the scoped filesystem outside the agent.
-
- This is designed to be called via ``asyncio.gather`` alongside agent
- graph compilation so the two run concurrently.
- """
- if not query:
- return _KBResult()
-
- try:
- search_results = await search_knowledge_base(
- query=query,
- search_space_id=search_space_id,
- top_k=top_k,
- )
-
- if not search_results:
- return _KBResult()
-
- new_files, _ = await build_scoped_filesystem(
- documents=search_results,
- search_space_id=search_space_id,
- )
-
- if not new_files:
- return _KBResult()
-
- doc_paths = [
- p
- for p, v in new_files.items()
- if p.startswith("/documents/") and v is not None
- ]
- tool_call_id = f"auto_ls_{uuid.uuid4().hex[:12]}"
- ai_msg = AIMessage(
- content="",
- tool_calls=[
- {"name": "ls", "args": {"path": "/documents"}, "id": tool_call_id}
- ],
- )
- tool_msg = ToolMessage(
- content=str(doc_paths) if doc_paths else "No documents found.",
- tool_call_id=tool_call_id,
- )
- return _KBResult(files=new_files, ls_ai_msg=ai_msg, ls_tool_msg=tool_msg)
-
- except Exception:
- logger.warning(
- "KB pre-computation failed, proceeding without KB", exc_info=True
- )
- return _KBResult()
-
-
-# ---------------------------------------------------------------------------
-# Filesystem middleware — no save_document, no persistence
-# ---------------------------------------------------------------------------
-
-
-class AutocompleteFilesystemMiddleware(SurfSenseFilesystemMiddleware):
- """Filesystem middleware for autocomplete — read-only exploration only.
-
- Strips ``save_document`` (permanent KB persistence) and passes
- ``search_space_id=None`` so ``write_file`` / ``edit_file`` stay ephemeral.
- """
-
- def __init__(self) -> None:
- super().__init__(search_space_id=None, created_by_id=None)
- self.tools = [t for t in self.tools if t.name != "save_document"]
-
-
-# ---------------------------------------------------------------------------
-# Agent factory
-# ---------------------------------------------------------------------------
-
-
-async def _compile_agent(
- llm: BaseChatModel,
- app_name: str,
- window_title: str,
-) -> Any:
- """Compile the agent graph (CPU-bound, runs in a thread)."""
- system_prompt = _build_autocomplete_system_prompt(app_name, window_title)
- final_system_prompt = system_prompt + "\n\n" + BASE_AGENT_PROMPT
-
- middleware = [
- AutocompleteFilesystemMiddleware(),
- PatchToolCallsMiddleware(),
- AnthropicPromptCachingMiddleware(unsupported_model_behavior="ignore"),
- ]
-
- agent = await asyncio.to_thread(
- create_agent,
- llm,
- system_prompt=final_system_prompt,
- tools=[],
- middleware=middleware,
- )
- return agent.with_config({"recursion_limit": 200})
-
-
-async def create_autocomplete_agent(
- llm: BaseChatModel,
- *,
- search_space_id: int,
- kb_query: str,
- app_name: str = "",
- window_title: str = "",
-) -> tuple[Any, _KBResult]:
- """Create the autocomplete agent and pre-compute KB in parallel.
-
- Returns ``(agent, kb_result)`` so the caller can inject the pre-computed
- filesystem into the agent's initial state without any middleware delay.
- """
- agent, kb = await asyncio.gather(
- _compile_agent(llm, app_name, window_title),
- precompute_kb_filesystem(search_space_id, kb_query),
- )
- return agent, kb
-
-
-# ---------------------------------------------------------------------------
-# JSON suggestion parsing (with fallback)
-# ---------------------------------------------------------------------------
-
-
-def _parse_suggestions(raw: str) -> list[str]:
- """Extract a list of suggestion strings from the agent's output.
-
- Tries, in order:
- 1. Direct ``json.loads``
- 2. Extract content between ```json ... ``` fences
- 3. Find the first ``[`` … ``]`` span
- Falls back to wrapping the raw text as a single suggestion.
- """
- text = raw.strip()
- if not text:
- return []
-
- for candidate in _json_candidates(text):
- try:
- parsed = json.loads(candidate)
- if isinstance(parsed, list) and all(isinstance(s, str) for s in parsed):
- return [s for s in parsed if s.strip()]
- except (json.JSONDecodeError, ValueError):
- continue
-
- return [text]
-
-
-def _json_candidates(text: str) -> list[str]:
- """Yield candidate JSON strings from raw text."""
- candidates = [text]
-
- fence = re.search(r"```(?:json)?\s*\n?(.*?)```", text, re.DOTALL)
- if fence:
- candidates.append(fence.group(1).strip())
-
- bracket = re.search(r"\[.*]", text, re.DOTALL)
- if bracket:
- candidates.append(bracket.group(0))
-
- return candidates
-
-
-# ---------------------------------------------------------------------------
-# Streaming helper
-# ---------------------------------------------------------------------------
-
-
-async def stream_autocomplete_agent(
- agent: Any,
- input_data: dict[str, Any],
- streaming_service: VercelStreamingService,
- *,
- emit_message_start: bool = True,
-) -> AsyncGenerator[str, None]:
- """Stream agent events as Vercel SSE, with thinking steps for tool calls.
-
- When ``emit_message_start`` is False the caller has already sent the
- ``message_start`` event (e.g. to show preparation steps before the agent
- runs).
- """
- thread_id = uuid.uuid4().hex
- config = {"configurable": {"thread_id": thread_id}}
-
- text_buffer: list[str] = []
- active_tool_depth = 0
- thinking_step_counter = 0
- tool_step_ids: dict[str, str] = {}
- step_titles: dict[str, str] = {}
- completed_step_ids: set[str] = set()
- last_active_step_id: str | None = None
-
- def next_thinking_step_id() -> str:
- nonlocal thinking_step_counter
- thinking_step_counter += 1
- return f"autocomplete-step-{thinking_step_counter}"
-
- def complete_current_step() -> str | None:
- nonlocal last_active_step_id
- if last_active_step_id and last_active_step_id not in completed_step_ids:
- completed_step_ids.add(last_active_step_id)
- title = step_titles.get(last_active_step_id, "Done")
- event = streaming_service.format_thinking_step(
- step_id=last_active_step_id,
- title=title,
- status="complete",
- )
- last_active_step_id = None
- return event
- return None
-
- if emit_message_start:
- yield streaming_service.format_message_start()
-
- gen_step_id = next_thinking_step_id()
- last_active_step_id = gen_step_id
- step_titles[gen_step_id] = "Generating suggestions"
- yield streaming_service.format_thinking_step(
- step_id=gen_step_id,
- title="Generating suggestions",
- status="in_progress",
- )
-
- try:
- async for event in agent.astream_events(
- input_data, config=config, version="v2"
- ):
- event_type = event.get("event", "")
- if event_type == "on_chat_model_stream":
- if active_tool_depth > 0:
- continue
- if "surfsense:internal" in event.get("tags", []):
- continue
- chunk = event.get("data", {}).get("chunk")
- if chunk and hasattr(chunk, "content"):
- content = chunk.content
- if content and isinstance(content, str):
- text_buffer.append(content)
-
- elif event_type == "on_chat_model_end":
- if active_tool_depth > 0:
- continue
- if "surfsense:internal" in event.get("tags", []):
- continue
- output = event.get("data", {}).get("output")
- if output and hasattr(output, "content"):
- if getattr(output, "tool_calls", None):
- continue
- content = output.content
- if content and isinstance(content, str) and not text_buffer:
- text_buffer.append(content)
-
- elif event_type == "on_tool_start":
- active_tool_depth += 1
- tool_name = event.get("name", "unknown_tool")
- run_id = event.get("run_id", "")
- tool_input = event.get("data", {}).get("input", {})
-
- step_event = complete_current_step()
- if step_event:
- yield step_event
-
- tool_step_id = next_thinking_step_id()
- tool_step_ids[run_id] = tool_step_id
- last_active_step_id = tool_step_id
-
- title, items = _describe_tool_call(tool_name, tool_input)
- step_titles[tool_step_id] = title
- yield streaming_service.format_thinking_step(
- step_id=tool_step_id,
- title=title,
- status="in_progress",
- items=items,
- )
-
- elif event_type == "on_tool_end":
- active_tool_depth = max(0, active_tool_depth - 1)
- run_id = event.get("run_id", "")
- step_id = tool_step_ids.pop(run_id, None)
- if step_id and step_id not in completed_step_ids:
- completed_step_ids.add(step_id)
- title = step_titles.get(step_id, "Done")
- yield streaming_service.format_thinking_step(
- step_id=step_id,
- title=title,
- status="complete",
- )
- if last_active_step_id == step_id:
- last_active_step_id = None
-
- step_event = complete_current_step()
- if step_event:
- yield step_event
-
- raw_text = "".join(text_buffer)
- suggestions = _parse_suggestions(raw_text)
-
- yield streaming_service.format_data("suggestions", {"options": suggestions})
-
- yield streaming_service.format_finish()
- yield streaming_service.format_done()
-
- except Exception as e:
- logger.error(f"Autocomplete agent streaming error: {e}", exc_info=True)
- yield streaming_service.format_error("Autocomplete failed. Please try again.")
- yield streaming_service.format_done()
-
-
-def _describe_tool_call(tool_name: str, tool_input: Any) -> tuple[str, list[str]]:
- """Return a human-readable (title, items) for a tool call thinking step."""
- inp = tool_input if isinstance(tool_input, dict) else {}
- if tool_name == "ls":
- path = inp.get("path", "/")
- return "Listing files", [path]
- if tool_name == "read_file":
- fp = inp.get("file_path", "")
- display = fp if len(fp) <= 80 else "…" + fp[-77:]
- return "Reading file", [display]
- if tool_name == "write_file":
- fp = inp.get("file_path", "")
- display = fp if len(fp) <= 80 else "…" + fp[-77:]
- return "Writing file", [display]
- if tool_name == "edit_file":
- fp = inp.get("file_path", "")
- display = fp if len(fp) <= 80 else "…" + fp[-77:]
- return "Editing file", [display]
- if tool_name == "glob":
- pat = inp.get("pattern", "")
- base = inp.get("path", "/")
- return "Searching files", [f"{pat} in {base}"]
- if tool_name == "grep":
- pat = inp.get("pattern", "")
- path = inp.get("path", "")
- display_pat = pat[:60] + ("…" if len(pat) > 60 else "")
- return "Searching content", [
- f'"{display_pat}"' + (f" in {path}" if path else "")
- ]
- return f"Using {tool_name}", []
diff --git a/surfsense_backend/app/routes/__init__.py b/surfsense_backend/app/routes/__init__.py
index 8df930f30..fafd4d356 100644
--- a/surfsense_backend/app/routes/__init__.py
+++ b/surfsense_backend/app/routes/__init__.py
@@ -3,7 +3,6 @@ from fastapi import APIRouter
from .airtable_add_connector_route import (
router as airtable_add_connector_router,
)
-from .autocomplete_routes import router as autocomplete_router
from .chat_comments_routes import router as chat_comments_router
from .circleback_webhook_route import router as circleback_webhook_router
from .clickup_add_connector_route import router as clickup_add_connector_router
@@ -106,4 +105,3 @@ router.include_router(stripe_router) # Stripe checkout for additional page pack
router.include_router(youtube_router) # YouTube playlist resolution
router.include_router(prompts_router)
router.include_router(memory_router) # User personal memory (memory.md style)
-router.include_router(autocomplete_router) # Lightweight autocomplete with KB context
diff --git a/surfsense_backend/app/routes/autocomplete_routes.py b/surfsense_backend/app/routes/autocomplete_routes.py
deleted file mode 100644
index a11b7dbc1..000000000
--- a/surfsense_backend/app/routes/autocomplete_routes.py
+++ /dev/null
@@ -1,45 +0,0 @@
-from fastapi import APIRouter, Depends
-from fastapi.responses import StreamingResponse
-from pydantic import BaseModel, Field
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.db import User, get_async_session
-from app.services.new_streaming_service import VercelStreamingService
-from app.services.vision_autocomplete_service import stream_vision_autocomplete
-from app.users import current_active_user
-from app.utils.rbac import check_search_space_access
-
-router = APIRouter(prefix="/autocomplete", tags=["autocomplete"])
-
-MAX_SCREENSHOT_SIZE = 20 * 1024 * 1024 # 20 MB base64 ceiling
-
-
-class VisionAutocompleteRequest(BaseModel):
- screenshot: str = Field(..., max_length=MAX_SCREENSHOT_SIZE)
- search_space_id: int
- app_name: str = ""
- window_title: str = ""
-
-
-@router.post("/vision/stream")
-async def vision_autocomplete_stream(
- body: VisionAutocompleteRequest,
- user: User = Depends(current_active_user),
- session: AsyncSession = Depends(get_async_session),
-):
- await check_search_space_access(session, user, body.search_space_id)
-
- return StreamingResponse(
- stream_vision_autocomplete(
- body.screenshot,
- body.search_space_id,
- session,
- app_name=body.app_name,
- window_title=body.window_title,
- ),
- media_type="text/event-stream",
- headers={
- **VercelStreamingService.get_response_headers(),
- "X-Accel-Buffering": "no",
- },
- )
diff --git a/surfsense_backend/app/routes/new_chat_routes.py b/surfsense_backend/app/routes/new_chat_routes.py
index 85a8658ec..cbc660222 100644
--- a/surfsense_backend/app/routes/new_chat_routes.py
+++ b/surfsense_backend/app/routes/new_chat_routes.py
@@ -24,9 +24,9 @@ from sqlalchemy.orm import selectinload
from app.agents.new_chat.filesystem_selection import (
ClientPlatform,
- LocalFilesystemMount,
FilesystemMode,
FilesystemSelection,
+ LocalFilesystemMount,
)
from app.config import config
from app.db import (
@@ -64,6 +64,10 @@ from app.services.token_tracking_service import record_token_usage
from app.tasks.chat.stream_new_chat import stream_new_chat, stream_resume_chat
from app.users import current_active_user
from app.utils.rbac import check_permission
+from app.utils.user_message_multimodal import (
+ split_langchain_human_content,
+ split_persisted_user_content_parts,
+)
_logger = logging.getLogger(__name__)
_background_tasks: set[asyncio.Task] = set()
@@ -1237,6 +1241,10 @@ async def handle_new_chat(
# connection (the "Exception terminating connection" errors).
await session.close()
+ image_urls = (
+ [p.as_data_url() for p in request.user_images] if request.user_images else None
+ )
+
return StreamingResponse(
stream_new_chat(
user_query=request.user_query,
@@ -1252,6 +1260,7 @@ async def handle_new_chat(
disabled_tools=request.disabled_tools,
filesystem_selection=filesystem_selection,
request_id=getattr(http_request.state, "request_id", "unknown"),
+ user_image_data_urls=image_urls,
),
media_type="text/event-stream",
headers={
@@ -1360,6 +1369,7 @@ async def regenerate_response(
target_checkpoint_id = None
user_query_to_use = request.user_query
+ regenerate_image_urls: list[str] = []
# Look through checkpoints to find the right one
# We want to find the checkpoint just before the last HumanMessage
@@ -1385,9 +1395,13 @@ async def regenerate_response(
prev_messages = prev_channel_values.get("messages", [])
for msg in reversed(prev_messages):
if isinstance(msg, HumanMessage):
- user_query_to_use = msg.content
+ q, imgs = split_langchain_human_content(msg.content)
+ user_query_to_use = q
+ regenerate_image_urls = imgs
break
- if user_query_to_use:
+ if user_query_to_use is not None and (
+ str(user_query_to_use).strip() or regenerate_image_urls
+ ):
break
target_checkpoint_id = cp_tuple.config["configurable"][
@@ -1405,7 +1419,9 @@ async def regenerate_response(
state_messages = channel_values.get("messages", [])
for msg in state_messages:
if isinstance(msg, HumanMessage):
- user_query_to_use = msg.content
+ q, imgs = split_langchain_human_content(msg.content)
+ user_query_to_use = q
+ regenerate_image_urls = imgs
break
else:
# Use the oldest checkpoint
@@ -1431,20 +1447,28 @@ async def regenerate_response(
if isinstance(content, str):
user_query_to_use = content
elif isinstance(content, list):
- # Extract text from content parts
- for part in content:
- if isinstance(part, dict) and part.get("type") == "text":
- user_query_to_use = part.get("text", "")
- break
- elif isinstance(part, str):
- user_query_to_use = part
- break
+ plain, imgs = split_persisted_user_content_parts(content)
+ user_query_to_use = plain
+ regenerate_image_urls = imgs
+
+ if isinstance(user_query_to_use, list):
+ user_query_to_use, regenerate_image_urls = split_langchain_human_content(
+ user_query_to_use
+ )
+
+ if request.user_images is not None:
+ regenerate_image_urls = [p.as_data_url() for p in request.user_images]
if user_query_to_use is None:
raise HTTPException(
status_code=400,
detail="Could not determine user query for regeneration. Please provide a user_query.",
)
+ if not str(user_query_to_use).strip() and not regenerate_image_urls:
+ raise HTTPException(
+ status_code=400,
+ detail="Could not determine user query for regeneration. Please provide a user_query.",
+ )
# Get the last two messages to delete AFTER streaming succeeds
# This prevents data loss if streaming fails
@@ -1483,7 +1507,7 @@ async def regenerate_response(
streaming_completed = False
try:
async for chunk in stream_new_chat(
- user_query=user_query_to_use,
+ user_query=str(user_query_to_use),
search_space_id=request.search_space_id,
chat_id=thread_id,
user_id=str(user.id),
@@ -1497,6 +1521,7 @@ async def regenerate_response(
disabled_tools=request.disabled_tools,
filesystem_selection=filesystem_selection,
request_id=getattr(http_request.state, "request_id", "unknown"),
+ user_image_data_urls=regenerate_image_urls or None,
):
yield chunk
streaming_completed = True
diff --git a/surfsense_backend/app/schemas/new_chat.py b/surfsense_backend/app/schemas/new_chat.py
index 1222deab2..477fdf2ca 100644
--- a/surfsense_backend/app/schemas/new_chat.py
+++ b/surfsense_backend/app/schemas/new_chat.py
@@ -7,12 +7,13 @@ These schemas follow the assistant-ui ThreadHistoryAdapter pattern:
"""
from datetime import datetime
-from typing import Any, Literal
+from typing import Any, Literal, Self
from uuid import UUID
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
from app.db import ChatVisibility, NewChatMessageRole
+from app.utils.user_message_multimodal import decode_base64_image, to_data_url
from .base import IDModel, TimestampModel
@@ -173,6 +174,26 @@ class LocalFilesystemMountPayload(BaseModel):
root_path: str
+MAX_NEW_CHAT_IMAGE_BYTES = 8 * 1024 * 1024
+MAX_NEW_CHAT_IMAGES = 4
+
+
+class NewChatUserImagePart(BaseModel):
+ """One inline image for a user turn (raw base64 body, no data: URL prefix)."""
+
+ media_type: Literal["image/png", "image/jpeg", "image/webp"]
+ data: str = Field(..., min_length=1)
+
+ @field_validator("data")
+ @classmethod
+ def _validate_payload(cls, v: str) -> str:
+ decode_base64_image(v, max_bytes=MAX_NEW_CHAT_IMAGE_BYTES)
+ return v
+
+ def as_data_url(self) -> str:
+ return to_data_url(self.media_type, self.data)
+
+
class NewChatRequest(BaseModel):
"""Request schema for the deep agent chat endpoint."""
@@ -192,6 +213,20 @@ class NewChatRequest(BaseModel):
filesystem_mode: Literal["cloud", "desktop_local_folder"] = "cloud"
client_platform: Literal["web", "desktop"] = "web"
local_filesystem_mounts: list[LocalFilesystemMountPayload] | None = None
+ user_images: list[NewChatUserImagePart] | None = Field(
+ default=None,
+ description="Optional images for this user turn",
+ )
+
+ @model_validator(mode="after")
+ def _require_text_or_images(self) -> Self:
+ has_text = bool(self.user_query.strip())
+ has_images = bool(self.user_images)
+ if not has_text and not has_images:
+ raise ValueError("Provide non-empty user_query and/or user_images")
+ if self.user_images is not None and len(self.user_images) > MAX_NEW_CHAT_IMAGES:
+ raise ValueError(f"At most {MAX_NEW_CHAT_IMAGES} images allowed")
+ return self
class RegenerateRequest(BaseModel):
@@ -203,6 +238,9 @@ class RegenerateRequest(BaseModel):
2. Reload: Leave user_query empty to regenerate the last AI response with the same query
Both operations rewind the LangGraph checkpointer to the appropriate state.
+
+ For edit, optional user_images (when not None) replaces image URLs resolved from
+ checkpoint/DB so the client can send the full user turn (text and/or images).
"""
search_space_id: int
@@ -215,6 +253,16 @@ class RegenerateRequest(BaseModel):
filesystem_mode: Literal["cloud", "desktop_local_folder"] = "cloud"
client_platform: Literal["web", "desktop"] = "web"
local_filesystem_mounts: list[LocalFilesystemMountPayload] | None = None
+ user_images: list[NewChatUserImagePart] | None = Field(
+ default=None,
+ description="If set, use these images for the regenerated turn (edit); overrides checkpoint/DB",
+ )
+
+ @model_validator(mode="after")
+ def _validate_regenerate_user_images(self) -> Self:
+ if self.user_images is not None and len(self.user_images) > MAX_NEW_CHAT_IMAGES:
+ raise ValueError(f"At most {MAX_NEW_CHAT_IMAGES} images allowed")
+ return self
# =============================================================================
diff --git a/surfsense_backend/app/services/vision_autocomplete_service.py b/surfsense_backend/app/services/vision_autocomplete_service.py
deleted file mode 100644
index c28962b31..000000000
--- a/surfsense_backend/app/services/vision_autocomplete_service.py
+++ /dev/null
@@ -1,158 +0,0 @@
-"""Vision autocomplete service — agent-based with scoped filesystem.
-
-Optimized pipeline:
-1. Start the SSE stream immediately so the UI shows progress.
-2. Derive a KB search query from window_title (no separate LLM call).
-3. Run KB filesystem pre-computation and agent graph compilation in PARALLEL.
-4. Inject pre-computed KB files as initial state and stream the agent.
-"""
-
-import logging
-from collections.abc import AsyncGenerator
-
-from langchain_core.messages import HumanMessage
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.agents.autocomplete import create_autocomplete_agent, stream_autocomplete_agent
-from app.services.llm_service import get_vision_llm
-from app.services.new_streaming_service import VercelStreamingService
-
-logger = logging.getLogger(__name__)
-
-PREP_STEP_ID = "autocomplete-prep"
-
-
-def _derive_kb_query(app_name: str, window_title: str) -> str:
- parts = [p for p in (window_title, app_name) if p]
- return " ".join(parts)
-
-
-def _is_vision_unsupported_error(e: Exception) -> bool:
- msg = str(e).lower()
- return "content must be a string" in msg or "does not support image" in msg
-
-
-# ---------------------------------------------------------------------------
-# Main entry point
-# ---------------------------------------------------------------------------
-
-
-async def stream_vision_autocomplete(
- screenshot_data_url: str,
- search_space_id: int,
- session: AsyncSession,
- *,
- app_name: str = "",
- window_title: str = "",
-) -> AsyncGenerator[str, None]:
- """Analyze a screenshot with a vision-LLM agent and stream a text completion."""
- streaming = VercelStreamingService()
- vision_error_msg = (
- "The selected model does not support vision. "
- "Please set a vision-capable model (e.g. GPT-4o, Gemini) in your search space settings."
- )
-
- llm = await get_vision_llm(session, search_space_id)
- if not llm:
- yield streaming.format_message_start()
- yield streaming.format_error("No Vision LLM configured for this search space")
- yield streaming.format_done()
- return
-
- # Start SSE stream immediately so the UI has something to show
- yield streaming.format_message_start()
-
- kb_query = _derive_kb_query(app_name, window_title)
-
- # Show a preparation step while KB search + agent compile run
- yield streaming.format_thinking_step(
- step_id=PREP_STEP_ID,
- title="Searching knowledge base",
- status="in_progress",
- items=[kb_query] if kb_query else [],
- )
-
- try:
- agent, kb = await create_autocomplete_agent(
- llm,
- search_space_id=search_space_id,
- kb_query=kb_query,
- app_name=app_name,
- window_title=window_title,
- )
- except Exception as e:
- if _is_vision_unsupported_error(e):
- logger.warning("Vision autocomplete: model does not support vision: %s", e)
- yield streaming.format_error(vision_error_msg)
- yield streaming.format_done()
- return
- logger.error("Failed to create autocomplete agent: %s", e, exc_info=True)
- yield streaming.format_error("Autocomplete failed. Please try again.")
- yield streaming.format_done()
- return
-
- has_kb = kb.has_documents
- doc_count = len(kb.files) if has_kb else 0 # type: ignore[arg-type]
-
- yield streaming.format_thinking_step(
- step_id=PREP_STEP_ID,
- title="Searching knowledge base",
- status="complete",
- items=[f"Found {doc_count} document{'s' if doc_count != 1 else ''}"]
- if kb_query
- else ["Skipped"],
- )
-
- # Build agent input with pre-computed KB as initial state
- if has_kb:
- instruction = (
- "Analyze this screenshot, then explore the knowledge base documents "
- "listed above — read the chunk index of any document whose title "
- "looks relevant and check matched chunks for useful facts. "
- "Finally, generate a concise autocomplete for the active text area, "
- "enhanced with any relevant KB information you found."
- )
- else:
- instruction = (
- "Analyze this screenshot and generate a concise autocomplete "
- "for the active text area based on what you see."
- )
-
- user_message = HumanMessage(
- content=[
- {"type": "text", "text": instruction},
- {"type": "image_url", "image_url": {"url": screenshot_data_url}},
- ]
- )
-
- input_data: dict = {"messages": [user_message]}
-
- if has_kb:
- input_data["files"] = kb.files
- input_data["messages"] = [kb.ls_ai_msg, kb.ls_tool_msg, user_message]
- logger.info(
- "Autocomplete: injected %d KB files into agent initial state", doc_count
- )
- else:
- logger.info(
- "Autocomplete: no KB documents found, proceeding with screenshot only"
- )
-
- # Stream the agent (message_start already sent above)
- try:
- async for sse in stream_autocomplete_agent(
- agent,
- input_data,
- streaming,
- emit_message_start=False,
- ):
- yield sse
- except Exception as e:
- if _is_vision_unsupported_error(e):
- logger.warning("Vision autocomplete: model does not support vision: %s", e)
- yield streaming.format_error(vision_error_msg)
- yield streaming.format_done()
- else:
- logger.error("Vision autocomplete streaming error: %s", e, exc_info=True)
- yield streaming.format_error("Autocomplete failed. Please try again.")
- yield streaming.format_done()
diff --git a/surfsense_backend/app/tasks/chat/stream_new_chat.py b/surfsense_backend/app/tasks/chat/stream_new_chat.py
index 5a6117808..396c7574e 100644
--- a/surfsense_backend/app/tasks/chat/stream_new_chat.py
+++ b/surfsense_backend/app/tasks/chat/stream_new_chat.py
@@ -31,7 +31,6 @@ from sqlalchemy.orm import selectinload
from app.agents.new_chat.chat_deepagent import create_surfsense_deep_agent
from app.agents.new_chat.checkpointer import get_checkpointer
from app.agents.new_chat.filesystem_selection import FilesystemSelection
-from app.config import config
from app.agents.new_chat.llm_config import (
AgentConfig,
create_chat_litellm_from_agent_config,
@@ -62,6 +61,7 @@ from app.services.connector_service import ConnectorService
from app.services.new_streaming_service import VercelStreamingService
from app.utils.content_utils import bootstrap_history_from_db
from app.utils.perf import get_perf_logger, log_system_snapshot, trim_native_heap
+from app.utils.user_message_multimodal import build_human_message_content
_background_tasks: set[asyncio.Task] = set()
_perf_log = get_perf_logger()
@@ -1350,6 +1350,7 @@ async def stream_new_chat(
disabled_tools: list[str] | None = None,
filesystem_selection: FilesystemSelection | None = None,
request_id: str | None = None,
+ user_image_data_urls: list[str] | None = None,
) -> AsyncGenerator[str, None]:
"""
Stream chat responses from the new SurfSense deep agent.
@@ -1625,8 +1626,10 @@ async def stream_new_chat(
# elif msg.role == "assistant":
# langchain_messages.append(AIMessage(content=msg.content))
# else:
- # Fallback: just use the current user query with attachment context
- langchain_messages.append(HumanMessage(content=final_query))
+ human_content = build_human_message_content(
+ final_query, list(user_image_data_urls or ())
+ )
+ langchain_messages.append(HumanMessage(content=human_content))
input_state = {
# Lets not pass this message atm because we are using the checkpointer to manage the conversation history
@@ -1687,8 +1690,13 @@ async def stream_new_chat(
action_verb = "Processing"
processing_parts = []
- query_text = user_query[:80] + ("..." if len(user_query) > 80 else "")
- processing_parts.append(query_text)
+ if user_query.strip():
+ query_text = user_query[:80] + ("..." if len(user_query) > 80 else "")
+ processing_parts.append(query_text)
+ elif user_image_data_urls:
+ processing_parts.append(f"[{len(user_image_data_urls)} image(s)]")
+ else:
+ processing_parts.append("(message)")
if mentioned_surfsense_docs:
doc_names = []
@@ -1750,8 +1758,13 @@ async def stream_new_chat(
_turn_accumulator.set(None)
+ title_seed = user_query.strip() or (
+ f"[{len(user_image_data_urls or [])} image(s)]"
+ if user_image_data_urls
+ else ""
+ )
prompt = TITLE_GENERATION_PROMPT.replace(
- "{user_query}", user_query[:500]
+ "{user_query}", title_seed[:500] or "(message)"
)
messages = [{"role": "user", "content": prompt}]
@@ -1947,10 +1960,15 @@ async def stream_new_chat(
# Fire background memory extraction if the agent didn't handle it.
# Shared threads write to team memory; private threads write to user memory.
if not stream_result.agent_called_update_memory:
+ memory_seed = user_query.strip() or (
+ f"[{len(user_image_data_urls or [])} image(s)]"
+ if user_image_data_urls
+ else "(message)"
+ )
if visibility == ChatVisibility.SEARCH_SPACE:
task = asyncio.create_task(
extract_and_save_team_memory(
- user_message=user_query,
+ user_message=memory_seed,
search_space_id=search_space_id,
llm=llm,
author_display_name=current_user_display_name,
@@ -1961,7 +1979,7 @@ async def stream_new_chat(
elif user_id:
task = asyncio.create_task(
extract_and_save_memory(
- user_message=user_query,
+ user_message=memory_seed,
user_id=user_id,
llm=llm,
)
diff --git a/surfsense_backend/app/utils/user_message_multimodal.py b/surfsense_backend/app/utils/user_message_multimodal.py
new file mode 100644
index 000000000..1d0691697
--- /dev/null
+++ b/surfsense_backend/app/utils/user_message_multimodal.py
@@ -0,0 +1,80 @@
+"""Helpers for multimodal user turns (text + inline images) in LangChain messages."""
+
+from __future__ import annotations
+
+import base64
+import binascii
+from typing import Any
+
+
+def build_human_message_content(final_query: str, image_data_urls: list[str]) -> str | list[dict[str, Any]]:
+ if not image_data_urls:
+ return final_query
+ parts: list[dict[str, Any]] = [{"type": "text", "text": final_query}]
+ for url in image_data_urls:
+ parts.append({"type": "image_url", "image_url": {"url": url}})
+ return parts
+
+
+def split_langchain_human_content(content: str | list[Any]) -> tuple[str, list[str]]:
+ """Return plain text and data URLs from a LangChain HumanMessage ``content`` value."""
+ if isinstance(content, str):
+ return content, []
+ if not isinstance(content, list):
+ return "", []
+
+ text_chunks: list[str] = []
+ urls: list[str] = []
+ for block in content:
+ if not isinstance(block, dict):
+ continue
+ btype = block.get("type")
+ if btype == "text":
+ t = block.get("text")
+ if isinstance(t, str) and t:
+ text_chunks.append(t)
+ elif btype == "image_url":
+ iu = block.get("image_url")
+ if isinstance(iu, dict):
+ u = iu.get("url")
+ if isinstance(u, str) and u.startswith("data:"):
+ urls.append(u)
+ elif isinstance(iu, str) and iu.startswith("data:"):
+ urls.append(iu)
+ return "\n".join(text_chunks), urls
+
+
+def decode_base64_image(data: str, *, max_bytes: int) -> bytes:
+ raw = data.strip()
+ if not raw:
+ raise ValueError("empty image payload")
+ try:
+ decoded = base64.b64decode(raw, validate=True)
+ except binascii.Error as e:
+ raise ValueError("invalid base64 image data") from e
+ if len(decoded) > max_bytes:
+ raise ValueError("image exceeds maximum size")
+ return decoded
+
+
+def to_data_url(media_type: str, raw_b64: str) -> str:
+ return f"data:{media_type};base64,{raw_b64.strip()}"
+
+
+def split_persisted_user_content_parts(parts: list[Any]) -> tuple[str, list[str]]:
+ """Extract plain text and data URLs from persisted assistant-ui style user ``content``."""
+ text_chunks: list[str] = []
+ urls: list[str] = []
+ for block in parts:
+ if not isinstance(block, dict):
+ continue
+ btype = block.get("type")
+ if btype == "text":
+ t = block.get("text")
+ if isinstance(t, str):
+ text_chunks.append(t)
+ elif btype == "image":
+ u = block.get("image")
+ if isinstance(u, str) and u.startswith("data:"):
+ urls.append(u)
+ return "".join(text_chunks), urls
diff --git a/surfsense_desktop/README.md b/surfsense_desktop/README.md
index 80efefba8..0f7a99e93 100644
--- a/surfsense_desktop/README.md
+++ b/surfsense_desktop/README.md
@@ -17,6 +17,8 @@ pnpm dev
This starts the Next.js dev server and Electron concurrently. Hot reload works — edit the web app and changes appear immediately.
+On **Linux**, `pnpm dev` runs Electron through `scripts/electron-dev.mjs`: it sets `ELECTRON_DISABLE_SANDBOX=1` for the sandbox issue and passes **`--ozone-platform=x11`** (XWayland) unless **`SURFSENSE_ELECTRON_WAYLAND=1`** is set, so dev tends to behave closer to X11 for shortcuts and Ozone. Packaged Linux builds are unchanged.
+
## Configuration
Two `.env` files control the build:
@@ -43,12 +45,13 @@ cd ../surfsense_desktop
pnpm build
```
-**Step 3** — Package into a distributable:
+**Step 3** — Package into a distributable (after steps 1–2):
```bash
pnpm dist:mac # macOS (.dmg + .zip)
pnpm dist:win # Windows (.exe)
pnpm dist:linux # Linux (.deb + .AppImage)
+pnpm pack:dir # optional: unpacked app only → release/… (run that binary yourself)
```
**Step 4** — Find the output:
diff --git a/surfsense_desktop/electron-builder.yml b/surfsense_desktop/electron-builder.yml
index 2c46c827a..b0014a57b 100644
--- a/surfsense_desktop/electron-builder.yml
+++ b/surfsense_desktop/electron-builder.yml
@@ -49,8 +49,8 @@ mac:
hardenedRuntime: false
gatekeeperAssess: false
extendInfo:
- NSAccessibilityUsageDescription: "SurfSense uses accessibility features to insert suggestions into the active application."
- NSScreenCaptureUsageDescription: "SurfSense uses screen capture to analyze your screen and provide context-aware writing suggestions."
+ NSAccessibilityUsageDescription: "SurfSense uses accessibility features to bring the app to the foreground and interact with the active application when you use desktop assists."
+ NSScreenCaptureUsageDescription: "SurfSense uses screen capture so you can attach a selected region to chat (Screenshot Assist) or capture the full screen from the composer."
NSAppleEventsUsageDescription: "SurfSense uses Apple Events to interact with the active application."
target:
- target: dmg
@@ -81,4 +81,5 @@ linux:
Categories: Utility;Office;
target:
- deb
+ - rpm
- AppImage
diff --git a/surfsense_desktop/package.json b/surfsense_desktop/package.json
index 638fd3ffc..e2712d8ea 100644
--- a/surfsense_desktop/package.json
+++ b/surfsense_desktop/package.json
@@ -4,15 +4,16 @@
"description": "SurfSense Desktop App",
"main": "dist/main.js",
"scripts": {
- "dev": "pnpm build && concurrently -k \"pnpm --dir ../surfsense_web dev\" \"wait-on http://localhost:3000 && electron .\"",
+ "dev": "pnpm build && concurrently -k \"pnpm --dir ../surfsense_web dev\" \"wait-on http://localhost:3000 && node scripts/electron-dev.mjs\"",
"build": "node scripts/build-electron.mjs",
"pack:dir": "pnpm build && electron-builder --dir --config electron-builder.yml",
+ "pack:dir:linux": "pnpm build && electron-builder --dir --linux --config electron-builder.yml -c.npmRebuild=false",
"dist": "pnpm build && electron-builder --config electron-builder.yml",
"dist:mac": "pnpm build && electron-builder --mac --config electron-builder.yml",
"dist:win": "pnpm build && electron-builder --win --config electron-builder.yml",
- "dist:linux": "pnpm build && electron-builder --linux --config electron-builder.yml",
+ "dist:linux": "pnpm build && electron-builder --linux --config electron-builder.yml -c.npmRebuild=false",
"typecheck": "tsc --noEmit",
- "postinstall": "electron-rebuild"
+ "postinstall": "node scripts/postinstall-rebuild.mjs"
},
"homepage": "https://github.com/MODSetter/SurfSense",
"author": {
diff --git a/surfsense_desktop/scripts/build-electron.mjs b/surfsense_desktop/scripts/build-electron.mjs
index 90d76ef7a..75a3cdf61 100644
--- a/surfsense_desktop/scripts/build-electron.mjs
+++ b/surfsense_desktop/scripts/build-electron.mjs
@@ -132,6 +132,18 @@ async function buildElectron() {
outfile: 'dist/preload.js',
});
+ await build({
+ ...shared,
+ entryPoints: ['src/modules/screen-capture/screen-region-preload.ts'],
+ outfile: 'dist/modules/screen-capture/screen-region-preload.js',
+ });
+
+ await build({
+ ...shared,
+ entryPoints: ['src/modules/screen-capture/window-picker-preload.ts'],
+ outfile: 'dist/modules/screen-capture/window-picker-preload.js',
+ });
+
console.log('Electron build complete');
resolveStandaloneSymlinks();
}
diff --git a/surfsense_desktop/scripts/electron-dev.mjs b/surfsense_desktop/scripts/electron-dev.mjs
new file mode 100644
index 000000000..64be03211
--- /dev/null
+++ b/surfsense_desktop/scripts/electron-dev.mjs
@@ -0,0 +1,24 @@
+/**
+ * Linux dev: (1) ELECTRON_DISABLE_SANDBOX before start — setuid chrome-sandbox in node_modules.
+ * (2) --ozone-platform=x11 — use X11 via XWayland so global shortcuts / GPU warnings match many
+ * Linux Electron setups better than native Wayland. Set SURFSENSE_ELECTRON_WAYLAND=1 to skip (2).
+ * Packaged apps are not launched through this script.
+ */
+import { spawnSync } from 'child_process';
+import { dirname, join } from 'path';
+import { fileURLToPath } from 'url';
+
+const root = join(dirname(fileURLToPath(import.meta.url)), '..');
+const cli = join(root, 'node_modules', 'electron', 'cli.js');
+
+const env = { ...process.env };
+const args = [cli, '.'];
+if (process.platform === 'linux') {
+ env.ELECTRON_DISABLE_SANDBOX = '1';
+ if (env.SURFSENSE_ELECTRON_WAYLAND !== '1') {
+ args.push('--ozone-platform=x11');
+ }
+}
+
+const r = spawnSync(process.execPath, args, { cwd: root, env, stdio: 'inherit' });
+process.exit(r.status === null ? 1 : r.status ?? 0);
diff --git a/surfsense_desktop/scripts/postinstall-rebuild.mjs b/surfsense_desktop/scripts/postinstall-rebuild.mjs
new file mode 100644
index 000000000..d1cfd0732
--- /dev/null
+++ b/surfsense_desktop/scripts/postinstall-rebuild.mjs
@@ -0,0 +1,25 @@
+/**
+ * node-mac-permissions is macOS-only; electron-rebuild would still compile it on Linux/Windows
+ * (missing `make`, wrong platform). We skip rebuild there.
+ */
+import { existsSync } from 'fs';
+import { spawnSync } from 'child_process';
+import { dirname, join } from 'path';
+import { fileURLToPath } from 'url';
+
+const root = join(dirname(fileURLToPath(import.meta.url)), '..');
+
+if (process.platform !== 'darwin') {
+ console.log('[surfsense-desktop] Skipping electron-rebuild on non-macOS (native permissions module is darwin-only).');
+ process.exit(0);
+}
+
+const bin = join(root, 'node_modules', '.bin', 'electron-rebuild');
+
+if (!existsSync(bin)) {
+ console.warn('[surfsense-desktop] electron-rebuild not found in node_modules/.bin, skipping.');
+ process.exit(0);
+}
+
+const result = spawnSync(bin, [], { cwd: root, stdio: 'inherit' });
+process.exit(result.status === null ? 1 : result.status);
diff --git a/surfsense_desktop/src/ipc/channels.ts b/surfsense_desktop/src/ipc/channels.ts
index ed4b49fad..8d2af5107 100644
--- a/surfsense_desktop/src/ipc/channels.ts
+++ b/surfsense_desktop/src/ipc/channels.ts
@@ -11,12 +11,13 @@ export const IPC_CHANNELS = {
REQUEST_ACCESSIBILITY: 'request-accessibility',
REQUEST_SCREEN_RECORDING: 'request-screen-recording',
RESTART_APP: 'restart-app',
- // Autocomplete
- AUTOCOMPLETE_CONTEXT: 'autocomplete-context',
- ACCEPT_SUGGESTION: 'accept-suggestion',
- DISMISS_SUGGESTION: 'dismiss-suggestion',
- SET_AUTOCOMPLETE_ENABLED: 'set-autocomplete-enabled',
- GET_AUTOCOMPLETE_ENABLED: 'get-autocomplete-enabled',
+ CAPTURE_FULL_SCREEN: 'capture-full-screen',
+ SCREEN_REGION_SUBMIT: 'screen-region:submit',
+ SCREEN_REGION_CANCEL: 'screen-region:cancel',
+ WINDOW_PICK_LIST: 'window-pick:list',
+ WINDOW_PICK_SUBMIT: 'window-pick:submit',
+ WINDOW_PICK_CANCEL: 'window-pick:cancel',
+ CHAT_SCREEN_CAPTURE: 'chat:screen-capture',
// Folder sync channels
FOLDER_SYNC_SELECT_FOLDER: 'folder-sync:select-folder',
FOLDER_SYNC_ADD_FOLDER: 'folder-sync:add-folder',
diff --git a/surfsense_desktop/src/ipc/handlers.ts b/surfsense_desktop/src/ipc/handlers.ts
index 2b06c7fb0..d918fd90d 100644
--- a/surfsense_desktop/src/ipc/handlers.ts
+++ b/surfsense_desktop/src/ipc/handlers.ts
@@ -2,10 +2,12 @@ import { app, ipcMain, shell } from 'electron';
import { IPC_CHANNELS } from './channels';
import {
getPermissionsStatus,
+ hasScreenRecordingPermission,
requestAccessibility,
requestScreenRecording,
restartApp,
} from '../modules/permissions';
+import { pickOpenWindowCapture } from '../modules/screen-capture';
import {
selectFolder,
addWatchedFolder,
@@ -27,8 +29,7 @@ import { getShortcuts, setShortcuts, type ShortcutConfig } from '../modules/shor
import { getAutoLaunchState, setAutoLaunch } from '../modules/auto-launch';
import { getActiveSearchSpaceId, setActiveSearchSpaceId } from '../modules/active-search-space';
import { reregisterQuickAsk } from '../modules/quick-ask';
-import { reregisterAutocomplete } from '../modules/autocomplete';
-import { reregisterGeneralAssist } from '../modules/tray';
+import { reregisterGeneralAssist, reregisterScreenshotAssist } from '../modules/tray';
import {
getDistinctId,
getMachineId,
@@ -85,6 +86,15 @@ export function registerIpcHandlers(): void {
restartApp();
});
+ ipcMain.handle(IPC_CHANNELS.CAPTURE_FULL_SCREEN, async () => {
+ if (!hasScreenRecordingPermission()) {
+ requestScreenRecording();
+ return null;
+ }
+ const picked = await pickOpenWindowCapture();
+ return picked?.dataUrl ?? null;
+ });
+
// Folder sync handlers
ipcMain.handle(IPC_CHANNELS.FOLDER_SYNC_SELECT_FOLDER, () => selectFolder());
@@ -192,8 +202,8 @@ export function registerIpcHandlers(): void {
ipcMain.handle(IPC_CHANNELS.SET_SHORTCUTS, async (_event, config: Partial) => {
const updated = await setShortcuts(config);
if (config.generalAssist) await reregisterGeneralAssist();
+ if (config.screenshotAssist) await reregisterScreenshotAssist();
if (config.quickAsk) await reregisterQuickAsk();
- if (config.autocomplete) await reregisterAutocomplete();
trackEvent('desktop_shortcut_updated', {
keys: Object.keys(config),
});
diff --git a/surfsense_desktop/src/main.ts b/surfsense_desktop/src/main.ts
index 399144bed..492c61f17 100644
--- a/surfsense_desktop/src/main.ts
+++ b/surfsense_desktop/src/main.ts
@@ -7,7 +7,6 @@ import { setupDeepLinks, handlePendingDeepLink, hasPendingDeepLink } from './mod
import { setupAutoUpdater } from './modules/auto-updater';
import { setupMenu } from './modules/menu';
import { registerQuickAsk, unregisterQuickAsk } from './modules/quick-ask';
-import { registerAutocomplete, unregisterAutocomplete } from './modules/autocomplete';
import { registerFolderWatcher, unregisterFolderWatcher } from './modules/folder-watcher';
import { registerIpcHandlers } from './ipc/handlers';
import { createTray, destroyTray } from './modules/tray';
@@ -60,7 +59,6 @@ app.whenReady().then(async () => {
}
await registerQuickAsk();
- await registerAutocomplete();
registerFolderWatcher();
setupAutoUpdater();
@@ -94,7 +92,6 @@ app.on('will-quit', async (e) => {
didCleanup = true;
e.preventDefault();
unregisterQuickAsk();
- unregisterAutocomplete();
unregisterFolderWatcher();
destroyTray();
await shutdownAnalytics();
diff --git a/surfsense_desktop/src/modules/autocomplete/index.ts b/surfsense_desktop/src/modules/autocomplete/index.ts
deleted file mode 100644
index d4eb727fd..000000000
--- a/surfsense_desktop/src/modules/autocomplete/index.ts
+++ /dev/null
@@ -1,143 +0,0 @@
-import { clipboard, globalShortcut, ipcMain, screen } from 'electron';
-import { IPC_CHANNELS } from '../../ipc/channels';
-import { getFrontmostApp, getWindowTitle, hasAccessibilityPermission, simulatePaste } from '../platform';
-import { hasScreenRecordingPermission, requestAccessibility, requestScreenRecording } from '../permissions';
-import { captureScreen } from './screenshot';
-import { createSuggestionWindow, destroySuggestion, getSuggestionWindow } from './suggestion-window';
-import { getShortcuts } from '../shortcuts';
-import { getActiveSearchSpaceId } from '../active-search-space';
-import { trackEvent } from '../analytics';
-
-let currentShortcut = '';
-let autocompleteEnabled = true;
-let savedClipboard = '';
-let sourceApp = '';
-
-function isSurfSenseWindow(): boolean {
- const app = getFrontmostApp();
- return app === 'Electron' || app === 'SurfSense' || app === 'surfsense-desktop';
-}
-
-async function triggerAutocomplete(): Promise {
- if (!autocompleteEnabled) return;
- if (isSurfSenseWindow()) return;
-
- if (!hasScreenRecordingPermission()) {
- requestScreenRecording();
- return;
- }
-
- sourceApp = getFrontmostApp();
- const windowTitle = getWindowTitle();
- savedClipboard = clipboard.readText();
-
- const screenshot = await captureScreen();
- if (!screenshot) {
- console.error('[autocomplete] Screenshot capture failed');
- return;
- }
-
- const searchSpaceId = await getActiveSearchSpaceId();
- if (!searchSpaceId) {
- console.warn('[autocomplete] No active search space. Select a search space first.');
- return;
- }
- trackEvent('desktop_autocomplete_triggered', { search_space_id: searchSpaceId });
- const cursor = screen.getCursorScreenPoint();
- const win = createSuggestionWindow(cursor.x, cursor.y);
-
- win.webContents.once('did-finish-load', () => {
- const sw = getSuggestionWindow();
- setTimeout(() => {
- if (sw && !sw.isDestroyed()) {
- sw.webContents.send(IPC_CHANNELS.AUTOCOMPLETE_CONTEXT, {
- screenshot,
- searchSpaceId,
- appName: sourceApp,
- windowTitle,
- });
- }
- }, 300);
- });
-}
-
-async function acceptAndInject(text: string): Promise {
- if (!sourceApp) return;
-
- if (!hasAccessibilityPermission()) {
- requestAccessibility();
- return;
- }
-
- clipboard.writeText(text);
- destroySuggestion();
-
- try {
- await new Promise((r) => setTimeout(r, 50));
- simulatePaste();
- await new Promise((r) => setTimeout(r, 100));
- clipboard.writeText(savedClipboard);
- } catch {
- clipboard.writeText(savedClipboard);
- }
-}
-
-let ipcRegistered = false;
-
-function registerIpcHandlers(): void {
- if (ipcRegistered) return;
- ipcRegistered = true;
-
- ipcMain.handle(IPC_CHANNELS.ACCEPT_SUGGESTION, async (_event, text: string) => {
- trackEvent('desktop_autocomplete_accepted');
- await acceptAndInject(text);
- });
- ipcMain.handle(IPC_CHANNELS.DISMISS_SUGGESTION, () => {
- trackEvent('desktop_autocomplete_dismissed');
- destroySuggestion();
- });
- ipcMain.handle(IPC_CHANNELS.SET_AUTOCOMPLETE_ENABLED, (_event, enabled: boolean) => {
- autocompleteEnabled = enabled;
- if (!enabled) {
- destroySuggestion();
- }
- });
- ipcMain.handle(IPC_CHANNELS.GET_AUTOCOMPLETE_ENABLED, () => autocompleteEnabled);
-}
-
-function autocompleteHandler(): void {
- const sw = getSuggestionWindow();
- if (sw && !sw.isDestroyed()) {
- destroySuggestion();
- return;
- }
- triggerAutocomplete();
-}
-
-async function registerShortcut(): Promise {
- const shortcuts = await getShortcuts();
- currentShortcut = shortcuts.autocomplete;
-
- const ok = globalShortcut.register(currentShortcut, autocompleteHandler);
-
- if (!ok) {
- console.error(`[autocomplete] Failed to register shortcut ${currentShortcut}`);
- } else {
- console.log(`[autocomplete] Registered shortcut ${currentShortcut}`);
- }
-}
-
-export async function registerAutocomplete(): Promise {
- registerIpcHandlers();
- await registerShortcut();
-}
-
-export function unregisterAutocomplete(): void {
- if (currentShortcut) globalShortcut.unregister(currentShortcut);
- destroySuggestion();
-}
-
-export async function reregisterAutocomplete(): Promise {
- unregisterAutocomplete();
- await registerShortcut();
-}
diff --git a/surfsense_desktop/src/modules/autocomplete/screenshot.ts b/surfsense_desktop/src/modules/autocomplete/screenshot.ts
deleted file mode 100644
index 22b7c1b14..000000000
--- a/surfsense_desktop/src/modules/autocomplete/screenshot.ts
+++ /dev/null
@@ -1,27 +0,0 @@
-import { desktopCapturer, screen } from 'electron';
-
-/**
- * Captures the primary display as a base64-encoded PNG data URL.
- * Uses the display's actual size for full-resolution capture.
- */
-export async function captureScreen(): Promise {
- try {
- const primaryDisplay = screen.getPrimaryDisplay();
- const { width, height } = primaryDisplay.size;
-
- const sources = await desktopCapturer.getSources({
- types: ['screen'],
- thumbnailSize: { width, height },
- });
-
- if (!sources.length) {
- console.error('[screenshot] No screen sources found');
- return null;
- }
-
- return sources[0].thumbnail.toDataURL();
- } catch (err) {
- console.error('[screenshot] Failed to capture screen:', err);
- return null;
- }
-}
diff --git a/surfsense_desktop/src/modules/autocomplete/suggestion-window.ts b/surfsense_desktop/src/modules/autocomplete/suggestion-window.ts
deleted file mode 100644
index 8f61b2901..000000000
--- a/surfsense_desktop/src/modules/autocomplete/suggestion-window.ts
+++ /dev/null
@@ -1,112 +0,0 @@
-import { BrowserWindow, screen, shell } from 'electron';
-import path from 'path';
-import { getServerPort } from '../server';
-
-const TOOLTIP_WIDTH = 420;
-const TOOLTIP_HEIGHT = 38;
-const MAX_HEIGHT = 400;
-
-let suggestionWindow: BrowserWindow | null = null;
-let resizeTimer: ReturnType | null = null;
-let cursorOrigin = { x: 0, y: 0 };
-
-const CURSOR_GAP = 20;
-
-function positionOnScreen(cursorX: number, cursorY: number, w: number, h: number): { x: number; y: number } {
- const display = screen.getDisplayNearestPoint({ x: cursorX, y: cursorY });
- const { x: dx, y: dy, width: dw, height: dh } = display.workArea;
-
- const x = Math.max(dx, Math.min(cursorX, dx + dw - w));
-
- const spaceBelow = (dy + dh) - (cursorY + CURSOR_GAP);
- const y = spaceBelow >= h
- ? cursorY + CURSOR_GAP
- : cursorY - h - CURSOR_GAP;
-
- return { x, y: Math.max(dy, y) };
-}
-
-function stopResizePolling(): void {
- if (resizeTimer) { clearInterval(resizeTimer); resizeTimer = null; }
-}
-
-function startResizePolling(win: BrowserWindow): void {
- stopResizePolling();
- let lastH = 0;
- resizeTimer = setInterval(async () => {
- if (!win || win.isDestroyed()) { stopResizePolling(); return; }
- try {
- const h: number = await win.webContents.executeJavaScript(
- `document.body.scrollHeight`
- );
- if (h > 0 && h !== lastH) {
- lastH = h;
- const clamped = Math.min(h, MAX_HEIGHT);
- const pos = positionOnScreen(cursorOrigin.x, cursorOrigin.y, TOOLTIP_WIDTH, clamped);
- win.setBounds({ x: pos.x, y: pos.y, width: TOOLTIP_WIDTH, height: clamped });
- }
- } catch {}
- }, 150);
-}
-
-export function getSuggestionWindow(): BrowserWindow | null {
- return suggestionWindow;
-}
-
-export function destroySuggestion(): void {
- stopResizePolling();
- if (suggestionWindow && !suggestionWindow.isDestroyed()) {
- suggestionWindow.close();
- }
- suggestionWindow = null;
-}
-
-export function createSuggestionWindow(x: number, y: number): BrowserWindow {
- destroySuggestion();
- cursorOrigin = { x, y };
-
- const pos = positionOnScreen(x, y, TOOLTIP_WIDTH, TOOLTIP_HEIGHT);
-
- suggestionWindow = new BrowserWindow({
- width: TOOLTIP_WIDTH,
- height: TOOLTIP_HEIGHT,
- x: pos.x,
- y: pos.y,
- frame: false,
- transparent: true,
- focusable: false,
- alwaysOnTop: true,
- skipTaskbar: true,
- hasShadow: true,
- type: 'panel',
- webPreferences: {
- preload: path.join(__dirname, 'preload.js'),
- contextIsolation: true,
- nodeIntegration: false,
- sandbox: true,
- },
- show: false,
- });
-
- suggestionWindow.loadURL(`http://localhost:${getServerPort()}/desktop/suggestion?t=${Date.now()}`);
-
- suggestionWindow.once('ready-to-show', () => {
- suggestionWindow?.showInactive();
- if (suggestionWindow) startResizePolling(suggestionWindow);
- });
-
- suggestionWindow.webContents.setWindowOpenHandler(({ url }) => {
- if (url.startsWith('http://localhost')) {
- return { action: 'allow' };
- }
- shell.openExternal(url);
- return { action: 'deny' };
- });
-
- suggestionWindow.on('closed', () => {
- stopResizePolling();
- suggestionWindow = null;
- });
-
- return suggestionWindow;
-}
diff --git a/surfsense_desktop/src/modules/general-assist.ts b/surfsense_desktop/src/modules/general-assist.ts
new file mode 100644
index 000000000..7d202caa2
--- /dev/null
+++ b/surfsense_desktop/src/modules/general-assist.ts
@@ -0,0 +1,5 @@
+import { showMainWindow } from './window';
+
+export function runGeneralAssistShortcut(): void {
+ showMainWindow('shortcut');
+}
diff --git a/surfsense_desktop/src/modules/screen-capture/index.ts b/surfsense_desktop/src/modules/screen-capture/index.ts
new file mode 100644
index 000000000..6c1c75509
--- /dev/null
+++ b/surfsense_desktop/src/modules/screen-capture/index.ts
@@ -0,0 +1,7 @@
+/**
+ * Window capture for Screenshot Assist and chat fullscreen: single-session
+ * desktopCapturer, region overlay, and shortcut entry point.
+ */
+export { pickOpenWindowCapture, type PickedWindowResult } from './window-picker';
+export { pickScreenRegion, captureCurrentDisplayDataUrl } from './screen-region-picker';
+export { runScreenshotAssistShortcut } from './screenshot-assist';
diff --git a/surfsense_desktop/src/modules/screen-capture/screen-region-picker.ts b/surfsense_desktop/src/modules/screen-capture/screen-region-picker.ts
new file mode 100644
index 000000000..fd771b0f7
--- /dev/null
+++ b/surfsense_desktop/src/modules/screen-capture/screen-region-picker.ts
@@ -0,0 +1,335 @@
+import { BrowserWindow, desktopCapturer, nativeImage, screen } from 'electron';
+import path from 'path';
+import { IPC_CHANNELS } from '../../ipc/channels';
+function fitNativeImageToWorkArea(img: Electron.NativeImage, display: Electron.Display): Electron.NativeImage {
+ const wa = display.workArea;
+ const { width: iw, height: ih } = img.getSize();
+ const scale = Math.min(1, wa.width / iw, wa.height / ih);
+ if (scale >= 1) return img;
+ return img.resize({
+ width: Math.max(1, Math.floor(iw * scale)),
+ height: Math.max(1, Math.floor(ih * scale)),
+ quality: 'best',
+ });
+}
+
+// One getSources per pick; overlay and final crop share that bitmap (avoids a second portal session, e.g. Wayland).
+
+let pickInProgress = false;
+
+type DisplayCaptureSnapshot = {
+ dataUrl: string;
+ width: number;
+ height: number;
+};
+
+async function captureDisplaySnapshot(display: Electron.Display): Promise {
+ try {
+ const sf = display.scaleFactor || 1;
+ const tw = Math.max(1, Math.round(display.size.width * sf));
+ const th = Math.max(1, Math.round(display.size.height * sf));
+ const sources = await desktopCapturer.getSources({
+ types: ['screen'],
+ thumbnailSize: { width: tw, height: th },
+ });
+ if (!sources.length) return null;
+ const idStr = String(display.id);
+ let chosen =
+ sources.find((s) => s.display_id === idStr) ||
+ sources.find((s) => s.display_id && s.display_id === idStr) ||
+ null;
+ if (!chosen && screen.getPrimaryDisplay().id === display.id) {
+ chosen = sources[0];
+ }
+ if (!chosen) chosen = sources[0];
+ const dataUrl = chosen.thumbnail.toDataURL();
+ const { width, height } = chosen.thumbnail.getSize();
+ return { dataUrl, width, height };
+ } catch {
+ return null;
+ }
+}
+
+export async function captureCurrentDisplayDataUrl(): Promise {
+ const display = screen.getDisplayNearestPoint(screen.getCursorScreenPoint());
+ const snapshot = await captureDisplaySnapshot(display);
+ return snapshot?.dataUrl ?? null;
+}
+
+function buildInjectScript(dataUrl: string, iw: number, ih: number): string {
+ return `(() => {
+ const api = window.surfsenseScreenRegion;
+ if (!api) return;
+ const dataUrl = ${JSON.stringify(dataUrl)};
+ const iw = ${iw};
+ const ih = ${ih};
+ document.body.style.margin = '0';
+ document.body.style.overflow = 'hidden';
+ document.body.style.background = '#000';
+ const img = document.createElement('img');
+ img.draggable = false;
+ img.src = dataUrl;
+ img.style.cssText = 'position:fixed;inset:0;width:100vw;height:100vh;object-fit:fill;user-select:none;pointer-events:none;';
+ const veil = document.createElement('div');
+ veil.style.cssText = 'position:fixed;inset:0;cursor:crosshair;background:rgba(0,0,0,0.15);';
+ const sel = document.createElement('div');
+ sel.style.cssText = 'position:fixed;border:2px solid #38bdf8;box-shadow:0 0 0 9999px rgba(0,0,0,0.45);display:none;pointer-events:none;z-index:2;';
+ document.body.appendChild(img);
+ document.body.appendChild(veil);
+ document.body.appendChild(sel);
+ let ax = 0, ay = 0, dragging = false;
+ function show(x0, y0, x1, y1) {
+ const l = Math.min(x0, x1), t = Math.min(y0, y1);
+ const w = Math.abs(x1 - x0), h = Math.abs(y1 - y0);
+ if (w < 2 || h < 2) { sel.style.display = 'none'; return; }
+ sel.style.display = 'block';
+ sel.style.left = l + 'px';
+ sel.style.top = t + 'px';
+ sel.style.width = w + 'px';
+ sel.style.height = h + 'px';
+ }
+ function mapRect(l, t, w, h) {
+ const vw = window.innerWidth, vh = window.innerHeight;
+ const sx = Math.round((l / vw) * iw);
+ const sy = Math.round((t / vh) * ih);
+ const sw = Math.max(1, Math.round((w / vw) * iw));
+ const sh = Math.max(1, Math.round((h / vh) * ih));
+ const cx = Math.min(Math.max(0, sx), iw - 1);
+ const cy = Math.min(Math.max(0, sy), ih - 1);
+ const cw = Math.min(sw, iw - cx);
+ const ch = Math.min(sh, ih - cy);
+ return { x: cx, y: cy, width: cw, height: ch };
+ }
+ function endDrag(clientX, clientY, pointerId) {
+ if (!dragging) return;
+ dragging = false;
+ if (typeof pointerId === 'number' && pointerId >= 0) {
+ try { veil.releasePointerCapture(pointerId); } catch (_) {}
+ }
+ const l = Math.min(ax, clientX), t = Math.min(ay, clientY);
+ const w = Math.abs(clientX - ax), h = Math.abs(clientY - ay);
+ if (w < 4 || h < 4) { sel.style.display = 'none'; return; }
+ api.submit(mapRect(l, t, w, h));
+ }
+ veil.addEventListener('pointerdown', (e) => {
+ if (e.button !== 0) return;
+ try { veil.setPointerCapture(e.pointerId); } catch (_) {}
+ dragging = true;
+ ax = e.clientX; ay = e.clientY;
+ show(ax, ay, ax, ay);
+ });
+ veil.addEventListener('pointermove', (e) => {
+ if (!dragging) return;
+ show(ax, ay, e.clientX, e.clientY);
+ });
+ veil.addEventListener('pointerup', (e) => {
+ endDrag(e.clientX, e.clientY, e.pointerId);
+ });
+ window.addEventListener('pointerup', (e) => {
+ endDrag(e.clientX, e.clientY, e.pointerId);
+ });
+ document.addEventListener(
+ 'mouseup',
+ (e) => {
+ endDrag(e.clientX, e.clientY, -1);
+ },
+ true
+ );
+ veil.addEventListener('pointercancel', (e) => {
+ if (!dragging) return;
+ dragging = false;
+ try { veil.releasePointerCapture(e.pointerId); } catch (_) {}
+ sel.style.display = 'none';
+ });
+ window.addEventListener('keydown', (e) => {
+ if (e.key === 'Escape') { api.cancel(); return; }
+ if (e.key === 'Enter' && sel.style.display === 'block') {
+ const l = parseFloat(sel.style.left), t = parseFloat(sel.style.top);
+ const w = parseFloat(sel.style.width), h = parseFloat(sel.style.height);
+ if (w >= 4 && h >= 4) api.submit(mapRect(l, t, w, h));
+ }
+ });
+ })();`;
+}
+
+export function pickScreenRegion(opts?: { windowDataUrl?: string }): Promise {
+ if (pickInProgress) return Promise.resolve(null);
+ pickInProgress = true;
+
+ return new Promise((resolve) => {
+ const display = screen.getDisplayNearestPoint(screen.getCursorScreenPoint());
+ let settled = false;
+ let overlay: BrowserWindow | null = null;
+ /** webContents for listener removal after `BrowserWindow` may already be destroyed. */
+ let overlayWc: Electron.WebContents | null = null;
+
+ const cleanupListeners = () => {
+ const wc = overlayWc;
+ overlayWc = null;
+ if (!wc || wc.isDestroyed()) return;
+ wc.removeListener('before-input-event', onBeforeInput);
+ wc.ipc.removeListener(IPC_CHANNELS.SCREEN_REGION_SUBMIT, onSubmit);
+ wc.ipc.removeListener(IPC_CHANNELS.SCREEN_REGION_CANCEL, onCancel);
+ };
+
+ const finish = (result: string | null) => {
+ if (settled) return;
+ settled = true;
+ pickInProgress = false;
+ cleanupListeners();
+ if (overlay && !overlay.isDestroyed()) {
+ overlay.removeAllListeners('closed');
+ overlay.close();
+ }
+ overlay = null;
+ resolve(result);
+ };
+
+ let snapshot: DisplayCaptureSnapshot | null = null;
+ let cropSource: Electron.NativeImage | null = null;
+
+ const onSubmit = (
+ _event: Electron.IpcMainEvent,
+ rect: { x: number; y: number; width: number; height: number }
+ ) => {
+ if (settled || !overlay || overlay.isDestroyed()) return;
+ if (!rect || rect.width < 1 || rect.height < 1) {
+ finish(null);
+ return;
+ }
+ if (!snapshot || !cropSource) {
+ finish(null);
+ return;
+ }
+ try {
+ const iw = snapshot.width;
+ const ih = snapshot.height;
+ const { width: cw, height: ch } = cropSource.getSize();
+ const scaleX = cw / iw;
+ const scaleY = ch / ih;
+ const ox = Math.floor(rect.x * scaleX);
+ const oy = Math.floor(rect.y * scaleY);
+ const ow = Math.min(Math.floor(rect.width * scaleX), cw - ox);
+ const oh = Math.min(Math.floor(rect.height * scaleY), ch - oy);
+ const cropped = cropSource.crop({
+ x: ox,
+ y: oy,
+ width: Math.max(1, ow),
+ height: Math.max(1, oh),
+ });
+ finish(cropped.toDataURL());
+ } catch {
+ finish(null);
+ }
+ };
+
+ const onCancel = (_event: Electron.IpcMainEvent) => {
+ if (settled || !overlay || overlay.isDestroyed()) return;
+ finish(null);
+ };
+
+ const onBeforeInput = (_event: Electron.Event, input: Electron.Input) => {
+ if (input.type === 'keyDown' && input.key === 'Escape') {
+ finish(null);
+ }
+ };
+
+ const openOverlay = (
+ cap: DisplayCaptureSnapshot,
+ crop: Electron.NativeImage,
+ bounds: { x: number; y: number; width: number; height: number }
+ ) => {
+ snapshot = cap;
+ cropSource = crop;
+
+ overlay = new BrowserWindow({
+ x: bounds.x,
+ y: bounds.y,
+ width: bounds.width,
+ height: bounds.height,
+ frame: false,
+ transparent: true,
+ fullscreenable: false,
+ skipTaskbar: true,
+ alwaysOnTop: true,
+ focusable: true,
+ show: false,
+ autoHideMenuBar: true,
+ backgroundColor: '#00000000',
+ webPreferences: {
+ preload: path.join(__dirname, 'modules', 'screen-capture', 'screen-region-preload.js'),
+ contextIsolation: true,
+ nodeIntegration: false,
+ sandbox: true,
+ },
+ });
+
+ overlayWc = overlay.webContents;
+ overlayWc.on('before-input-event', onBeforeInput);
+ overlayWc.ipc.on(IPC_CHANNELS.SCREEN_REGION_SUBMIT, onSubmit);
+ overlayWc.ipc.on(IPC_CHANNELS.SCREEN_REGION_CANCEL, onCancel);
+
+ overlay.setIgnoreMouseEvents(false);
+ overlay.loadURL(
+ 'data:text/html;charset=utf-8,' +
+ encodeURIComponent('')
+ );
+
+ overlay.on('closed', () => {
+ if (!settled) finish(null);
+ });
+
+ overlay.webContents.once('did-finish-load', () => {
+ if (!overlay || overlay.isDestroyed()) return;
+ overlay.webContents
+ .executeJavaScript(buildInjectScript(cap.dataUrl, cap.width, cap.height), true)
+ .then(() => {
+ overlay?.show();
+ overlay?.focus();
+ })
+ .catch(() => {
+ finish(null);
+ });
+ });
+ };
+
+ void (async () => {
+ try {
+ if (opts?.windowDataUrl) {
+ const fullRes = nativeImage.createFromDataURL(opts.windowDataUrl);
+ if (fullRes.isEmpty()) {
+ finish(null);
+ return;
+ }
+ const fitted = fitNativeImageToWorkArea(fullRes, display);
+ const fw = fitted.getSize().width;
+ const fh = fitted.getSize().height;
+ const wa = display.workArea;
+ const x = wa.x + Math.floor((wa.width - fw) / 2);
+ const y = wa.y + Math.floor((wa.height - fh) / 2);
+ openOverlay(
+ { dataUrl: fitted.toDataURL(), width: fw, height: fh },
+ fullRes,
+ { x, y, width: fw, height: fh }
+ );
+ return;
+ }
+
+ const cap = await captureDisplaySnapshot(display);
+ if (!cap) {
+ finish(null);
+ return;
+ }
+ const crop = nativeImage.createFromDataURL(cap.dataUrl);
+ openOverlay(cap, crop, {
+ x: display.bounds.x,
+ y: display.bounds.y,
+ width: display.bounds.width,
+ height: display.bounds.height,
+ });
+ } catch {
+ finish(null);
+ }
+ })();
+ });
+}
diff --git a/surfsense_desktop/src/modules/screen-capture/screen-region-preload.ts b/surfsense_desktop/src/modules/screen-capture/screen-region-preload.ts
new file mode 100644
index 000000000..4263e0f6e
--- /dev/null
+++ b/surfsense_desktop/src/modules/screen-capture/screen-region-preload.ts
@@ -0,0 +1,11 @@
+import { contextBridge, ipcRenderer } from 'electron';
+import { IPC_CHANNELS } from '../../ipc/channels';
+
+contextBridge.exposeInMainWorld('surfsenseScreenRegion', {
+ submit: (rect: { x: number; y: number; width: number; height: number }) => {
+ ipcRenderer.send(IPC_CHANNELS.SCREEN_REGION_SUBMIT, rect);
+ },
+ cancel: () => {
+ ipcRenderer.send(IPC_CHANNELS.SCREEN_REGION_CANCEL);
+ },
+});
diff --git a/surfsense_desktop/src/modules/screen-capture/screenshot-assist.ts b/surfsense_desktop/src/modules/screen-capture/screenshot-assist.ts
new file mode 100644
index 000000000..171b98a57
--- /dev/null
+++ b/surfsense_desktop/src/modules/screen-capture/screenshot-assist.ts
@@ -0,0 +1,26 @@
+import { IPC_CHANNELS } from '../../ipc/channels';
+import { trackEvent } from '../analytics';
+import { pickScreenRegion } from './screen-region-picker';
+import { pickOpenWindowCapture } from './window-picker';
+import { getMainWindow, showMainWindow } from '../window';
+import { hasScreenRecordingPermission, requestScreenRecording } from '../permissions';
+
+export async function runScreenshotAssistShortcut(): Promise {
+ if (!hasScreenRecordingPermission()) {
+ requestScreenRecording();
+ return;
+ }
+
+ const picked = await pickOpenWindowCapture();
+ if (!picked) return;
+
+ const url = await pickScreenRegion({ windowDataUrl: picked.dataUrl });
+ if (!url) return;
+
+ showMainWindow('shortcut');
+ const mw = getMainWindow();
+ if (mw && !mw.isDestroyed()) {
+ mw.webContents.send(IPC_CHANNELS.CHAT_SCREEN_CAPTURE, url);
+ trackEvent('desktop_screenshot_assist_region_to_chat', {});
+ }
+}
diff --git a/surfsense_desktop/src/modules/screen-capture/window-picker-preload.ts b/surfsense_desktop/src/modules/screen-capture/window-picker-preload.ts
new file mode 100644
index 000000000..dd0acd81e
--- /dev/null
+++ b/surfsense_desktop/src/modules/screen-capture/window-picker-preload.ts
@@ -0,0 +1,15 @@
+import { contextBridge, ipcRenderer } from 'electron';
+import { IPC_CHANNELS } from '../../ipc/channels';
+
+contextBridge.exposeInMainWorld('surfsenseWindowPick', {
+ list: () =>
+ ipcRenderer.invoke(IPC_CHANNELS.WINDOW_PICK_LIST) as Promise<
+ { id: string; name: string; thumbDataUrl: string }[]
+ >,
+ submit: (sourceId: string) => {
+ ipcRenderer.send(IPC_CHANNELS.WINDOW_PICK_SUBMIT, sourceId);
+ },
+ cancel: () => {
+ ipcRenderer.send(IPC_CHANNELS.WINDOW_PICK_CANCEL);
+ },
+});
diff --git a/surfsense_desktop/src/modules/screen-capture/window-picker.ts b/surfsense_desktop/src/modules/screen-capture/window-picker.ts
new file mode 100644
index 000000000..b66e23c5c
--- /dev/null
+++ b/surfsense_desktop/src/modules/screen-capture/window-picker.ts
@@ -0,0 +1,244 @@
+import { BrowserWindow, desktopCapturer, ipcMain, screen } from 'electron';
+import path from 'path';
+import { IPC_CHANNELS } from '../../ipc/channels';
+
+let pickInProgress = false;
+
+const PREVIEW_THUMB = { width: 280, height: 180 } as const;
+
+function maxCaptureThumbSize(): { width: number; height: number } {
+ const d = screen.getPrimaryDisplay();
+ const sf = d.scaleFactor || 1;
+ const w = Math.min(3840, Math.max(1280, Math.round(d.size.width * sf)));
+ const h = Math.min(2160, Math.max(720, Math.round(d.size.height * sf)));
+ return { width: w, height: h };
+}
+
+function isDesktopWindowSourceId(s: string): boolean {
+ return typeof s === 'string' && s.startsWith('window:');
+}
+
+export type PickedWindowResult = {
+ sourceId: string;
+ /** Same pixels as the one `desktopCapturer` snapshot (max thumbnail size). */
+ dataUrl: string;
+};
+
+function buildPickerInjectScript(): string {
+ return `(async function () {
+ const api = window.surfsenseWindowPick;
+ if (!api) return;
+ const items = await api.list();
+ document.body.style.cssText =
+ 'margin:0;font-family:system-ui,-apple-system,sans-serif;background:#0f172a;color:#e2e8f0;min-height:100vh;padding:16px;box-sizing:border-box;';
+ const top = document.createElement('div');
+ top.style.cssText =
+ 'display:flex;justify-content:space-between;align-items:center;margin-bottom:12px;flex-wrap:wrap;gap:8px;';
+ const t = document.createElement('strong');
+ t.textContent = 'Open windows';
+ const hint = document.createElement('span');
+ hint.style.cssText = 'opacity:0.75;font-size:13px;';
+ hint.textContent = 'Click a window · Esc to cancel';
+ top.appendChild(t);
+ top.appendChild(hint);
+ document.body.appendChild(top);
+ if (!items || !items.length) {
+ const p = document.createElement('p');
+ p.style.cssText = 'line-height:1.5;max-width:42rem;';
+ p.textContent =
+ 'No windows were returned by the system. On Linux, allow screen capture when prompted. If other apps are open, try again.';
+ document.body.appendChild(p);
+ return;
+ }
+ const grid = document.createElement('div');
+ grid.style.cssText =
+ 'display:grid;grid-template-columns:repeat(auto-fill,minmax(200px,1fr));gap:12px;max-height:calc(100vh - 88px);overflow:auto;padding-bottom:8px;';
+ for (const it of items) {
+ const card = document.createElement('button');
+ card.type = 'button';
+ card.style.cssText =
+ 'text-align:left;background:#1e293b;border:1px solid #334155;border-radius:8px;padding:8px;cursor:pointer;color:inherit;';
+ card.addEventListener('mouseenter', function () {
+ card.style.borderColor = '#38bdf8';
+ });
+ card.addEventListener('mouseleave', function () {
+ card.style.borderColor = '#334155';
+ });
+ const img = document.createElement('img');
+ img.alt = '';
+ img.src =
+ it.thumbDataUrl ||
+ 'data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///ywAAAAAAQABAAACAUwAOw==';
+ img.style.cssText =
+ 'width:100%;height:100px;object-fit:cover;border-radius:4px;background:#000;display:block;';
+ const cap = document.createElement('div');
+ cap.textContent = it.name || '(untitled)';
+ cap.style.cssText =
+ 'margin-top:6px;font-size:12px;line-height:1.35;overflow:hidden;text-overflow:ellipsis;display:-webkit-box;-webkit-line-clamp:2;-webkit-box-orient:vertical;';
+ card.appendChild(img);
+ card.appendChild(cap);
+ card.addEventListener('click', function () {
+ api.submit(it.id);
+ });
+ grid.appendChild(card);
+ }
+ document.body.appendChild(grid);
+ window.addEventListener('keydown', function (e) {
+ if (e.key === 'Escape') api.cancel();
+ });
+ })();`;
+}
+
+/**
+ * One OS / Chromium capture session: `getSources` runs once (important on Wayland /
+ * PipeWire so the portal is not opened again for the same flow). Opens our grid to
+ * choose a window; resolves with the chosen snapshot for region or full-frame use.
+ */
+export function pickOpenWindowCapture(): Promise {
+ if (pickInProgress) return Promise.resolve(null);
+ pickInProgress = true;
+
+ return new Promise((resolve) => {
+ let settled = false;
+ let picker: BrowserWindow | null = null;
+ let pickerWc: Electron.WebContents | null = null;
+ /** Filled once before the grid runs — reused for list + final image (no second getSources). */
+ let sessionSources: Electron.DesktopCapturerSource[] = [];
+
+ const finish = (result: PickedWindowResult | null) => {
+ if (settled) return;
+ settled = true;
+ pickInProgress = false;
+ ipcMain.removeHandler(IPC_CHANNELS.WINDOW_PICK_LIST);
+ const wc = pickerWc;
+ pickerWc = null;
+ if (wc && !wc.isDestroyed()) {
+ wc.removeListener('before-input-event', onBeforeInput);
+ wc.ipc.removeListener(IPC_CHANNELS.WINDOW_PICK_SUBMIT, onSubmit);
+ wc.ipc.removeListener(IPC_CHANNELS.WINDOW_PICK_CANCEL, onCancel);
+ }
+ if (picker && !picker.isDestroyed()) {
+ picker.removeAllListeners('closed');
+ picker.close();
+ }
+ picker = null;
+ resolve(result);
+ };
+
+ const onSubmit = (_event: Electron.IpcMainEvent, sourceId: string) => {
+ if (settled || !picker || picker.isDestroyed()) return;
+ if (!isDesktopWindowSourceId(sourceId)) {
+ finish(null);
+ return;
+ }
+ const hit = sessionSources.find((s) => s.id === sourceId);
+ if (!hit || hit.thumbnail.isEmpty()) {
+ finish(null);
+ return;
+ }
+ finish({ sourceId, dataUrl: hit.thumbnail.toDataURL() });
+ };
+
+ const onCancel = () => {
+ if (settled || !picker || picker.isDestroyed()) return;
+ finish(null);
+ };
+
+ const onBeforeInput = (_event: Electron.Event, input: Electron.Input) => {
+ if (input.type === 'keyDown' && input.key === 'Escape') {
+ finish(null);
+ }
+ };
+
+ ipcMain.handle(IPC_CHANNELS.WINDOW_PICK_LIST, async () => {
+ return sessionSources.map((s, i) => {
+ let thumbDataUrl = '';
+ if (!s.thumbnail.isEmpty()) {
+ try {
+ const sm = s.thumbnail.resize({
+ width: PREVIEW_THUMB.width,
+ height: PREVIEW_THUMB.height,
+ quality: 'good',
+ });
+ thumbDataUrl = sm.toDataURL();
+ } catch {
+ thumbDataUrl = s.thumbnail.toDataURL();
+ }
+ }
+ return {
+ id: s.id,
+ name: (s.name || '').trim() || `Window ${i + 1}`,
+ thumbDataUrl,
+ };
+ });
+ });
+
+ picker = new BrowserWindow({
+ width: 760,
+ height: 560,
+ show: false,
+ center: true,
+ autoHideMenuBar: true,
+ title: 'SurfSense — choose window',
+ webPreferences: {
+ preload: path.join(__dirname, 'modules', 'screen-capture', 'window-picker-preload.js'),
+ contextIsolation: true,
+ nodeIntegration: false,
+ sandbox: true,
+ },
+ });
+
+ pickerWc = picker.webContents;
+
+ pickerWc.on('before-input-event', onBeforeInput);
+ pickerWc.ipc.on(IPC_CHANNELS.WINDOW_PICK_SUBMIT, onSubmit);
+ pickerWc.ipc.on(IPC_CHANNELS.WINDOW_PICK_CANCEL, onCancel);
+
+ picker.on('closed', () => {
+ if (!settled) finish(null);
+ });
+
+ picker
+ .loadURL(
+ 'data:text/html;charset=utf-8,' +
+ encodeURIComponent('')
+ )
+ .catch(() => finish(null));
+
+ picker.webContents.once('did-finish-load', () => {
+ void (async () => {
+ if (!picker || picker.isDestroyed()) return;
+ let selfId = '';
+ try {
+ selfId = picker.getMediaSourceId();
+ } catch {
+ selfId = '';
+ }
+ try {
+ const { width, height } = maxCaptureThumbSize();
+ const sources = await desktopCapturer.getSources({
+ types: ['window'],
+ thumbnailSize: { width, height },
+ fetchWindowIcons: false,
+ });
+ sessionSources = sources.filter((s) => !(selfId && s.id === selfId));
+ } catch {
+ sessionSources = [];
+ }
+ if (sessionSources.length === 1) {
+ const only = sessionSources[0];
+ if (!only.thumbnail.isEmpty()) {
+ finish({ sourceId: only.id, dataUrl: only.thumbnail.toDataURL() });
+ return;
+ }
+ }
+ try {
+ await picker.webContents.executeJavaScript(buildPickerInjectScript(), true);
+ if (!picker.isDestroyed()) picker.show();
+ } catch {
+ finish(null);
+ }
+ })();
+ });
+ });
+}
diff --git a/surfsense_desktop/src/modules/shortcuts.ts b/surfsense_desktop/src/modules/shortcuts.ts
index 6948a005e..64687f7db 100644
--- a/surfsense_desktop/src/modules/shortcuts.ts
+++ b/surfsense_desktop/src/modules/shortcuts.ts
@@ -1,13 +1,13 @@
export interface ShortcutConfig {
generalAssist: string;
quickAsk: string;
- autocomplete: string;
+ screenshotAssist: string;
}
const DEFAULTS: ShortcutConfig = {
generalAssist: 'CommandOrControl+Shift+S',
quickAsk: 'CommandOrControl+Alt+S',
- autocomplete: 'CommandOrControl+Shift+Space',
+ screenshotAssist: 'CommandOrControl+Shift+Space',
};
const STORE_KEY = 'shortcuts';
@@ -27,14 +27,30 @@ async function getStore() {
export async function getShortcuts(): Promise {
const s = await getStore();
- const stored = s.get(STORE_KEY) as Partial | undefined;
- return { ...DEFAULTS, ...stored };
+ const raw = (s.get(STORE_KEY) as Record | undefined) ?? {};
+ const legacyAutocomplete = raw.autocomplete;
+ const { autocomplete: _drop, ...rest } = raw;
+ let merged: ShortcutConfig = { ...DEFAULTS, ...rest };
+ if (
+ typeof legacyAutocomplete === 'string' &&
+ legacyAutocomplete.length > 0 &&
+ !('screenshotAssist' in raw)
+ ) {
+ merged = { ...merged, screenshotAssist: legacyAutocomplete };
+ s.set(STORE_KEY, {
+ generalAssist: merged.generalAssist,
+ quickAsk: merged.quickAsk,
+ screenshotAssist: merged.screenshotAssist,
+ });
+ }
+ return merged;
}
export async function setShortcuts(config: Partial): Promise {
const s = await getStore();
- const current = (s.get(STORE_KEY) as ShortcutConfig) ?? DEFAULTS;
- const merged = { ...current, ...config };
+ const raw = (s.get(STORE_KEY) as Record | undefined) ?? {};
+ const { autocomplete: _drop, ...current } = raw;
+ const merged = { ...DEFAULTS, ...current, ...config };
s.set(STORE_KEY, merged);
return merged;
}
diff --git a/surfsense_desktop/src/modules/tray.ts b/surfsense_desktop/src/modules/tray.ts
index 88444cc54..5fb1acbdf 100644
--- a/surfsense_desktop/src/modules/tray.ts
+++ b/surfsense_desktop/src/modules/tray.ts
@@ -1,13 +1,16 @@
-import { app, globalShortcut, Menu, nativeImage, Tray } from 'electron';
+import { app, globalShortcut, Menu, nativeImage, Tray, type NativeImage } from 'electron';
import path from 'path';
-import { getMainWindow, createMainWindow } from './window';
+import { runGeneralAssistShortcut } from './general-assist';
+import { runScreenshotAssistShortcut } from './screen-capture';
+import { showMainWindow } from './window';
import { getShortcuts } from './shortcuts';
import { trackEvent } from './analytics';
let tray: Tray | null = null;
-let currentShortcut: string | null = null;
+let registeredGeneralAssist: string | null = null;
+let registeredScreenshotAssist: string | null = null;
-function getTrayIcon(): nativeImage {
+function getTrayIcon(): NativeImage {
const iconName = process.platform === 'win32' ? 'icon.ico' : 'icon.png';
const iconPath = app.isPackaged
? path.join(process.resourcesPath, 'assets', iconName)
@@ -16,34 +19,29 @@ function getTrayIcon(): nativeImage {
return img.resize({ width: 16, height: 16 });
}
-function showMainWindow(source: 'tray_click' | 'tray_menu' | 'shortcut' = 'tray_click'): void {
- const existing = getMainWindow();
- const reopened = !existing || existing.isDestroyed();
- if (reopened) {
- createMainWindow('/dashboard');
- } else {
- existing.show();
- existing.focus();
+function registerOne(
+ previous: string | null,
+ accelerator: string,
+ onFire: () => void | Promise,
+ label: string
+): string | null {
+ if (previous) {
+ globalShortcut.unregister(previous);
}
- trackEvent('desktop_main_window_shown', { source, reopened });
-}
-
-function registerShortcut(accelerator: string): void {
- if (currentShortcut) {
- globalShortcut.unregister(currentShortcut);
- currentShortcut = null;
- }
- if (!accelerator) return;
+ if (!accelerator) return null;
try {
- const ok = globalShortcut.register(accelerator, () => showMainWindow('shortcut'));
+ const ok = globalShortcut.register(accelerator, () => {
+ void Promise.resolve(onFire());
+ });
if (ok) {
- currentShortcut = accelerator;
- } else {
- console.warn(`[tray] Failed to register General Assist shortcut: ${accelerator}`);
+ console.log(`[hotkeys] Register ${label} ${accelerator}: OK`);
+ return accelerator;
}
+ console.warn(`[hotkeys] Register ${label} ${accelerator}: FAILED (OS or another app may own this chord)`);
} catch (err) {
- console.error(`[tray] Error registering General Assist shortcut:`, err);
+ console.error(`[tray] Error registering ${label} shortcut:`, err);
}
+ return null;
}
export async function createTray(): Promise {
@@ -68,18 +66,48 @@ export async function createTray(): Promise {
tray.on('double-click', () => showMainWindow('tray_click'));
const shortcuts = await getShortcuts();
- registerShortcut(shortcuts.generalAssist);
+ registeredGeneralAssist = registerOne(
+ null,
+ shortcuts.generalAssist,
+ runGeneralAssistShortcut,
+ 'General Assist'
+ );
+ registeredScreenshotAssist = registerOne(
+ null,
+ shortcuts.screenshotAssist,
+ runScreenshotAssistShortcut,
+ 'Screenshot Assist'
+ );
}
export async function reregisterGeneralAssist(): Promise {
const shortcuts = await getShortcuts();
- registerShortcut(shortcuts.generalAssist);
+ registeredGeneralAssist = registerOne(
+ registeredGeneralAssist,
+ shortcuts.generalAssist,
+ runGeneralAssistShortcut,
+ 'General Assist'
+ );
+}
+
+export async function reregisterScreenshotAssist(): Promise {
+ const shortcuts = await getShortcuts();
+ registeredScreenshotAssist = registerOne(
+ registeredScreenshotAssist,
+ shortcuts.screenshotAssist,
+ runScreenshotAssistShortcut,
+ 'Screenshot Assist'
+ );
}
export function destroyTray(): void {
- if (currentShortcut) {
- globalShortcut.unregister(currentShortcut);
- currentShortcut = null;
+ if (registeredGeneralAssist) {
+ globalShortcut.unregister(registeredGeneralAssist);
+ registeredGeneralAssist = null;
+ }
+ if (registeredScreenshotAssist) {
+ globalShortcut.unregister(registeredScreenshotAssist);
+ registeredScreenshotAssist = null;
}
tray?.destroy();
tray = null;
diff --git a/surfsense_desktop/src/modules/window.ts b/surfsense_desktop/src/modules/window.ts
index c925bf947..8b7c02133 100644
--- a/surfsense_desktop/src/modules/window.ts
+++ b/surfsense_desktop/src/modules/window.ts
@@ -1,5 +1,6 @@
import { app, BrowserWindow, shell, session } from 'electron';
import path from 'path';
+import { trackEvent } from './analytics';
import { showErrorDialog } from './errors';
import { getServerPort } from './server';
import { setActiveSearchSpaceId } from './active-search-space';
@@ -93,3 +94,15 @@ export function createMainWindow(initialPath = '/dashboard'): BrowserWindow {
return mainWindow;
}
+
+export function showMainWindow(source: 'tray_click' | 'tray_menu' | 'shortcut' = 'tray_click'): void {
+ const existing = getMainWindow();
+ const reopened = !existing || existing.isDestroyed();
+ if (reopened) {
+ createMainWindow('/dashboard');
+ } else {
+ existing.show();
+ existing.focus();
+ }
+ trackEvent('desktop_main_window_shown', { source, reopened });
+}
diff --git a/surfsense_desktop/src/preload.ts b/surfsense_desktop/src/preload.ts
index 100825c0f..7d72e9da5 100644
--- a/surfsense_desktop/src/preload.ts
+++ b/surfsense_desktop/src/preload.ts
@@ -17,6 +17,13 @@ contextBridge.exposeInMainWorld('electronAPI', {
ipcRenderer.removeListener(IPC_CHANNELS.DEEP_LINK, listener);
};
},
+ onChatScreenCapture: (callback: (dataUrl: string) => void) => {
+ const listener = (_event: unknown, dataUrl: string) => callback(dataUrl);
+ ipcRenderer.on(IPC_CHANNELS.CHAT_SCREEN_CAPTURE, listener);
+ return () => {
+ ipcRenderer.removeListener(IPC_CHANNELS.CHAT_SCREEN_CAPTURE, listener);
+ };
+ },
getQuickAskText: () => ipcRenderer.invoke(IPC_CHANNELS.QUICK_ASK_TEXT),
setQuickAskMode: (mode: string) => ipcRenderer.invoke(IPC_CHANNELS.SET_QUICK_ASK_MODE, mode),
getQuickAskMode: () => ipcRenderer.invoke(IPC_CHANNELS.GET_QUICK_ASK_MODE),
@@ -25,20 +32,8 @@ contextBridge.exposeInMainWorld('electronAPI', {
getPermissionsStatus: () => ipcRenderer.invoke(IPC_CHANNELS.GET_PERMISSIONS_STATUS),
requestAccessibility: () => ipcRenderer.invoke(IPC_CHANNELS.REQUEST_ACCESSIBILITY),
requestScreenRecording: () => ipcRenderer.invoke(IPC_CHANNELS.REQUEST_SCREEN_RECORDING),
+ captureFullScreen: () => ipcRenderer.invoke(IPC_CHANNELS.CAPTURE_FULL_SCREEN),
restartApp: () => ipcRenderer.invoke(IPC_CHANNELS.RESTART_APP),
- // Autocomplete
- onAutocompleteContext: (callback: (data: { screenshot: string; searchSpaceId?: string; appName?: string; windowTitle?: string }) => void) => {
- const listener = (_event: unknown, data: { screenshot: string; searchSpaceId?: string; appName?: string; windowTitle?: string }) => callback(data);
- ipcRenderer.on(IPC_CHANNELS.AUTOCOMPLETE_CONTEXT, listener);
- return () => {
- ipcRenderer.removeListener(IPC_CHANNELS.AUTOCOMPLETE_CONTEXT, listener);
- };
- },
- acceptSuggestion: (text: string) => ipcRenderer.invoke(IPC_CHANNELS.ACCEPT_SUGGESTION, text),
- dismissSuggestion: () => ipcRenderer.invoke(IPC_CHANNELS.DISMISS_SUGGESTION),
- setAutocompleteEnabled: (enabled: boolean) => ipcRenderer.invoke(IPC_CHANNELS.SET_AUTOCOMPLETE_ENABLED, enabled),
- getAutocompleteEnabled: () => ipcRenderer.invoke(IPC_CHANNELS.GET_AUTOCOMPLETE_ENABLED),
-
// Folder sync
selectFolder: () => ipcRenderer.invoke(IPC_CHANNELS.FOLDER_SYNC_SELECT_FOLDER),
addWatchedFolder: (config: any) => ipcRenderer.invoke(IPC_CHANNELS.FOLDER_SYNC_ADD_FOLDER, config),
diff --git a/surfsense_web/app/dashboard/[search_space_id]/client-layout.tsx b/surfsense_web/app/dashboard/[search_space_id]/client-layout.tsx
index eceb46231..d95aab6e8 100644
--- a/surfsense_web/app/dashboard/[search_space_id]/client-layout.tsx
+++ b/surfsense_web/app/dashboard/[search_space_id]/client-layout.tsx
@@ -6,6 +6,7 @@ import { useTranslations } from "next-intl";
import type React from "react";
import { useCallback, useEffect, useRef, useState } from "react";
import { toast } from "sonner";
+import { pendingUserImageDataUrlsAtom } from "@/atoms/chat/pending-user-images.atom";
import { myAccessAtom } from "@/atoms/members/members-query.atoms";
import { updateLLMPreferencesMutationAtom } from "@/atoms/new-llm-config/new-llm-config-mutation.atoms";
import {
@@ -33,6 +34,7 @@ export function DashboardClientLayout({
const pathname = usePathname();
const { search_space_id } = useParams();
const setActiveSearchSpaceIdState = useSetAtom(activeSearchSpaceIdAtom);
+ const setPendingUserImageUrls = useSetAtom(pendingUserImageDataUrlsAtom);
const {
data: preferences = {},
@@ -142,6 +144,14 @@ export function DashboardClientLayout({
const electronAPI = useElectronAPI();
+ useEffect(() => {
+ if (!electronAPI?.onChatScreenCapture) return;
+ return electronAPI.onChatScreenCapture((dataUrl: string) => {
+ if (typeof dataUrl !== "string" || !dataUrl.startsWith("data:image/")) return;
+ setPendingUserImageUrls((prev) => [...prev, dataUrl]);
+ });
+ }, [electronAPI, setPendingUserImageUrls]);
+
useEffect(() => {
const activeSeacrhSpaceId =
typeof search_space_id === "string"
diff --git a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx
index c363d69d3..7773a438a 100644
--- a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx
+++ b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx
@@ -25,6 +25,7 @@ import {
mentionedDocumentsAtom,
messageDocumentsMapAtom,
} from "@/atoms/chat/mentioned-documents.atom";
+import { pendingUserImageDataUrlsAtom } from "@/atoms/chat/pending-user-images.atom";
import {
clearPlanOwnerRegistry,
// extractWriteTodosFromContent,
@@ -44,8 +45,8 @@ import {
} from "@/components/assistant-ui/token-usage-context";
import { useChatSessionStateSync } from "@/hooks/use-chat-session-state";
import { useMessagesSync } from "@/hooks/use-messages-sync";
-import { documentsApiService } from "@/lib/apis/documents-api.service";
import { getAgentFilesystemSelection } from "@/lib/agent-filesystem";
+import { documentsApiService } from "@/lib/apis/documents-api.service";
import { getBearerToken } from "@/lib/auth-utils";
import { convertToThreadMessage } from "@/lib/chat/message-utils";
import {
@@ -75,6 +76,10 @@ import {
type ThreadListResponse,
type ThreadRecord,
} from "@/lib/chat/thread-persistence";
+import {
+ extractUserTurnForNewChatApi,
+ type NewChatUserImagePayload,
+} from "@/lib/chat/user-turn-api-parts";
import { NotFoundError } from "@/lib/error";
import {
trackChatCreated,
@@ -228,6 +233,8 @@ export default function NewChatPage() {
const updateChatTabTitle = useSetAtom(updateChatTabTitleAtom);
const removeChatTab = useSetAtom(removeChatTabAtom);
const setAgentCreatedDocuments = useSetAtom(agentCreatedDocumentsAtom);
+ const pendingUserImageUrls = useAtomValue(pendingUserImageDataUrlsAtom);
+ const setPendingUserImageUrls = useSetAtom(pendingUserImageDataUrlsAtom);
// Get current user for author info in shared chats
const { data: currentUser } = useAtomValue(currentUserAtom);
@@ -489,18 +496,12 @@ export default function NewChatPage() {
abortControllerRef.current = null;
}
- // Extract user query text from content parts
- let userQuery = "";
- for (const part of message.content) {
- if (part.type === "text") {
- userQuery += part.text;
- }
- }
+ const urlsSnapshot = [...pendingUserImageUrls];
+ const { userQuery, userImages } = extractUserTurnForNewChatApi(message, urlsSnapshot);
- if (!userQuery.trim()) return;
+ if (!userQuery.trim() && userImages.length === 0) return;
- // Check if podcast is already generating
- if (isPodcastGenerating() && looksLikePodcastRequest(userQuery)) {
+ if (userQuery.trim() && isPodcastGenerating() && looksLikePodcastRequest(userQuery)) {
toast.warning("A podcast is already being generated.");
return;
}
@@ -540,6 +541,10 @@ export default function NewChatPage() {
}
}
+ if (urlsSnapshot.length > 0) {
+ setPendingUserImageUrls((prev) => prev.filter((u) => !urlsSnapshot.includes(u)));
+ }
+
// Add user message to state
const userMsgId = `msg-user-${Date.now()}`;
@@ -555,10 +560,27 @@ export default function NewChatPage() {
}
: undefined;
+ const existingImageUrls = new Set(
+ message.content
+ .filter(
+ (p): p is { type: "image"; image: string } =>
+ typeof p === "object" &&
+ p !== null &&
+ "type" in p &&
+ p.type === "image" &&
+ "image" in p
+ )
+ .map((p) => p.image)
+ );
+ const extraImageParts = urlsSnapshot
+ .filter((u) => !existingImageUrls.has(u))
+ .map((image) => ({ type: "image" as const, image }));
+ const userDisplayContent = [...message.content, ...extraImageParts];
+
const userMessage: ThreadMessageLike = {
id: userMsgId,
role: "user",
- content: message.content,
+ content: userDisplayContent,
createdAt: new Date(),
metadata: authorMetadata,
};
@@ -566,7 +588,7 @@ export default function NewChatPage() {
// Track message sent
trackChatMessageSent(searchSpaceId, currentThreadId, {
- hasAttachments: false,
+ hasAttachments: userImages.length > 0,
hasMentionedDocuments:
mentionedDocumentIds.surfsense_doc_ids.length > 0 ||
mentionedDocumentIds.document_ids.length > 0,
@@ -590,7 +612,7 @@ export default function NewChatPage() {
}));
}
- const persistContent: unknown[] = [...message.content];
+ const persistContent: unknown[] = [...userDisplayContent];
if (allMentionedDocs.length > 0) {
persistContent.push({
@@ -655,8 +677,7 @@ export default function NewChatPage() {
const selection = await getAgentFilesystemSelection(searchSpaceId);
if (
selection.filesystem_mode === "desktop_local_folder" &&
- (!selection.local_filesystem_mounts ||
- selection.local_filesystem_mounts.length === 0)
+ (!selection.local_filesystem_mounts || selection.local_filesystem_mounts.length === 0)
) {
toast.error("Select a local folder before using Local Folder mode.");
return;
@@ -704,6 +725,7 @@ export default function NewChatPage() {
? mentionedDocumentIds.surfsense_doc_ids
: undefined,
disabled_tools: disabledTools.length > 0 ? disabledTools : undefined,
+ ...(userImages.length > 0 ? { user_images: userImages } : {}),
}),
signal: controller.signal,
});
@@ -835,14 +857,7 @@ export default function NewChatPage() {
});
} else {
const tcId = `interrupt-${action.name}`;
- addToolCall(
- contentPartsState,
- toolsWithUI,
- tcId,
- action.name,
- action.args,
- true
- );
+ addToolCall(contentPartsState, toolsWithUI, tcId, action.name, action.args, true);
updateToolCall(contentPartsState, tcId, {
result: { __interrupt__: true, ...interruptData },
});
@@ -980,6 +995,9 @@ export default function NewChatPage() {
disabledTools,
updateChatTabTitle,
tokenUsageStore,
+ pendingUserImageUrls,
+ setPendingUserImageUrls,
+ toolsWithUI,
]
);
@@ -1180,14 +1198,7 @@ export default function NewChatPage() {
});
} else {
const tcId = `interrupt-${action.name}`;
- addToolCall(
- contentPartsState,
- toolsWithUI,
- tcId,
- action.name,
- action.args,
- true
- );
+ addToolCall(contentPartsState, toolsWithUI, tcId, action.name, action.args, true);
updateToolCall(contentPartsState, tcId, {
result: {
__interrupt__: true,
@@ -1252,7 +1263,7 @@ export default function NewChatPage() {
abortControllerRef.current = null;
}
},
- [pendingInterrupt, messages, searchSpaceId, tokenUsageStore]
+ [pendingInterrupt, messages, searchSpaceId, tokenUsageStore, toolsWithUI]
);
useEffect(() => {
@@ -1320,15 +1331,24 @@ export default function NewChatPage() {
* Handle regeneration (edit or reload) by calling the regenerate endpoint
* and streaming the response. This rewinds the LangGraph checkpointer state.
*
- * @param newUserQuery - The new user query (for edit). Pass null/undefined for reload.
+ * @param newUserQuery - `null` = reload with same turn from the server. A string = edit
+ * (including an empty string when the edited turn is images-only); pass `editExtras` for images/content.
*/
const handleRegenerate = useCallback(
- async (newUserQuery?: string | null) => {
+ async (
+ newUserQuery: string | null,
+ editExtras?: {
+ userMessageContent: ThreadMessageLike["content"];
+ userImages: NewChatUserImagePayload[];
+ }
+ ) => {
if (!threadId) {
toast.error("Cannot regenerate: no active chat thread");
return;
}
+ const isEdit = newUserQuery !== null;
+
// Abort any previous streaming request
if (abortControllerRef.current) {
abortControllerRef.current.abort();
@@ -1342,11 +1362,11 @@ export default function NewChatPage() {
}
// Extract the original user query BEFORE removing messages (for reload mode)
- let userQueryToDisplay = newUserQuery;
+ let userQueryToDisplay: string | undefined;
let originalUserMessageContent: ThreadMessageLike["content"] | null = null;
let originalUserMessageMetadata: ThreadMessageLike["metadata"] | undefined;
- if (!newUserQuery) {
+ if (!isEdit) {
// Reload mode - find and preserve the last user message content
const lastUserMessage = [...messages].reverse().find((m) => m.role === "user");
if (lastUserMessage) {
@@ -1360,6 +1380,8 @@ export default function NewChatPage() {
}
}
}
+ } else {
+ userQueryToDisplay = newUserQuery;
}
// Remove the last two messages (user + assistant) from the UI immediately
@@ -1395,11 +1417,11 @@ export default function NewChatPage() {
const userMessage: ThreadMessageLike = {
id: userMsgId,
role: "user",
- content: newUserQuery
- ? [{ type: "text", text: newUserQuery }]
+ content: isEdit
+ ? (editExtras?.userMessageContent ?? [{ type: "text", text: newUserQuery ?? "" }])
: originalUserMessageContent || [{ type: "text", text: userQueryToDisplay || "" }],
createdAt: new Date(),
- metadata: newUserQuery ? undefined : originalUserMessageMetadata,
+ metadata: isEdit ? undefined : originalUserMessageMetadata,
};
setMessages((prev) => [...prev, userMessage]);
@@ -1416,20 +1438,24 @@ export default function NewChatPage() {
try {
const selection = await getAgentFilesystemSelection(searchSpaceId);
+ const requestBody: Record = {
+ search_space_id: searchSpaceId,
+ user_query: newUserQuery,
+ disabled_tools: disabledTools.length > 0 ? disabledTools : undefined,
+ filesystem_mode: selection.filesystem_mode,
+ client_platform: selection.client_platform,
+ local_filesystem_mounts: selection.local_filesystem_mounts,
+ };
+ if (isEdit) {
+ requestBody.user_images = editExtras?.userImages ?? [];
+ }
const response = await fetch(getRegenerateUrl(threadId), {
method: "POST",
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${token}`,
},
- body: JSON.stringify({
- search_space_id: searchSpaceId,
- user_query: newUserQuery || null,
- disabled_tools: disabledTools.length > 0 ? disabledTools : undefined,
- filesystem_mode: selection.filesystem_mode,
- client_platform: selection.client_platform,
- local_filesystem_mounts: selection.local_filesystem_mounts,
- }),
+ body: JSON.stringify(requestBody),
signal: controller.signal,
});
@@ -1519,8 +1545,8 @@ export default function NewChatPage() {
if (contentParts.length > 0) {
try {
// Persist user message (for both edit and reload modes, since backend deleted it)
- const userContentToPersist = newUserQuery
- ? [{ type: "text", text: newUserQuery }]
+ const userContentToPersist = isEdit
+ ? (editExtras?.userMessageContent ?? [{ type: "text", text: newUserQuery ?? "" }])
: originalUserMessageContent || [{ type: "text", text: userQueryToDisplay || "" }];
const savedUserMessage = await appendMessage(threadId, {
@@ -1579,27 +1605,21 @@ export default function NewChatPage() {
abortControllerRef.current = null;
}
},
- [threadId, searchSpaceId, messages, disabledTools, tokenUsageStore]
+ [threadId, searchSpaceId, messages, disabledTools, tokenUsageStore, toolsWithUI]
);
// Handle editing a message - truncates history and regenerates with new query
const onEdit = useCallback(
async (message: AppendMessage) => {
- // Extract the new user query from the message content
- let newUserQuery = "";
- for (const part of message.content) {
- if (part.type === "text") {
- newUserQuery += part.text;
- }
- }
-
- if (!newUserQuery.trim()) {
+ const { userQuery, userImages } = extractUserTurnForNewChatApi(message, []);
+ const queryForApi = userQuery.trim();
+ if (!queryForApi && userImages.length === 0) {
toast.error("Cannot edit with empty message");
return;
}
- // Call regenerate with the new query
- await handleRegenerate(newUserQuery.trim());
+ const userMessageContent = message.content as unknown as ThreadMessageLike["content"];
+ await handleRegenerate(queryForApi, { userMessageContent, userImages });
},
[handleRegenerate]
);
diff --git a/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/DesktopContent.tsx b/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/DesktopContent.tsx
index 9861f5536..3368066c1 100644
--- a/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/DesktopContent.tsx
+++ b/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/DesktopContent.tsx
@@ -20,7 +20,6 @@ import { searchSpacesApiService } from "@/lib/apis/search-spaces-api.service";
export function DesktopContent() {
const api = useElectronAPI();
const [loading, setLoading] = useState(true);
- const [enabled, setEnabled] = useState(true);
const [searchSpaces, setSearchSpaces] = useState([]);
const [activeSpaceId, setActiveSpaceId] = useState(null);
@@ -41,14 +40,12 @@ export function DesktopContent() {
setAutoLaunchSupported(hasAutoLaunchApi);
Promise.all([
- api.getAutocompleteEnabled(),
api.getActiveSearchSpace?.() ?? Promise.resolve(null),
searchSpacesApiService.getSearchSpaces(),
hasAutoLaunchApi ? api.getAutoLaunch() : Promise.resolve(null),
])
- .then(([autoEnabled, spaceId, spaces, autoLaunch]) => {
+ .then(([spaceId, spaces, autoLaunch]) => {
if (!mounted) return;
- setEnabled(autoEnabled);
setActiveSpaceId(spaceId);
if (spaces) setSearchSpaces(spaces);
if (autoLaunch) {
@@ -86,11 +83,6 @@ export function DesktopContent() {
);
}
- const handleToggle = async (checked: boolean) => {
- setEnabled(checked);
- await api.setAutocompleteEnabled(checked);
- };
-
const handleAutoLaunchToggle = async (checked: boolean) => {
if (!autoLaunchSupported || !api.setAutoLaunch) {
toast.error("Please update the desktop app to configure launch on startup");
@@ -133,13 +125,12 @@ export function DesktopContent() {
return (
- {/* Default Search Space */}
Default Search Space
- Choose which search space General Assist, Quick Assist, and Extreme Assist operate
- against.
+ Choose which search space General Assist, Screenshot Assist, and Quick Assist use by
+ default.
@@ -164,7 +155,6 @@ export function DesktopContent() {
- {/* Launch on Startup */}
@@ -215,29 +205,6 @@ export function DesktopContent() {
-
- {/* Extreme Assist Toggle */}
-
-
- Extreme Assist
-
- Get inline writing suggestions powered by your knowledge base as you type in any app.
-
-
-
-
-
-
-
- Show suggestions while typing in other applications.
-
);
}
diff --git a/surfsense_web/app/desktop/login/page.tsx b/surfsense_web/app/desktop/login/page.tsx
index 451143949..c8ec4dfce 100644
--- a/surfsense_web/app/desktop/login/page.tsx
+++ b/surfsense_web/app/desktop/login/page.tsx
@@ -2,7 +2,7 @@
import { IconBrandGoogleFilled } from "@tabler/icons-react";
import { useAtom } from "jotai";
-import { BrainCog, Eye, EyeOff, Rocket, RotateCcw, Zap } from "lucide-react";
+import { Crop, Eye, EyeOff, Rocket, RotateCcw, Zap } from "lucide-react";
import Image from "next/image";
import { useRouter } from "next/navigation";
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
@@ -21,28 +21,33 @@ import { setBearerToken } from "@/lib/auth-utils";
import { AUTH_TYPE, BACKEND_URL } from "@/lib/env-config";
const isGoogleAuth = AUTH_TYPE === "GOOGLE";
-type ShortcutKey = "generalAssist" | "quickAsk" | "autocomplete";
+type ShortcutKey = "generalAssist" | "quickAsk" | "screenshotAssist";
type ShortcutMap = typeof DEFAULT_SHORTCUTS;
-const HOTKEY_ROWS: Array<{ key: ShortcutKey; label: string; description: string; icon: React.ElementType }> = [
+const HOTKEY_ROWS: Array<{
+ key: ShortcutKey;
+ label: string;
+ description: string;
+ icon: React.ElementType;
+}> = [
{
key: "generalAssist",
label: "General Assist",
description: "Launch SurfSense instantly from any application",
icon: Rocket,
},
+ {
+ key: "screenshotAssist",
+ label: "Screenshot Assist",
+ description: "Draw a region on screen to attach that capture to chat",
+ icon: Crop,
+ },
{
key: "quickAsk",
label: "Quick Assist",
description: "Select text anywhere, then ask AI to explain, rewrite, or act on it",
icon: Zap,
},
- {
- key: "autocomplete",
- label: "Extreme Assist",
- description: "AI drafts text using your screen context and knowledge base",
- icon: BrainCog,
- },
];
function acceleratorToKeys(accel: string, isMac: boolean): string[] {
@@ -182,7 +187,7 @@ export default function DesktopLoginPage() {
}, [api]);
const updateShortcut = useCallback(
- (key: "generalAssist" | "quickAsk" | "autocomplete", accelerator: string) => {
+ (key: ShortcutKey, accelerator: string) => {
setShortcuts((prev) => {
const updated = { ...prev, [key]: accelerator };
api?.setShortcuts?.({ [key]: accelerator }).catch(() => {
@@ -196,7 +201,7 @@ export default function DesktopLoginPage() {
);
const resetShortcut = useCallback(
- (key: "generalAssist" | "quickAsk" | "autocomplete") => {
+ (key: ShortcutKey) => {
updateShortcut(key, DEFAULT_SHORTCUTS[key]);
},
[updateShortcut]
@@ -369,7 +374,9 @@ export default function DesktopLoginPage() {
)}
diff --git a/surfsense_web/app/desktop/permissions/page.tsx b/surfsense_web/app/desktop/permissions/page.tsx
index a2fadc8ff..e30a76f83 100644
--- a/surfsense_web/app/desktop/permissions/page.tsx
+++ b/surfsense_web/app/desktop/permissions/page.tsx
@@ -19,14 +19,15 @@ const STEPS = [
id: "screen-recording",
title: "Screen Recording",
description:
- "Lets SurfSense capture your screen to understand context and provide smart writing suggestions.",
+ "Lets SurfSense capture a region of your screen, full display, or browser (where supported) to attach to chat in Screenshot Assist, or to capture the full display from the composer.",
action: "requestScreenRecording",
field: "screenRecording" as const,
},
{
id: "accessibility",
title: "Accessibility",
- description: "Lets SurfSense insert suggestions seamlessly, right where you\u2019re typing.",
+ description:
+ "Lets SurfSense bring the app to the foreground and work with the active application (for example Quick Assist) when you use desktop shortcuts.",
action: "requestAccessibility",
field: "accessibility" as const,
},
@@ -131,7 +132,8 @@ export default function DesktopPermissionsPage() {
System Permissions
- SurfSense needs two macOS permissions to provide context-aware writing suggestions.
+ SurfSense needs two macOS permissions for Screenshot Assist and for desktop features that
+ require focusing the app or the active application.
- Your agent can search and read messages from Teams channels you have access to,
- and send messages on your behalf. Make sure you're a member of the teams
- you want to interact with.
+ Your agent can search and read messages from Teams channels you have access to, and send
+ messages on your behalf. Make sure you're a member of the teams you want to interact
+ with.
diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-edit-view.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-edit-view.tsx
index b2b40dfd6..e7895c2e9 100644
--- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-edit-view.tsx
+++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-edit-view.tsx
@@ -16,7 +16,7 @@ import { DateRangeSelector } from "../../components/date-range-selector";
import { PeriodicSyncConfig } from "../../components/periodic-sync-config";
import { SummaryConfig } from "../../components/summary-config";
import { VisionLLMConfig } from "../../components/vision-llm-config";
-import { LIVE_CONNECTOR_TYPES, getReauthEndpoint } from "../../constants/connector-constants";
+import { getReauthEndpoint, LIVE_CONNECTOR_TYPES } from "../../constants/connector-constants";
import { getConnectorDisplayName } from "../../tabs/all-connectors-tab";
import { MCPServiceConfig } from "../components/mcp-service-config";
import { getConnectorConfigComponent } from "../index";
@@ -380,8 +380,8 @@ export const ConnectorEditView: FC = ({
{/* Fixed Footer - Action buttons */}
- {showDisconnectConfirm ? (
-
+ {showDisconnectConfirm ? (
+
{isLive
? "Your agent will lose access to this service."
diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/indexing-configuration-view.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/indexing-configuration-view.tsx
index 690333523..982b0be11 100644
--- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/indexing-configuration-view.tsx
+++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/indexing-configuration-view.tsx
@@ -12,7 +12,10 @@ import { DateRangeSelector } from "../../components/date-range-selector";
import { PeriodicSyncConfig } from "../../components/periodic-sync-config";
import { SummaryConfig } from "../../components/summary-config";
import { VisionLLMConfig } from "../../components/vision-llm-config";
-import { LIVE_CONNECTOR_TYPES, type IndexingConfigState } from "../../constants/connector-constants";
+import {
+ type IndexingConfigState,
+ LIVE_CONNECTOR_TYPES,
+} from "../../constants/connector-constants";
import { getConnectorDisplayName } from "../../tabs/all-connectors-tab";
import { getConnectorConfigComponent } from "../index";
diff --git a/surfsense_web/components/assistant-ui/connector-popup/tabs/active-connectors-tab.tsx b/surfsense_web/components/assistant-ui/connector-popup/tabs/active-connectors-tab.tsx
index fe9aab14f..755086ba5 100644
--- a/surfsense_web/components/assistant-ui/connector-popup/tabs/active-connectors-tab.tsx
+++ b/surfsense_web/components/assistant-ui/connector-popup/tabs/active-connectors-tab.tsx
@@ -9,7 +9,11 @@ import { getConnectorIcon } from "@/contracts/enums/connectorIcons";
import type { SearchSourceConnector } from "@/contracts/types/connector.types";
import { getDocumentTypeLabel } from "@/lib/documents/document-type-labels";
import { cn } from "@/lib/utils";
-import { COMPOSIO_CONNECTORS, LIVE_CONNECTOR_TYPES, OAUTH_CONNECTORS } from "../constants/connector-constants";
+import {
+ COMPOSIO_CONNECTORS,
+ LIVE_CONNECTOR_TYPES,
+ OAUTH_CONNECTORS,
+} from "../constants/connector-constants";
import { getDocumentCountForConnector } from "../utils/connector-document-mapping";
import { getConnectorDisplayName } from "./all-connectors-tab";
diff --git a/surfsense_web/components/assistant-ui/connector-popup/views/connector-accounts-list-view.tsx b/surfsense_web/components/assistant-ui/connector-popup/views/connector-accounts-list-view.tsx
index b3c087599..8aee7e005 100644
--- a/surfsense_web/components/assistant-ui/connector-popup/views/connector-accounts-list-view.tsx
+++ b/surfsense_web/components/assistant-ui/connector-popup/views/connector-accounts-list-view.tsx
@@ -13,7 +13,7 @@ import type { SearchSourceConnector } from "@/contracts/types/connector.types";
import { authenticatedFetch } from "@/lib/auth-utils";
import { formatRelativeDate } from "@/lib/format-date";
import { cn } from "@/lib/utils";
-import { LIVE_CONNECTOR_TYPES, getReauthEndpoint } from "../constants/connector-constants";
+import { getReauthEndpoint, LIVE_CONNECTOR_TYPES } from "../constants/connector-constants";
import { useConnectorStatus } from "../hooks/use-connector-status";
import { getConnectorDisplayName } from "../tabs/all-connectors-tab";
@@ -182,11 +182,14 @@ export const ConnectorAccountsListView: FC = ({
diff --git a/surfsense_web/components/settings/user-settings-dialog.tsx b/surfsense_web/components/settings/user-settings-dialog.tsx
index cc36392ae..6740aad92 100644
--- a/surfsense_web/components/settings/user-settings-dialog.tsx
+++ b/surfsense_web/components/settings/user-settings-dialog.tsx
@@ -1,7 +1,16 @@
"use client";
import { useAtom } from "jotai";
-import { Brain, CircleUser, Globe, Keyboard, KeyRound, Monitor, ReceiptText, Sparkles } from "lucide-react";
+import {
+ Brain,
+ CircleUser,
+ Globe,
+ Keyboard,
+ KeyRound,
+ Monitor,
+ ReceiptText,
+ Sparkles,
+} from "lucide-react";
import dynamic from "next/dynamic";
import { useTranslations } from "next-intl";
import { useMemo } from "react";
@@ -53,9 +62,9 @@ const DesktopContent = dynamic(
);
const DesktopShortcutsContent = dynamic(
() =>
- import("@/app/dashboard/[search_space_id]/user-settings/components/DesktopShortcutsContent").then(
- (m) => ({ default: m.DesktopShortcutsContent })
- ),
+ import(
+ "@/app/dashboard/[search_space_id]/user-settings/components/DesktopShortcutsContent"
+ ).then((m) => ({ default: m.DesktopShortcutsContent })),
{ ssr: false }
);
const MemoryContent = dynamic(
diff --git a/surfsense_web/components/tool-ui/generic-hitl-approval.tsx b/surfsense_web/components/tool-ui/generic-hitl-approval.tsx
index c83bf55d5..ceb1d0209 100644
--- a/surfsense_web/components/tool-ui/generic-hitl-approval.tsx
+++ b/surfsense_web/components/tool-ui/generic-hitl-approval.tsx
@@ -118,7 +118,9 @@ function GenericApprovalCard({
setProcessing();
onDecision({ type: "approve" });
connectorsApiService.trustMCPTool(mcpConnectorId, toolName).catch(() => {
- toast.error("Failed to save 'Always Allow' preference. The tool will still require approval next time.");
+ toast.error(
+ "Failed to save 'Always Allow' preference. The tool will still require approval next time."
+ );
});
}, [phase, setProcessing, onDecision, isMCPTool, mcpConnectorId, toolName]);
diff --git a/surfsense_web/components/tool-ui/google-calendar/create-event.tsx b/surfsense_web/components/tool-ui/google-calendar/create-event.tsx
index 9427c989b..523be31f6 100644
--- a/surfsense_web/components/tool-ui/google-calendar/create-event.tsx
+++ b/surfsense_web/components/tool-ui/google-calendar/create-event.tsx
@@ -2,7 +2,14 @@
import type { ToolCallMessagePartProps } from "@assistant-ui/react";
import { useSetAtom } from "jotai";
-import { ClockIcon, CornerDownLeftIcon, GlobeIcon, MapPinIcon, Pencil, UsersIcon } from "lucide-react";
+import {
+ ClockIcon,
+ CornerDownLeftIcon,
+ GlobeIcon,
+ MapPinIcon,
+ Pencil,
+ UsersIcon,
+} from "lucide-react";
import { useCallback, useEffect, useMemo, useState } from "react";
import type { ExtraField } from "@/atoms/chat/hitl-edit-panel.atom";
import { openHitlEditPanelAtom } from "@/atoms/chat/hitl-edit-panel.atom";
diff --git a/surfsense_web/content/docs/connectors/baidu-search.mdx b/surfsense_web/content/docs/connectors/baidu-search.mdx
new file mode 100644
index 000000000..56d048d5b
--- /dev/null
+++ b/surfsense_web/content/docs/connectors/baidu-search.mdx
@@ -0,0 +1,121 @@
+---
+title: Baidu Search
+description: Search the Chinese web with Baidu AI Search in SurfSense
+---
+
+# Baidu Search Integration Setup Guide
+
+This guide walks you through connecting Baidu AI Search to SurfSense for Chinese web search and AI-powered research.
+
+## How it works
+
+The Baidu Search connector uses Baidu AI Search through Qianfan AppBuilder's intelligent search generation API. It is a live search connector: SurfSense queries Baidu when the assistant needs current web results instead of periodically indexing content into your knowledge base.
+
+- Baidu Search is best for Simplified Chinese queries and China-focused web content.
+- Results are merged with SurfSense's other configured web search engines.
+- The connector returns Baidu references as sources that can be cited in chat responses.
+
+---
+
+## Authorization
+
+
+You need a Baidu Qianfan AppBuilder API key to use this connector. The key is encrypted and stored securely by SurfSense.
+
+
+### Step 1: Get Your Baidu AI Search API Key
+
+1. Open the [Baidu AI Search product page](https://cloud.baidu.com/product/ai-search.html) and sign in with your Baidu Cloud account.
+2. Open Qianfan AppBuilder or the AI Search console from Baidu Cloud.
+3. Create or select an application that has access to Baidu AI Search.
+4. Generate an API key for the application.
+5. Copy the API key. SurfSense uses it as the `BAIDU_API_KEY` connector setting.
+
+
+Keep this key private. Do not paste it into chat messages, issue reports, screenshots, or public repositories.
+
+
+---
+
+## Connecting to SurfSense
+
+1. Navigate to **Connectors** → **Add Connector** → **Baidu Search**.
+2. Fill in the required fields:
+
+| Field | Description | Example |
+|-------|-------------|---------|
+| **Connector Name** | A friendly name to identify this connector | `Baidu Search` |
+| **Baidu AppBuilder API Key** | Your Qianfan AppBuilder API key | `bce-v3/...` |
+
+3. Click **Connect** to save the connector.
+4. Ask a current Chinese web query in chat, such as `今天中国人工智能行业有什么重要新闻？`.
+
+### Optional Advanced Settings
+
+SurfSense stores advanced Baidu options in the connector config. If your deployment exposes these fields, use the following values:
+
+| Setting | Description | Default |
+|---------|-------------|---------|
+| `BAIDU_MODEL` | The model Baidu AI Search uses for answer generation | `ernie-3.5-8k` |
+| `BAIDU_SEARCH_SOURCE` | Baidu search source version | `baidu_search_v2` |
+| `BAIDU_ENABLE_DEEP_SEARCH` | Enables Baidu's deeper search mode when supported by your account | `false` |
+
+SurfSense calls Baidu's intelligent search generation endpoint:
+
+```text
+POST https://qianfan.baidubce.com/v2/ai_search/chat/completions
+```
+
+For request and response details, see Baidu's [intelligent search generation API documentation](https://cloud.baidu.com/doc/qianfan/s/Omh4su4s0).
+
+---
+
+## When to Use Baidu Search
+
+| Use Case | Why Baidu Search Helps |
+|----------|------------------------|
+| Chinese news and current events | Better coverage for China-focused sources |
+| Chinese company, product, or policy research | More local web results than global search engines alone |
+| Mandarin-language fact finding | Native Chinese search and summarization behavior |
+| Cross-checking web search | Adds another source alongside SearXNG, Tavily, or Linkup |
+
+
+Baidu Search does not create indexed documents in your knowledge base. It runs when the assistant calls web search, then returns live sources for that answer.
+
+
+---
+
+## Troubleshooting
+
+**No Baidu results appear**
+
+- Confirm the Baidu Search connector is active in the current search space.
+- Try a Chinese query with clear search intent, for example `百度智能云千帆 AppBuilder 最新功能`.
+- Check whether other web search engines are returning results. If none are, review the general [Web Search](/docs/how-to/web-search) setup.
+
+**Authentication failed**
+
+- Verify that the API key was copied from Qianfan AppBuilder, not another Baidu Cloud product.
+- Regenerate the API key if it was rotated, expired, or copied with extra whitespace.
+- Make sure the related application has access to Baidu AI Search.
+
+**Requests time out**
+
+- Baidu AI Search can take longer than ordinary keyword search because it performs search and summarization.
+- Retry with a narrower query.
+- If you self-host SurfSense, verify that the backend container can reach `qianfan.baidubce.com`.
+
+**Results are not relevant**
+
+- Use Chinese keywords for China-focused topics.
+- Include entity names, dates, or locations in the query.
+- Compare with SearXNG or another configured live search connector for broader coverage.
+
+---
+
+## Verification Checklist
+
+- The Baidu Search connector appears in your connector list.
+- A Chinese current-events query triggers web search in chat.
+- Chat responses include Baidu-backed sources with titles and URLs.
+- Invalid API keys fail without breaking other configured search engines.
diff --git a/surfsense_web/content/docs/connectors/index.mdx b/surfsense_web/content/docs/connectors/index.mdx
index ef8d214ef..9b8fa5f93 100644
--- a/surfsense_web/content/docs/connectors/index.mdx
+++ b/surfsense_web/content/docs/connectors/index.mdx
@@ -83,6 +83,11 @@ Connect SurfSense to your favorite tools and services. Browse the available inte
description="Connect your GitHub repositories to SurfSense"
href="/docs/connectors/github"
/>
+
+Live search connectors only run for the search space where they are configured. They do not replace SearXNG globally.
+
+
## Docker Setup
SearXNG is included in both `docker-compose.yml` and `docker-compose.dev.yml` and works out of the box with no configuration needed.
diff --git a/surfsense_web/contracts/enums/toolIcons.tsx b/surfsense_web/contracts/enums/toolIcons.tsx
index 3bc639d33..bc63bc1b0 100644
--- a/surfsense_web/contracts/enums/toolIcons.tsx
+++ b/surfsense_web/contracts/enums/toolIcons.tsx
@@ -1,8 +1,8 @@
import {
BookOpen,
Brain,
- FileUser,
FileText,
+ FileUser,
Film,
Globe,
ImageIcon,
diff --git a/surfsense_web/lib/chat/display-media-capture.ts b/surfsense_web/lib/chat/display-media-capture.ts
new file mode 100644
index 000000000..c2fb69aae
--- /dev/null
+++ b/surfsense_web/lib/chat/display-media-capture.ts
@@ -0,0 +1,120 @@
+/** `getDisplayMedia` → single PNG frame (data URL). */
+function getImageCaptureCtor():
+ | (new (
+ track: MediaStreamTrack
+ ) => { grabFrame: () => Promise })
+ | undefined {
+ if (typeof window === "undefined") return undefined;
+ const IC = (
+ window as unknown as {
+ ImageCapture?: new (track: MediaStreamTrack) => { grabFrame: () => Promise };
+ }
+ ).ImageCapture;
+ return typeof IC === "function" ? IC : undefined;
+}
+
+function stopAllTracks(stream: MediaStream): void {
+ for (const t of stream.getTracks()) {
+ t.stop();
+ }
+}
+
+async function captureTrackToPngDataUrl(
+ track: MediaStreamTrack,
+ stream: MediaStream
+): Promise {
+ const ImageCtor = getImageCaptureCtor();
+ if (ImageCtor !== undefined) {
+ try {
+ const ic = new ImageCtor(track);
+ const bitmap = await ic.grabFrame();
+ try {
+ const canvas = document.createElement("canvas");
+ canvas.width = bitmap.width;
+ canvas.height = bitmap.height;
+ const ctx = canvas.getContext("2d");
+ if (!ctx) {
+ stopAllTracks(stream);
+ return null;
+ }
+ ctx.drawImage(bitmap, 0, 0);
+ stopAllTracks(stream);
+ return canvas.toDataURL("image/png");
+ } finally {
+ if ("close" in bitmap && typeof bitmap.close === "function") {
+ bitmap.close();
+ }
+ }
+ } catch {
+ /* fall through to
