mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-05-21 18:55:16 +02:00
Merge commit '61f4d05cd1' into dev_mod
This commit is contained in:
DESKTOP-RTLN3BA\$punk
commit
e6433f78c4
62 changed files with 1747 additions and 1523 deletions
12
README.es.md
12
README.es.md
|
|
@ -41,7 +41,7 @@ NotebookLM es una de las mejores y más útiles plataformas de IA que existen, p
|
|||
- **Sin Dependencia de Proveedores** - Configura cualquier modelo LLM, de imagen, TTS y STT.
|
||||
- **25+ Fuentes de Datos Externas** - Agrega tus fuentes desde Google Drive, OneDrive, Dropbox, Notion y muchos otros servicios externos.
|
||||
- **Soporte Multijugador en Tiempo Real** - Trabaja fácilmente con los miembros de tu equipo en un notebook compartido.
|
||||
- **Aplicación de Escritorio** - Obtén asistencia de IA en cualquier aplicación con Quick Assist, General Assist, Extreme Assist y sincronización de carpetas locales.
|
||||
- **Aplicación de Escritorio** - Obtén asistencia de IA en cualquier aplicación con Quick Assist, General Assist, Screenshot Assist y sincronización de carpetas locales.
|
||||
|
||||
...y más por venir.
|
||||
|
||||
|
|
@ -84,9 +84,9 @@ https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7
|
|||
|
||||
<p align="center"><img src="surfsense_web/public/homepage/hero_tutorial/quick_assist.gif" alt="Quick Assist" /></p>
|
||||
|
||||
- Aplicación de Escritorio — Extreme Assist
|
||||
- Aplicación de Escritorio — Screenshot Assist
|
||||
|
||||
<p align="center"><img src="surfsense_web/public/homepage/hero_tutorial/extreme_assist.gif" alt="Extreme Assist" /></p>
|
||||
<p align="center"><img src="surfsense_web/public/homepage/hero_tutorial/screenshot_assist.gif" alt="Screenshot Assist" /></p>
|
||||
|
||||
- Aplicación de Escritorio — Watch Local Folder
|
||||
|
||||
|
|
@ -150,7 +150,7 @@ La aplicación de escritorio incluye estas potentes funciones:
|
|||
|
||||
- **General Assist** — Lanza SurfSense al instante desde cualquier aplicación con un atajo global.
|
||||
- **Quick Assist** — Selecciona texto en cualquier lugar, luego pide a la IA que lo explique, reescriba o actúe sobre él.
|
||||
- **Extreme Assist** — Obtén sugerencias de escritura en línea impulsadas por tu base de conocimiento mientras escribes en cualquier aplicación.
|
||||
- **Screenshot Assist** — Selecciona una región de tu pantalla y adjúntala al chat para que las respuestas se basen en tu base de conocimiento.
|
||||
- **Watch Local Folder** — Vigila una carpeta local y sincroniza automáticamente los cambios de archivos con tu base de conocimiento. **Pro tip:** Apúntalo a tu bóveda de Obsidian para mantener tus notas buscables en SurfSense.
|
||||
|
||||
Todas las funciones operan contra tu espacio de búsqueda elegido, por lo que tus respuestas siempre están basadas en tus propios datos.
|
||||
|
|
@ -199,14 +199,14 @@ Todas las funciones operan contra tu espacio de búsqueda elegido, por lo que tu
|
|||
| **Generación de Videos** | Resúmenes en video cinemáticos vía Veo 3 (solo Ultra) | Disponible (NotebookLM es mejor aquí, mejorando activamente) |
|
||||
| **Generación de Presentaciones** | Diapositivas más atractivas pero no editables | Crea presentaciones editables basadas en diapositivas |
|
||||
| **Generación de Podcasts** | Resúmenes de audio con hosts e idiomas personalizables | Disponible con múltiples proveedores TTS (NotebookLM es mejor aquí, mejorando activamente) |
|
||||
| **Aplicación de Escritorio** | No | Aplicación nativa con General Assist, Quick Assist, Extreme Assist y sincronización de carpetas locales |
|
||||
| **Aplicación de Escritorio** | No | Aplicación nativa con General Assist, Quick Assist, Screenshot Assist y sincronización de carpetas locales |
|
||||
| **Extensión de Navegador** | No | Extensión multi-navegador para guardar cualquier página web, incluyendo páginas protegidas por autenticación |
|
||||
|
||||
<details>
|
||||
<summary><b>Lista completa de Fuentes Externas</b></summary>
|
||||
<a id="fuentes-externas"></a>
|
||||
|
||||
Motores de Búsqueda (Tavily, LinkUp) · SearxNG · Google Drive · OneDrive · Dropbox · Slack · Microsoft Teams · Linear · Jira · ClickUp · Confluence · BookStack · Notion · Gmail · Videos de YouTube · GitHub · Discord · Airtable · Google Calendar · Luma · Circleback · Elasticsearch · Obsidian, y más por venir.
|
||||
Motores de Búsqueda (SearXNG, Tavily, LinkUp, Baidu Search) · Google Drive · OneDrive · Dropbox · Slack · Microsoft Teams · Linear · Jira · ClickUp · Confluence · BookStack · Notion · Gmail · Videos de YouTube · GitHub · Discord · Airtable · Google Calendar · Luma · Circleback · Elasticsearch · Obsidian, y más por venir.
|
||||
|
||||
</details>
|
||||
|
||||
|
|
|
|||
12
README.hi.md
12
README.hi.md
|
|
@ -41,7 +41,7 @@ NotebookLM वहाँ उपलब्ध सबसे अच्छे और
|
|||
- **कोई विक्रेता लॉक-इन नहीं** - किसी भी LLM, इमेज, TTS और STT मॉडल को कॉन्फ़िगर करें।
|
||||
- **25+ बाहरी डेटा स्रोत** - Google Drive, OneDrive, Dropbox, Notion और कई अन्य बाहरी सेवाओं से अपने स्रोत जोड़ें।
|
||||
- **रीयल-टाइम मल्टीप्लेयर सपोर्ट** - एक साझा notebook में अपनी टीम के सदस्यों के साथ आसानी से काम करें।
|
||||
- **डेस्कटॉप ऐप** - Quick Assist, General Assist, Extreme Assist और लोकल फ़ोल्डर सिंक के साथ किसी भी एप्लिकेशन में AI सहायता प्राप्त करें।
|
||||
- **डेस्कटॉप ऐप** - Quick Assist, General Assist, Screenshot Assist और लोकल फ़ोल्डर सिंक के साथ किसी भी एप्लिकेशन में AI सहायता प्राप्त करें।
|
||||
|
||||
...और भी बहुत कुछ आने वाला है।
|
||||
|
||||
|
|
@ -84,9 +84,9 @@ https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7
|
|||
|
||||
<p align="center"><img src="surfsense_web/public/homepage/hero_tutorial/quick_assist.gif" alt="Quick Assist" /></p>
|
||||
|
||||
- डेस्कटॉप ऐप — Extreme Assist
|
||||
- डेस्कटॉप ऐप — Screenshot Assist
|
||||
|
||||
<p align="center"><img src="surfsense_web/public/homepage/hero_tutorial/extreme_assist.gif" alt="Extreme Assist" /></p>
|
||||
<p align="center"><img src="surfsense_web/public/homepage/hero_tutorial/screenshot_assist.gif" alt="Screenshot Assist" /></p>
|
||||
|
||||
- डेस्कटॉप ऐप — Watch Local Folder
|
||||
|
||||
|
|
@ -150,7 +150,7 @@ SurfSense एक डेस्कटॉप ऐप भी प्रदान क
|
|||
|
||||
- **General Assist** — एक ग्लोबल शॉर्टकट से किसी भी एप्लिकेशन से तुरंत SurfSense लॉन्च करें।
|
||||
- **Quick Assist** — कहीं भी टेक्स्ट चुनें, फिर AI से समझाने, फिर से लिखने या उस पर कार्रवाई करने को कहें।
|
||||
- **Extreme Assist** — किसी भी ऐप में टाइप करते समय अपनी नॉलेज बेस से संचालित इनलाइन लेखन सुझाव प्राप्त करें।
|
||||
- **Screenshot Assist** — स्क्रीन पर एक क्षेत्र चुनें और उसे चैट में जोड़ें, ताकि उत्तर आपकी नॉलेज बेस पर आधारित रहें।
|
||||
- **Watch Local Folder** — एक लोकल फ़ोल्डर को वॉच करें और फ़ाइल परिवर्तनों को स्वचालित रूप से अपनी नॉलेज बेस में सिंक करें। **Pro tip:** इसे अपने Obsidian vault पर पॉइंट करें ताकि आपके नोट्स SurfSense में सर्च करने योग्य रहें।
|
||||
|
||||
सभी सुविधाएं आपके चुने हुए सर्च स्पेस पर काम करती हैं, ताकि आपके उत्तर हमेशा आपके अपने डेटा पर आधारित हों।
|
||||
|
|
@ -199,14 +199,14 @@ SurfSense एक डेस्कटॉप ऐप भी प्रदान क
|
|||
| **वीडियो जनरेशन** | Veo 3 के माध्यम से सिनेमैटिक वीडियो ओवरव्यू (केवल Ultra) | उपलब्ध (NotebookLM यहाँ बेहतर है, सक्रिय रूप से सुधार हो रहा है) |
|
||||
| **प्रेजेंटेशन जनरेशन** | बेहतर दिखने वाली स्लाइड्स लेकिन संपादन योग्य नहीं | संपादन योग्य, स्लाइड आधारित प्रेजेंटेशन बनाएं |
|
||||
| **पॉडकास्ट जनरेशन** | कस्टमाइज़ेबल होस्ट और भाषाओं के साथ ऑडियो ओवरव्यू | कई TTS प्रदाताओं के साथ उपलब्ध (NotebookLM यहाँ बेहतर है, सक्रिय रूप से सुधार हो रहा है) |
|
||||
| **डेस्कटॉप ऐप** | नहीं | General Assist, Quick Assist, Extreme Assist और लोकल फ़ोल्डर सिंक के साथ नेटिव ऐप |
|
||||
| **डेस्कटॉप ऐप** | नहीं | General Assist, Quick Assist, Screenshot Assist और लोकल फ़ोल्डर सिंक के साथ नेटिव ऐप |
|
||||
| **ब्राउज़र एक्सटेंशन** | नहीं | किसी भी वेबपेज को सहेजने के लिए क्रॉस-ब्राउज़र एक्सटेंशन, प्रमाणीकरण सुरक्षित पेज सहित |
|
||||
|
||||
<details>
|
||||
<summary><b>बाहरी स्रोतों की पूरी सूची</b></summary>
|
||||
<a id="बाहरी-स्रोत"></a>
|
||||
|
||||
सर्च इंजन (Tavily, LinkUp) · SearxNG · Google Drive · OneDrive · Dropbox · Slack · Microsoft Teams · Linear · Jira · ClickUp · Confluence · BookStack · Notion · Gmail · YouTube वीडियो · GitHub · Discord · Airtable · Google Calendar · Luma · Circleback · Elasticsearch · Obsidian, और भी बहुत कुछ आने वाला है।
|
||||
सर्च इंजन (SearXNG, Tavily, LinkUp, Baidu Search) · Google Drive · OneDrive · Dropbox · Slack · Microsoft Teams · Linear · Jira · ClickUp · Confluence · BookStack · Notion · Gmail · YouTube वीडियो · GitHub · Discord · Airtable · Google Calendar · Luma · Circleback · Elasticsearch · Obsidian, और भी बहुत कुछ आने वाला है।
|
||||
|
||||
</details>
|
||||
|
||||
|
|
|
|||
12
README.md
12
README.md
|
|
@ -42,7 +42,7 @@ NotebookLM is one of the best and most useful AI platforms out there, but once y
|
|||
- **25+ External Data Sources** - Add your sources from Google Drive, OneDrive, Dropbox, Notion, and many other external services.
|
||||
- **Real-Time Multiplayer Support** - Work easily with your team members in a shared notebook.
|
||||
- **AI File Sorting** - Automatically organize your documents into a smart folder hierarchy using AI-powered categorization by source, date, and topic.
|
||||
- **Desktop App** - Get AI assistance in any application with Quick Assist, General Assist, Extreme Assist, and local folder sync.
|
||||
- **Desktop App** - Get AI assistance in any application with Quick Assist, General Assist, Screenshot Assist, and local folder sync.
|
||||
|
||||
...and more to come.
|
||||
|
||||
|
|
@ -85,9 +85,9 @@ https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7
|
|||
|
||||
<p align="center"><img src="surfsense_web/public/homepage/hero_tutorial/quick_assist.gif" alt="Quick Assist" /></p>
|
||||
|
||||
- Desktop App — Extreme Assist
|
||||
- Desktop App — Screenshot Assist
|
||||
|
||||
<p align="center"><img src="surfsense_web/public/homepage/hero_tutorial/extreme_assist.gif" alt="Extreme Assist" /></p>
|
||||
<p align="center"><img src="surfsense_web/public/homepage/hero_tutorial/screenshot_assist.gif" alt="Screenshot Assist" /></p>
|
||||
|
||||
- Desktop App — Watch Local Folder
|
||||
|
||||
|
|
@ -151,7 +151,7 @@ The desktop app includes these powerful features:
|
|||
|
||||
- **General Assist** — Launch SurfSense instantly from any application with a global shortcut.
|
||||
- **Quick Assist** — Select text anywhere, then ask AI to explain, rewrite, or act on it.
|
||||
- **Extreme Assist** — Get inline writing suggestions powered by your knowledge base as you type in any app.
|
||||
- **Screenshot Assist** — Select a region on your screen and attach it to chat so answers stay grounded in your knowledge base.
|
||||
- **Watch Local Folder** — Watch a local folder and automatically sync file changes to your knowledge base. **Pro tip:** Point it at your Obsidian vault to keep your notes searchable in SurfSense.
|
||||
|
||||
All features operate against your chosen search space, so your answers are always grounded in your own data.
|
||||
|
|
@ -201,14 +201,14 @@ All features operate against your chosen search space, so your answers are alway
|
|||
| **Presentation Generation** | Better looking slides but not editable | Create editable, slide-based presentations |
|
||||
| **Podcast Generation** | Audio Overviews with customizable hosts and languages | Available with multiple TTS providers (NotebookLM is better here, actively improving) |
|
||||
| **AI File Sorting** | No | LLM-powered auto-categorization into source, date, category, and subcategory folders |
|
||||
| **Desktop App** | No | Native app with General Assist, Quick Assist, Extreme Assist, and local folder sync |
|
||||
| **Desktop App** | No | Native app with General Assist, Quick Assist, Screenshot Assist, and local folder sync |
|
||||
| **Browser Extension** | No | Cross-browser extension to save any webpage, including auth-protected pages |
|
||||
|
||||
<details>
|
||||
<summary><b>Full list of External Sources</b></summary>
|
||||
<a id="external-sources"></a>
|
||||
|
||||
Search Engines (Tavily, LinkUp) · SearxNG · Google Drive · OneDrive · Dropbox · Slack · Microsoft Teams · Linear · Jira · ClickUp · Confluence · BookStack · Notion · Gmail · YouTube Videos · GitHub · Discord · Airtable · Google Calendar · Luma · Circleback · Elasticsearch · Obsidian, and more to come.
|
||||
Search Engines (SearXNG, Tavily, LinkUp, Baidu Search) · Google Drive · OneDrive · Dropbox · Slack · Microsoft Teams · Linear · Jira · ClickUp · Confluence · BookStack · Notion · Gmail · YouTube Videos · GitHub · Discord · Airtable · Google Calendar · Luma · Circleback · Elasticsearch · Obsidian, and more to come.
|
||||
|
||||
</details>
|
||||
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ O NotebookLM é uma das melhores e mais úteis plataformas de IA disponíveis, m
|
|||
- **Sem Dependência de Fornecedor** - Configure qualquer modelo LLM, de imagem, TTS e STT.
|
||||
- **25+ Fontes de Dados Externas** - Adicione suas fontes do Google Drive, OneDrive, Dropbox, Notion e muitos outros serviços externos.
|
||||
- **Suporte Multiplayer em Tempo Real** - Trabalhe facilmente com os membros da sua equipe em um notebook compartilhado.
|
||||
- **Aplicativo Desktop** - Obtenha assistência de IA em qualquer aplicativo com Quick Assist, General Assist, Extreme Assist e sincronização de pastas locais.
|
||||
- **Aplicativo Desktop** - Obtenha assistência de IA em qualquer aplicativo com Quick Assist, General Assist, Screenshot Assist e sincronização de pastas locais.
|
||||
|
||||
...e mais por vir.
|
||||
|
||||
|
|
@ -84,9 +84,9 @@ https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7
|
|||
|
||||
<p align="center"><img src="surfsense_web/public/homepage/hero_tutorial/quick_assist.gif" alt="Quick Assist" /></p>
|
||||
|
||||
- Aplicativo Desktop — Extreme Assist
|
||||
- Aplicativo Desktop — Screenshot Assist
|
||||
|
||||
<p align="center"><img src="surfsense_web/public/homepage/hero_tutorial/extreme_assist.gif" alt="Extreme Assist" /></p>
|
||||
<p align="center"><img src="surfsense_web/public/homepage/hero_tutorial/screenshot_assist.gif" alt="Screenshot Assist" /></p>
|
||||
|
||||
- Aplicativo Desktop — Watch Local Folder
|
||||
|
||||
|
|
@ -150,7 +150,7 @@ O aplicativo desktop inclui estes recursos poderosos:
|
|||
|
||||
- **General Assist** — Abra o SurfSense instantaneamente de qualquer aplicativo com um atalho global.
|
||||
- **Quick Assist** — Selecione texto em qualquer lugar, depois peça à IA para explicar, reescrever ou agir sobre ele.
|
||||
- **Extreme Assist** — Receba sugestões de escrita em linha alimentadas pela sua base de conhecimento enquanto digita em qualquer aplicativo.
|
||||
- **Screenshot Assist** — Selecione uma região da tela e anexe ao chat para respostas fundamentadas na sua base de conhecimento.
|
||||
- **Watch Local Folder** — Monitore uma pasta local e sincronize automaticamente as alterações de arquivos com sua base de conhecimento. **Pro tip:** Aponte para seu cofre do Obsidian para manter suas notas pesquisáveis no SurfSense.
|
||||
|
||||
Todos os recursos operam no espaço de busca escolhido, para que suas respostas sejam sempre baseadas nos seus próprios dados.
|
||||
|
|
@ -199,14 +199,14 @@ Todos os recursos operam no espaço de busca escolhido, para que suas respostas
|
|||
| **Geração de Vídeos** | Visões gerais cinemáticas via Veo 3 (apenas Ultra) | Disponível (NotebookLM é melhor aqui, melhorando ativamente) |
|
||||
| **Geração de Apresentações** | Slides mais bonitos mas não editáveis | Cria apresentações editáveis baseadas em slides |
|
||||
| **Geração de Podcasts** | Visões gerais em áudio com hosts e idiomas personalizáveis | Disponível com múltiplos provedores TTS (NotebookLM é melhor aqui, melhorando ativamente) |
|
||||
| **Aplicativo Desktop** | Não | Aplicativo nativo com General Assist, Quick Assist, Extreme Assist e sincronização de pastas locais |
|
||||
| **Aplicativo Desktop** | Não | Aplicativo nativo com General Assist, Quick Assist, Screenshot Assist e sincronização de pastas locais |
|
||||
| **Extensão de Navegador** | Não | Extensão multi-navegador para salvar qualquer página web, incluindo páginas protegidas por autenticação |
|
||||
|
||||
<details>
|
||||
<summary><b>Lista completa de Fontes Externas</b></summary>
|
||||
<a id="fontes-externas"></a>
|
||||
|
||||
Mecanismos de Busca (Tavily, LinkUp) · SearxNG · Google Drive · OneDrive · Dropbox · Slack · Microsoft Teams · Linear · Jira · ClickUp · Confluence · BookStack · Notion · Gmail · Vídeos do YouTube · GitHub · Discord · Airtable · Google Calendar · Luma · Circleback · Elasticsearch · Obsidian, e mais por vir.
|
||||
Mecanismos de Busca (SearXNG, Tavily, LinkUp, Baidu Search) · Google Drive · OneDrive · Dropbox · Slack · Microsoft Teams · Linear · Jira · ClickUp · Confluence · BookStack · Notion · Gmail · Vídeos do YouTube · GitHub · Discord · Airtable · Google Calendar · Luma · Circleback · Elasticsearch · Obsidian, e mais por vir.
|
||||
|
||||
</details>
|
||||
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ NotebookLM 是目前最好、最实用的 AI 平台之一，但当你开始经
|
|||
- **无供应商锁定** - 配置任何 LLM、图像、TTS 和 STT 模型。
|
||||
- **25+ 外部数据源** - 从 Google Drive、OneDrive、Dropbox、Notion 和许多其他外部服务添加你的来源。
|
||||
- **实时多人协作支持** - 在共享笔记本中轻松与团队成员协作。
|
||||
- **桌面应用** - 通过 Quick Assist、General Assist、Extreme Assist 和本地文件夹同步在任何应用程序中获得 AI 助手。
|
||||
- **桌面应用** - 通过 Quick Assist、General Assist、Screenshot Assist 和本地文件夹同步在任何应用程序中获得 AI 助手。
|
||||
|
||||
...更多功能即将推出。
|
||||
|
||||
|
|
@ -84,9 +84,9 @@ https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7
|
|||
|
||||
<p align="center"><img src="surfsense_web/public/homepage/hero_tutorial/quick_assist.gif" alt="Quick Assist" /></p>
|
||||
|
||||
- 桌面应用 — Extreme Assist
|
||||
- 桌面应用 — Screenshot Assist
|
||||
|
||||
<p align="center"><img src="surfsense_web/public/homepage/hero_tutorial/extreme_assist.gif" alt="Extreme Assist" /></p>
|
||||
<p align="center"><img src="surfsense_web/public/homepage/hero_tutorial/screenshot_assist.gif" alt="Screenshot Assist" /></p>
|
||||
|
||||
- 桌面应用 — Watch Local Folder
|
||||
|
||||
|
|
@ -150,7 +150,7 @@ SurfSense 还提供桌面应用，将 AI 助手带到您计算机上的每个应
|
|||
|
||||
- **General Assist** — 通过全局快捷键从任何应用程序即时启动 SurfSense。
|
||||
- **Quick Assist** — 在任何位置选中文本，然后让 AI 解释、改写或对其执行操作。
|
||||
- **Extreme Assist** — 在任何应用中输入时，获得基于您知识库的内联写作建议。
|
||||
- **Screenshot Assist** — 在屏幕上框选区域并附加到聊天，让回复基于您的知识库。
|
||||
- **Watch Local Folder** — 监视本地文件夹，自动将文件更改同步到您的知识库。**Pro tip：** 将其指向您的 Obsidian vault，让笔记在 SurfSense 中随时可搜索。
|
||||
|
||||
所有功能均基于您选择的搜索空间运行，确保回答始终以您自己的数据为依据。
|
||||
|
|
@ -199,14 +199,14 @@ SurfSense 还提供桌面应用，将 AI 助手带到您计算机上的每个应
|
|||
| **视频生成** | 通过 Veo 3 的电影级视频概览（仅 Ultra） | 可用（NotebookLM 在此方面更好，正在积极改进） |
|
||||
| **演示文稿生成** | 更美观的幻灯片但不可编辑 | 创建可编辑的幻灯片式演示文稿 |
|
||||
| **播客生成** | 可自定义主持人和语言的音频概览 | 可用，支持多种 TTS 提供商（NotebookLM 在此方面更好，正在积极改进） |
|
||||
| **桌面应用** | 否 | 原生应用，包含 General Assist、Quick Assist、Extreme Assist 和本地文件夹同步 |
|
||||
| **桌面应用** | 否 | 原生应用，包含 General Assist、Quick Assist、Screenshot Assist 和本地文件夹同步 |
|
||||
| **浏览器扩展** | 否 | 跨浏览器扩展，保存任何网页，包括需要身份验证的页面 |
|
||||
|
||||
<details>
|
||||
<summary><b>外部数据源完整列表</b></summary>
|
||||
<a id="外部数据源"></a>
|
||||
|
||||
搜索引擎（Tavily、LinkUp）· SearxNG · Google Drive · OneDrive · Dropbox · Slack · Microsoft Teams · Linear · Jira · ClickUp · Confluence · BookStack · Notion · Gmail · YouTube 视频 · GitHub · Discord · Airtable · Google Calendar · Luma · Circleback · Elasticsearch · Obsidian，更多即将推出。
|
||||
搜索引擎（SearXNG、Tavily、LinkUp、Baidu Search）· Google Drive · OneDrive · Dropbox · Slack · Microsoft Teams · Linear · Jira · ClickUp · Confluence · BookStack · Notion · Gmail · YouTube 视频 · GitHub · Discord · Airtable · Google Calendar · Luma · Circleback · Elasticsearch · Obsidian，更多即将推出。
|
||||
|
||||
</details>
|
||||
|
||||
|
|
|
|||
123
docker/docker-compose.deps-only.yml
Normal file
123
docker/docker-compose.deps-only.yml
Normal file
|
|
@ -0,0 +1,123 @@
|
|||
# =============================================================================
|
||||
# SurfSense — Dependencies only (no backend / frontend / Celery images)
|
||||
# =============================================================================
|
||||
# Postgres, Redis, SearXNG, pgAdmin, Zero — run API + Next + Celery on the host.
|
||||
# Celery is not Dockerized here: use `uv run` from surfsense_backend/ (no extra
|
||||
# backend image build just for workers).
|
||||
#
|
||||
# From repo root (SurfSense/):
|
||||
# docker compose -f docker/docker-compose.deps-only.yml up -d
|
||||
#
|
||||
# Compose variable substitution uses `docker/.env` (copy from .env.example).
|
||||
# Bind mounts use ./postgresql.conf and ./searxng in this directory.
|
||||
#
|
||||
# Local Celery (from surfsense_backend/, after Redis is up):
|
||||
# uv run celery -A celery_worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo --queues=surfsense,surfsense.connectors
|
||||
# uv run celery -A celery_worker.celery_app beat --loglevel=info
|
||||
#
|
||||
# Host setup:
|
||||
# - Backend .env: DATABASE_URL=postgresql+asyncpg://postgres:postgres@localhost:5432/surfsense
|
||||
# - Backend .env: SEARXNG_DEFAULT_HOST=http://localhost:${SEARXNG_PORT:-8888}
|
||||
# - Backend .env: CELERY_BROKER_URL / REDIS_APP_URL → redis://localhost:6379/0
|
||||
# - Web .env: NEXT_PUBLIC_ZERO_CACHE_URL=http://localhost:${ZERO_CACHE_PORT:-4848}
|
||||
# =============================================================================
|
||||
|
||||
name: surfsense-deps
|
||||
|
||||
services:
|
||||
db:
|
||||
image: pgvector/pgvector:pg17
|
||||
ports:
|
||||
- "${POSTGRES_PORT:-5432}:5432"
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
- ./postgresql.conf:/etc/postgresql/postgresql.conf:ro
|
||||
environment:
|
||||
- POSTGRES_USER=${DB_USER:-postgres}
|
||||
- POSTGRES_PASSWORD=${DB_PASSWORD:-postgres}
|
||||
- POSTGRES_DB=${DB_NAME:-surfsense}
|
||||
command: postgres -c config_file=/etc/postgresql/postgresql.conf
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-postgres} -d ${DB_NAME:-surfsense}"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
pgadmin:
|
||||
image: dpage/pgadmin4
|
||||
ports:
|
||||
- "${PGADMIN_PORT:-5050}:80"
|
||||
environment:
|
||||
- PGADMIN_DEFAULT_EMAIL=${PGADMIN_DEFAULT_EMAIL:-admin@surfsense.com}
|
||||
- PGADMIN_DEFAULT_PASSWORD=${PGADMIN_DEFAULT_PASSWORD:-surfsense}
|
||||
volumes:
|
||||
- pgadmin_data:/var/lib/pgadmin
|
||||
depends_on:
|
||||
- db
|
||||
|
||||
redis:
|
||||
image: redis:8-alpine
|
||||
ports:
|
||||
- "${REDIS_PORT:-6379}:6379"
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
command: redis-server --appendonly yes
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
searxng:
|
||||
image: searxng/searxng:2026.3.13-3c1f68c59
|
||||
ports:
|
||||
- "${SEARXNG_PORT:-8888}:8080"
|
||||
volumes:
|
||||
- ./searxng:/etc/searxng
|
||||
environment:
|
||||
- SEARXNG_SECRET=${SEARXNG_SECRET:-surfsense-searxng-secret}
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "--spider", "-q", "http://localhost:8080/healthz"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
zero-cache:
|
||||
image: rocicorp/zero:0.26.2
|
||||
ports:
|
||||
- "${ZERO_CACHE_PORT:-4848}:4848"
|
||||
extra_hosts:
|
||||
- "host.docker.internal:host-gateway"
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
- ZERO_UPSTREAM_DB=postgresql://${DB_USER:-postgres}:${DB_PASSWORD:-postgres}@db:5432/${DB_NAME:-surfsense}?sslmode=${DB_SSLMODE:-disable}
|
||||
- ZERO_CVR_DB=postgresql://${DB_USER:-postgres}:${DB_PASSWORD:-postgres}@db:5432/${DB_NAME:-surfsense}?sslmode=${DB_SSLMODE:-disable}
|
||||
- ZERO_CHANGE_DB=postgresql://${DB_USER:-postgres}:${DB_PASSWORD:-postgres}@db:5432/${DB_NAME:-surfsense}?sslmode=${DB_SSLMODE:-disable}
|
||||
- ZERO_REPLICA_FILE=/data/zero.db
|
||||
- ZERO_ADMIN_PASSWORD=${ZERO_ADMIN_PASSWORD:-surfsense-zero-admin}
|
||||
- ZERO_APP_PUBLICATIONS=${ZERO_APP_PUBLICATIONS:-zero_publication}
|
||||
- ZERO_NUM_SYNC_WORKERS=${ZERO_NUM_SYNC_WORKERS:-4}
|
||||
- ZERO_UPSTREAM_MAX_CONNS=${ZERO_UPSTREAM_MAX_CONNS:-20}
|
||||
- ZERO_CVR_MAX_CONNS=${ZERO_CVR_MAX_CONNS:-30}
|
||||
- ZERO_QUERY_URL=${ZERO_QUERY_URL:-http://host.docker.internal:3000/api/zero/query}
|
||||
- ZERO_MUTATE_URL=${ZERO_MUTATE_URL:-http://host.docker.internal:3000/api/zero/mutate}
|
||||
volumes:
|
||||
- zero_cache_data:/data
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:4848/keepalive"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
name: surfsense-deps-postgres
|
||||
pgadmin_data:
|
||||
name: surfsense-deps-pgadmin
|
||||
redis_data:
|
||||
name: surfsense-deps-redis
|
||||
zero_cache_data:
|
||||
name: surfsense-deps-zero-cache
|
||||
|
|
@ -79,40 +79,44 @@ def _terminate_blocked_pids(conn, table: str) -> None:
|
|||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
# asyncpg requires LOCK TABLE inside a transaction block. Alembic already
|
||||
# opened one via context.begin_transaction(), but the driver still errors
|
||||
# unless we use an explicit SAVEPOINT (nested transaction) for this block.
|
||||
tx = conn.begin_nested() if conn.in_transaction() else conn.begin()
|
||||
with tx:
|
||||
conn.execute(sa.text("SET lock_timeout = '10s'"))
|
||||
|
||||
conn.execute(sa.text("SET lock_timeout = '10s'"))
|
||||
for tbl in sorted(TABLES_WITH_FULL_IDENTITY):
|
||||
_terminate_blocked_pids(conn, tbl)
|
||||
conn.execute(sa.text(f'LOCK TABLE "{tbl}" IN ACCESS EXCLUSIVE MODE'))
|
||||
|
||||
for tbl in sorted(TABLES_WITH_FULL_IDENTITY):
|
||||
_terminate_blocked_pids(conn, tbl)
|
||||
conn.execute(sa.text(f'LOCK TABLE "{tbl}" IN ACCESS EXCLUSIVE MODE'))
|
||||
for tbl in TABLES_WITH_FULL_IDENTITY:
|
||||
conn.execute(sa.text(f'ALTER TABLE "{tbl}" REPLICA IDENTITY DEFAULT'))
|
||||
|
||||
for tbl in TABLES_WITH_FULL_IDENTITY:
|
||||
conn.execute(sa.text(f'ALTER TABLE "{tbl}" REPLICA IDENTITY DEFAULT'))
|
||||
conn.execute(sa.text(f"DROP PUBLICATION IF EXISTS {PUBLICATION_NAME}"))
|
||||
|
||||
conn.execute(sa.text(f"DROP PUBLICATION IF EXISTS {PUBLICATION_NAME}"))
|
||||
has_zero_ver = conn.execute(
|
||||
sa.text(
|
||||
"SELECT 1 FROM information_schema.columns "
|
||||
"WHERE table_name = 'documents' AND column_name = '_0_version'"
|
||||
)
|
||||
).fetchone()
|
||||
|
||||
has_zero_ver = conn.execute(
|
||||
sa.text(
|
||||
"SELECT 1 FROM information_schema.columns "
|
||||
"WHERE table_name = 'documents' AND column_name = '_0_version'"
|
||||
cols = DOCUMENT_COLS + (['"_0_version"'] if has_zero_ver else [])
|
||||
col_list = ", ".join(cols)
|
||||
|
||||
conn.execute(
|
||||
sa.text(
|
||||
f"CREATE PUBLICATION {PUBLICATION_NAME} FOR TABLE "
|
||||
f"notifications, "
|
||||
f"documents ({col_list}), "
|
||||
f"folders, "
|
||||
f"search_source_connectors, "
|
||||
f"new_chat_messages, "
|
||||
f"chat_comments, "
|
||||
f"chat_session_state"
|
||||
)
|
||||
)
|
||||
).fetchone()
|
||||
|
||||
cols = DOCUMENT_COLS + (['"_0_version"'] if has_zero_ver else [])
|
||||
col_list = ", ".join(cols)
|
||||
|
||||
conn.execute(
|
||||
sa.text(
|
||||
f"CREATE PUBLICATION {PUBLICATION_NAME} FOR TABLE "
|
||||
f"notifications, "
|
||||
f"documents ({col_list}), "
|
||||
f"folders, "
|
||||
f"search_source_connectors, "
|
||||
f"new_chat_messages, "
|
||||
f"chat_comments, "
|
||||
f"chat_session_state"
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
|
|
|
|||
|
|
@ -12,8 +12,6 @@ from __future__ import annotations
|
|||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "121"
|
||||
|
|
@ -23,16 +21,30 @@ depends_on: str | Sequence[str] | None = None
|
|||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"user",
|
||||
sa.Column("memory_md", sa.Text(), nullable=True, server_default=""),
|
||||
)
|
||||
op.add_column(
|
||||
"searchspaces",
|
||||
sa.Column("shared_memory_md", sa.Text(), nullable=True, server_default=""),
|
||||
# Idempotent: column(s) may already exist after a failed run or manual DDL.
|
||||
op.execute(
|
||||
"""
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM information_schema.columns
|
||||
WHERE table_schema = 'public' AND table_name = 'user'
|
||||
AND column_name = 'memory_md'
|
||||
) THEN
|
||||
ALTER TABLE "user" ADD COLUMN memory_md TEXT DEFAULT '';
|
||||
END IF;
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM information_schema.columns
|
||||
WHERE table_schema = 'public' AND table_name = 'searchspaces'
|
||||
AND column_name = 'shared_memory_md'
|
||||
) THEN
|
||||
ALTER TABLE searchspaces ADD COLUMN shared_memory_md TEXT DEFAULT '';
|
||||
END IF;
|
||||
END$$;
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("searchspaces", "shared_memory_md")
|
||||
op.drop_column("user", "memory_md")
|
||||
op.execute("ALTER TABLE searchspaces DROP COLUMN IF EXISTS shared_memory_md")
|
||||
op.execute('ALTER TABLE "user" DROP COLUMN IF EXISTS memory_md')
|
||||
|
|
|
|||
|
|
@ -1,11 +0,0 @@
|
|||
"""Agent-based vision autocomplete with scoped filesystem exploration."""
|
||||
|
||||
from app.agents.autocomplete.autocomplete_agent import (
|
||||
create_autocomplete_agent,
|
||||
stream_autocomplete_agent,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"create_autocomplete_agent",
|
||||
"stream_autocomplete_agent",
|
||||
]
|
||||
|
|
@ -7,7 +7,6 @@ from .agent_revert_route import router as agent_revert_router
|
|||
from .airtable_add_connector_route import (
|
||||
router as airtable_add_connector_router,
|
||||
)
|
||||
from .autocomplete_routes import router as autocomplete_router
|
||||
from .chat_comments_routes import router as chat_comments_router
|
||||
from .circleback_webhook_route import router as circleback_webhook_router
|
||||
from .clickup_add_connector_route import router as clickup_add_connector_router
|
||||
|
|
@ -118,4 +117,3 @@ router.include_router(stripe_router) # Stripe checkout for additional page pack
|
|||
router.include_router(youtube_router) # YouTube playlist resolution
|
||||
router.include_router(prompts_router)
|
||||
router.include_router(memory_router) # User personal memory (memory.md style)
|
||||
router.include_router(autocomplete_router) # Lightweight autocomplete with KB context
|
||||
|
|
|
|||
|
|
@ -1,45 +0,0 @@
|
|||
from fastapi import APIRouter, Depends
|
||||
from fastapi.responses import StreamingResponse
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db import User, get_async_session
|
||||
from app.services.new_streaming_service import VercelStreamingService
|
||||
from app.services.vision_autocomplete_service import stream_vision_autocomplete
|
||||
from app.users import current_active_user
|
||||
from app.utils.rbac import check_search_space_access
|
||||
|
||||
router = APIRouter(prefix="/autocomplete", tags=["autocomplete"])
|
||||
|
||||
MAX_SCREENSHOT_SIZE = 20 * 1024 * 1024 # 20 MB base64 ceiling
|
||||
|
||||
|
||||
class VisionAutocompleteRequest(BaseModel):
|
||||
screenshot: str = Field(..., max_length=MAX_SCREENSHOT_SIZE)
|
||||
search_space_id: int
|
||||
app_name: str = ""
|
||||
window_title: str = ""
|
||||
|
||||
|
||||
@router.post("/vision/stream")
|
||||
async def vision_autocomplete_stream(
|
||||
body: VisionAutocompleteRequest,
|
||||
user: User = Depends(current_active_user),
|
||||
session: AsyncSession = Depends(get_async_session),
|
||||
):
|
||||
await check_search_space_access(session, user, body.search_space_id)
|
||||
|
||||
return StreamingResponse(
|
||||
stream_vision_autocomplete(
|
||||
body.screenshot,
|
||||
body.search_space_id,
|
||||
session,
|
||||
app_name=body.app_name,
|
||||
window_title=body.window_title,
|
||||
),
|
||||
media_type="text/event-stream",
|
||||
headers={
|
||||
**VercelStreamingService.get_response_headers(),
|
||||
"X-Accel-Buffering": "no",
|
||||
},
|
||||
)
|
||||
|
|
@ -64,6 +64,10 @@ from app.services.token_tracking_service import record_token_usage
|
|||
from app.tasks.chat.stream_new_chat import stream_new_chat, stream_resume_chat
|
||||
from app.users import current_active_user
|
||||
from app.utils.rbac import check_permission
|
||||
from app.utils.user_message_multimodal import (
|
||||
split_langchain_human_content,
|
||||
split_persisted_user_content_parts,
|
||||
)
|
||||
|
||||
_logger = logging.getLogger(__name__)
|
||||
_background_tasks: set[asyncio.Task] = set()
|
||||
|
|
@ -1237,6 +1241,10 @@ async def handle_new_chat(
|
|||
# connection (the "Exception terminating connection" errors).
|
||||
await session.close()
|
||||
|
||||
image_urls = (
|
||||
[p.as_data_url() for p in request.user_images] if request.user_images else None
|
||||
)
|
||||
|
||||
return StreamingResponse(
|
||||
stream_new_chat(
|
||||
user_query=request.user_query,
|
||||
|
|
@ -1252,6 +1260,7 @@ async def handle_new_chat(
|
|||
disabled_tools=request.disabled_tools,
|
||||
filesystem_selection=filesystem_selection,
|
||||
request_id=getattr(http_request.state, "request_id", "unknown"),
|
||||
user_image_data_urls=image_urls,
|
||||
),
|
||||
media_type="text/event-stream",
|
||||
headers={
|
||||
|
|
@ -1360,6 +1369,7 @@ async def regenerate_response(
|
|||
|
||||
target_checkpoint_id = None
|
||||
user_query_to_use = request.user_query
|
||||
regenerate_image_urls: list[str] = []
|
||||
|
||||
# Look through checkpoints to find the right one
|
||||
# We want to find the checkpoint just before the last HumanMessage
|
||||
|
|
@ -1385,9 +1395,13 @@ async def regenerate_response(
|
|||
prev_messages = prev_channel_values.get("messages", [])
|
||||
for msg in reversed(prev_messages):
|
||||
if isinstance(msg, HumanMessage):
|
||||
user_query_to_use = msg.content
|
||||
q, imgs = split_langchain_human_content(msg.content)
|
||||
user_query_to_use = q
|
||||
regenerate_image_urls = imgs
|
||||
break
|
||||
if user_query_to_use:
|
||||
if user_query_to_use is not None and (
|
||||
str(user_query_to_use).strip() or regenerate_image_urls
|
||||
):
|
||||
break
|
||||
|
||||
target_checkpoint_id = cp_tuple.config["configurable"][
|
||||
|
|
@ -1405,7 +1419,9 @@ async def regenerate_response(
|
|||
state_messages = channel_values.get("messages", [])
|
||||
for msg in state_messages:
|
||||
if isinstance(msg, HumanMessage):
|
||||
user_query_to_use = msg.content
|
||||
q, imgs = split_langchain_human_content(msg.content)
|
||||
user_query_to_use = q
|
||||
regenerate_image_urls = imgs
|
||||
break
|
||||
else:
|
||||
# Use the oldest checkpoint
|
||||
|
|
@ -1431,20 +1447,28 @@ async def regenerate_response(
|
|||
if isinstance(content, str):
|
||||
user_query_to_use = content
|
||||
elif isinstance(content, list):
|
||||
# Extract text from content parts
|
||||
for part in content:
|
||||
if isinstance(part, dict) and part.get("type") == "text":
|
||||
user_query_to_use = part.get("text", "")
|
||||
break
|
||||
elif isinstance(part, str):
|
||||
user_query_to_use = part
|
||||
break
|
||||
plain, imgs = split_persisted_user_content_parts(content)
|
||||
user_query_to_use = plain
|
||||
regenerate_image_urls = imgs
|
||||
|
||||
if isinstance(user_query_to_use, list):
|
||||
user_query_to_use, regenerate_image_urls = split_langchain_human_content(
|
||||
user_query_to_use
|
||||
)
|
||||
|
||||
if request.user_images is not None:
|
||||
regenerate_image_urls = [p.as_data_url() for p in request.user_images]
|
||||
|
||||
if user_query_to_use is None:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Could not determine user query for regeneration. Please provide a user_query.",
|
||||
)
|
||||
if not str(user_query_to_use).strip() and not regenerate_image_urls:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Could not determine user query for regeneration. Please provide a user_query.",
|
||||
)
|
||||
|
||||
# Get the last two messages to delete AFTER streaming succeeds
|
||||
# This prevents data loss if streaming fails
|
||||
|
|
@ -1483,7 +1507,7 @@ async def regenerate_response(
|
|||
streaming_completed = False
|
||||
try:
|
||||
async for chunk in stream_new_chat(
|
||||
user_query=user_query_to_use,
|
||||
user_query=str(user_query_to_use),
|
||||
search_space_id=request.search_space_id,
|
||||
chat_id=thread_id,
|
||||
user_id=str(user.id),
|
||||
|
|
@ -1497,6 +1521,7 @@ async def regenerate_response(
|
|||
disabled_tools=request.disabled_tools,
|
||||
filesystem_selection=filesystem_selection,
|
||||
request_id=getattr(http_request.state, "request_id", "unknown"),
|
||||
user_image_data_urls=regenerate_image_urls or None,
|
||||
):
|
||||
yield chunk
|
||||
streaming_completed = True
|
||||
|
|
|
|||
|
|
@ -7,12 +7,13 @@ These schemas follow the assistant-ui ThreadHistoryAdapter pattern:
|
|||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any, Literal
|
||||
from typing import Any, Literal, Self
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
from app.db import ChatVisibility, NewChatMessageRole
|
||||
from app.utils.user_message_multimodal import decode_base64_image, to_data_url
|
||||
|
||||
from .base import IDModel, TimestampModel
|
||||
|
||||
|
|
@ -173,6 +174,26 @@ class LocalFilesystemMountPayload(BaseModel):
|
|||
root_path: str
|
||||
|
||||
|
||||
MAX_NEW_CHAT_IMAGE_BYTES = 8 * 1024 * 1024
|
||||
MAX_NEW_CHAT_IMAGES = 4
|
||||
|
||||
|
||||
class NewChatUserImagePart(BaseModel):
|
||||
"""One inline image for a user turn (raw base64 body, no data: URL prefix)."""
|
||||
|
||||
media_type: Literal["image/png", "image/jpeg", "image/webp"]
|
||||
data: str = Field(..., min_length=1)
|
||||
|
||||
@field_validator("data")
|
||||
@classmethod
|
||||
def _validate_payload(cls, v: str) -> str:
|
||||
decode_base64_image(v, max_bytes=MAX_NEW_CHAT_IMAGE_BYTES)
|
||||
return v
|
||||
|
||||
def as_data_url(self) -> str:
|
||||
return to_data_url(self.media_type, self.data)
|
||||
|
||||
|
||||
class NewChatRequest(BaseModel):
|
||||
"""Request schema for the deep agent chat endpoint."""
|
||||
|
||||
|
|
@ -192,6 +213,20 @@ class NewChatRequest(BaseModel):
|
|||
filesystem_mode: Literal["cloud", "desktop_local_folder"] = "cloud"
|
||||
client_platform: Literal["web", "desktop"] = "web"
|
||||
local_filesystem_mounts: list[LocalFilesystemMountPayload] | None = None
|
||||
user_images: list[NewChatUserImagePart] | None = Field(
|
||||
default=None,
|
||||
description="Optional images for this user turn",
|
||||
)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _require_text_or_images(self) -> Self:
|
||||
has_text = bool(self.user_query.strip())
|
||||
has_images = bool(self.user_images)
|
||||
if not has_text and not has_images:
|
||||
raise ValueError("Provide non-empty user_query and/or user_images")
|
||||
if self.user_images is not None and len(self.user_images) > MAX_NEW_CHAT_IMAGES:
|
||||
raise ValueError(f"At most {MAX_NEW_CHAT_IMAGES} images allowed")
|
||||
return self
|
||||
|
||||
|
||||
class RegenerateRequest(BaseModel):
|
||||
|
|
@ -203,6 +238,9 @@ class RegenerateRequest(BaseModel):
|
|||
2. Reload: Leave user_query empty to regenerate the last AI response with the same query
|
||||
|
||||
Both operations rewind the LangGraph checkpointer to the appropriate state.
|
||||
|
||||
For edit, optional user_images (when not None) replaces image URLs resolved from
|
||||
checkpoint/DB so the client can send the full user turn (text and/or images).
|
||||
"""
|
||||
|
||||
search_space_id: int
|
||||
|
|
@ -215,6 +253,16 @@ class RegenerateRequest(BaseModel):
|
|||
filesystem_mode: Literal["cloud", "desktop_local_folder"] = "cloud"
|
||||
client_platform: Literal["web", "desktop"] = "web"
|
||||
local_filesystem_mounts: list[LocalFilesystemMountPayload] | None = None
|
||||
user_images: list[NewChatUserImagePart] | None = Field(
|
||||
default=None,
|
||||
description="If set, use these images for the regenerated turn (edit); overrides checkpoint/DB",
|
||||
)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _validate_regenerate_user_images(self) -> Self:
|
||||
if self.user_images is not None and len(self.user_images) > MAX_NEW_CHAT_IMAGES:
|
||||
raise ValueError(f"At most {MAX_NEW_CHAT_IMAGES} images allowed")
|
||||
return self
|
||||
|
||||
|
||||
# =============================================================================
|
||||
|
|
|
|||
|
|
@ -1,158 +0,0 @@
|
|||
"""Vision autocomplete service — agent-based with scoped filesystem.
|
||||
|
||||
Optimized pipeline:
|
||||
1. Start the SSE stream immediately so the UI shows progress.
|
||||
2. Derive a KB search query from window_title (no separate LLM call).
|
||||
3. Run KB filesystem pre-computation and agent graph compilation in PARALLEL.
|
||||
4. Inject pre-computed KB files as initial state and stream the agent.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from collections.abc import AsyncGenerator
|
||||
|
||||
from langchain_core.messages import HumanMessage
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.agents.autocomplete import create_autocomplete_agent, stream_autocomplete_agent
|
||||
from app.services.llm_service import get_vision_llm
|
||||
from app.services.new_streaming_service import VercelStreamingService
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
PREP_STEP_ID = "autocomplete-prep"
|
||||
|
||||
|
||||
def _derive_kb_query(app_name: str, window_title: str) -> str:
|
||||
parts = [p for p in (window_title, app_name) if p]
|
||||
return " ".join(parts)
|
||||
|
||||
|
||||
def _is_vision_unsupported_error(e: Exception) -> bool:
|
||||
msg = str(e).lower()
|
||||
return "content must be a string" in msg or "does not support image" in msg
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main entry point
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def stream_vision_autocomplete(
|
||||
screenshot_data_url: str,
|
||||
search_space_id: int,
|
||||
session: AsyncSession,
|
||||
*,
|
||||
app_name: str = "",
|
||||
window_title: str = "",
|
||||
) -> AsyncGenerator[str, None]:
|
||||
"""Analyze a screenshot with a vision-LLM agent and stream a text completion."""
|
||||
streaming = VercelStreamingService()
|
||||
vision_error_msg = (
|
||||
"The selected model does not support vision. "
|
||||
"Please set a vision-capable model (e.g. GPT-4o, Gemini) in your search space settings."
|
||||
)
|
||||
|
||||
llm = await get_vision_llm(session, search_space_id)
|
||||
if not llm:
|
||||
yield streaming.format_message_start()
|
||||
yield streaming.format_error("No Vision LLM configured for this search space")
|
||||
yield streaming.format_done()
|
||||
return
|
||||
|
||||
# Start SSE stream immediately so the UI has something to show
|
||||
yield streaming.format_message_start()
|
||||
|
||||
kb_query = _derive_kb_query(app_name, window_title)
|
||||
|
||||
# Show a preparation step while KB search + agent compile run
|
||||
yield streaming.format_thinking_step(
|
||||
step_id=PREP_STEP_ID,
|
||||
title="Searching knowledge base",
|
||||
status="in_progress",
|
||||
items=[kb_query] if kb_query else [],
|
||||
)
|
||||
|
||||
try:
|
||||
agent, kb = await create_autocomplete_agent(
|
||||
llm,
|
||||
search_space_id=search_space_id,
|
||||
kb_query=kb_query,
|
||||
app_name=app_name,
|
||||
window_title=window_title,
|
||||
)
|
||||
except Exception as e:
|
||||
if _is_vision_unsupported_error(e):
|
||||
logger.warning("Vision autocomplete: model does not support vision: %s", e)
|
||||
yield streaming.format_error(vision_error_msg)
|
||||
yield streaming.format_done()
|
||||
return
|
||||
logger.error("Failed to create autocomplete agent: %s", e, exc_info=True)
|
||||
yield streaming.format_error("Autocomplete failed. Please try again.")
|
||||
yield streaming.format_done()
|
||||
return
|
||||
|
||||
has_kb = kb.has_documents
|
||||
doc_count = len(kb.files) if has_kb else 0 # type: ignore[arg-type]
|
||||
|
||||
yield streaming.format_thinking_step(
|
||||
step_id=PREP_STEP_ID,
|
||||
title="Searching knowledge base",
|
||||
status="complete",
|
||||
items=[f"Found {doc_count} document{'s' if doc_count != 1 else ''}"]
|
||||
if kb_query
|
||||
else ["Skipped"],
|
||||
)
|
||||
|
||||
# Build agent input with pre-computed KB as initial state
|
||||
if has_kb:
|
||||
instruction = (
|
||||
"Analyze this screenshot, then explore the knowledge base documents "
|
||||
"listed above — read the chunk index of any document whose title "
|
||||
"looks relevant and check matched chunks for useful facts. "
|
||||
"Finally, generate a concise autocomplete for the active text area, "
|
||||
"enhanced with any relevant KB information you found."
|
||||
)
|
||||
else:
|
||||
instruction = (
|
||||
"Analyze this screenshot and generate a concise autocomplete "
|
||||
"for the active text area based on what you see."
|
||||
)
|
||||
|
||||
user_message = HumanMessage(
|
||||
content=[
|
||||
{"type": "text", "text": instruction},
|
||||
{"type": "image_url", "image_url": {"url": screenshot_data_url}},
|
||||
]
|
||||
)
|
||||
|
||||
input_data: dict = {"messages": [user_message]}
|
||||
|
||||
if has_kb:
|
||||
input_data["files"] = kb.files
|
||||
input_data["messages"] = [kb.ls_ai_msg, kb.ls_tool_msg, user_message]
|
||||
logger.info(
|
||||
"Autocomplete: injected %d KB files into agent initial state", doc_count
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
"Autocomplete: no KB documents found, proceeding with screenshot only"
|
||||
)
|
||||
|
||||
# Stream the agent (message_start already sent above)
|
||||
try:
|
||||
async for sse in stream_autocomplete_agent(
|
||||
agent,
|
||||
input_data,
|
||||
streaming,
|
||||
emit_message_start=False,
|
||||
):
|
||||
yield sse
|
||||
except Exception as e:
|
||||
if _is_vision_unsupported_error(e):
|
||||
logger.warning("Vision autocomplete: model does not support vision: %s", e)
|
||||
yield streaming.format_error(vision_error_msg)
|
||||
yield streaming.format_done()
|
||||
else:
|
||||
logger.error("Vision autocomplete streaming error: %s", e, exc_info=True)
|
||||
yield streaming.format_error("Autocomplete failed. Please try again.")
|
||||
yield streaming.format_done()
|
||||
|
|
@ -64,6 +64,7 @@ from app.services.connector_service import ConnectorService
|
|||
from app.services.new_streaming_service import VercelStreamingService
|
||||
from app.utils.content_utils import bootstrap_history_from_db
|
||||
from app.utils.perf import get_perf_logger, log_system_snapshot, trim_native_heap
|
||||
from app.utils.user_message_multimodal import build_human_message_content
|
||||
|
||||
_background_tasks: set[asyncio.Task] = set()
|
||||
_perf_log = get_perf_logger()
|
||||
|
|
@ -1394,6 +1395,7 @@ async def stream_new_chat(
|
|||
disabled_tools: list[str] | None = None,
|
||||
filesystem_selection: FilesystemSelection | None = None,
|
||||
request_id: str | None = None,
|
||||
user_image_data_urls: list[str] | None = None,
|
||||
) -> AsyncGenerator[str, None]:
|
||||
"""
|
||||
Stream chat responses from the new SurfSense deep agent.
|
||||
|
|
@ -1669,8 +1671,10 @@ async def stream_new_chat(
|
|||
# elif msg.role == "assistant":
|
||||
# langchain_messages.append(AIMessage(content=msg.content))
|
||||
# else:
|
||||
# Fallback: just use the current user query with attachment context
|
||||
langchain_messages.append(HumanMessage(content=final_query))
|
||||
human_content = build_human_message_content(
|
||||
final_query, list(user_image_data_urls or ())
|
||||
)
|
||||
langchain_messages.append(HumanMessage(content=human_content))
|
||||
|
||||
input_state = {
|
||||
# Lets not pass this message atm because we are using the checkpointer to manage the conversation history
|
||||
|
|
@ -1731,8 +1735,13 @@ async def stream_new_chat(
|
|||
action_verb = "Processing"
|
||||
|
||||
processing_parts = []
|
||||
query_text = user_query[:80] + ("..." if len(user_query) > 80 else "")
|
||||
processing_parts.append(query_text)
|
||||
if user_query.strip():
|
||||
query_text = user_query[:80] + ("..." if len(user_query) > 80 else "")
|
||||
processing_parts.append(query_text)
|
||||
elif user_image_data_urls:
|
||||
processing_parts.append(f"[{len(user_image_data_urls)} image(s)]")
|
||||
else:
|
||||
processing_parts.append("(message)")
|
||||
|
||||
if mentioned_surfsense_docs:
|
||||
doc_names = []
|
||||
|
|
@ -1794,8 +1803,13 @@ async def stream_new_chat(
|
|||
|
||||
_turn_accumulator.set(None)
|
||||
|
||||
title_seed = user_query.strip() or (
|
||||
f"[{len(user_image_data_urls or [])} image(s)]"
|
||||
if user_image_data_urls
|
||||
else ""
|
||||
)
|
||||
prompt = TITLE_GENERATION_PROMPT.replace(
|
||||
"{user_query}", user_query[:500]
|
||||
"{user_query}", title_seed[:500] or "(message)"
|
||||
)
|
||||
messages = [{"role": "user", "content": prompt}]
|
||||
|
||||
|
|
@ -1998,10 +2012,15 @@ async def stream_new_chat(
|
|||
# Fire background memory extraction if the agent didn't handle it.
|
||||
# Shared threads write to team memory; private threads write to user memory.
|
||||
if not stream_result.agent_called_update_memory:
|
||||
memory_seed = user_query.strip() or (
|
||||
f"[{len(user_image_data_urls or [])} image(s)]"
|
||||
if user_image_data_urls
|
||||
else "(message)"
|
||||
)
|
||||
if visibility == ChatVisibility.SEARCH_SPACE:
|
||||
task = asyncio.create_task(
|
||||
extract_and_save_team_memory(
|
||||
user_message=user_query,
|
||||
user_message=memory_seed,
|
||||
search_space_id=search_space_id,
|
||||
llm=llm,
|
||||
author_display_name=current_user_display_name,
|
||||
|
|
@ -2012,7 +2031,7 @@ async def stream_new_chat(
|
|||
elif user_id:
|
||||
task = asyncio.create_task(
|
||||
extract_and_save_memory(
|
||||
user_message=user_query,
|
||||
user_message=memory_seed,
|
||||
user_id=user_id,
|
||||
llm=llm,
|
||||
)
|
||||
|
|
|
|||
80
surfsense_backend/app/utils/user_message_multimodal.py
Normal file
80
surfsense_backend/app/utils/user_message_multimodal.py
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
"""Helpers for multimodal user turns (text + inline images) in LangChain messages."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
from typing import Any
|
||||
|
||||
|
||||
def build_human_message_content(final_query: str, image_data_urls: list[str]) -> str | list[dict[str, Any]]:
|
||||
if not image_data_urls:
|
||||
return final_query
|
||||
parts: list[dict[str, Any]] = [{"type": "text", "text": final_query}]
|
||||
for url in image_data_urls:
|
||||
parts.append({"type": "image_url", "image_url": {"url": url}})
|
||||
return parts
|
||||
|
||||
|
||||
def split_langchain_human_content(content: str | list[Any]) -> tuple[str, list[str]]:
|
||||
"""Return plain text and data URLs from a LangChain HumanMessage ``content`` value."""
|
||||
if isinstance(content, str):
|
||||
return content, []
|
||||
if not isinstance(content, list):
|
||||
return "", []
|
||||
|
||||
text_chunks: list[str] = []
|
||||
urls: list[str] = []
|
||||
for block in content:
|
||||
if not isinstance(block, dict):
|
||||
continue
|
||||
btype = block.get("type")
|
||||
if btype == "text":
|
||||
t = block.get("text")
|
||||
if isinstance(t, str) and t:
|
||||
text_chunks.append(t)
|
||||
elif btype == "image_url":
|
||||
iu = block.get("image_url")
|
||||
if isinstance(iu, dict):
|
||||
u = iu.get("url")
|
||||
if isinstance(u, str) and u.startswith("data:"):
|
||||
urls.append(u)
|
||||
elif isinstance(iu, str) and iu.startswith("data:"):
|
||||
urls.append(iu)
|
||||
return "\n".join(text_chunks), urls
|
||||
|
||||
|
||||
def decode_base64_image(data: str, *, max_bytes: int) -> bytes:
|
||||
raw = data.strip()
|
||||
if not raw:
|
||||
raise ValueError("empty image payload")
|
||||
try:
|
||||
decoded = base64.b64decode(raw, validate=True)
|
||||
except binascii.Error as e:
|
||||
raise ValueError("invalid base64 image data") from e
|
||||
if len(decoded) > max_bytes:
|
||||
raise ValueError("image exceeds maximum size")
|
||||
return decoded
|
||||
|
||||
|
||||
def to_data_url(media_type: str, raw_b64: str) -> str:
|
||||
return f"data:{media_type};base64,{raw_b64.strip()}"
|
||||
|
||||
|
||||
def split_persisted_user_content_parts(parts: list[Any]) -> tuple[str, list[str]]:
|
||||
"""Extract plain text and data URLs from persisted assistant-ui style user ``content``."""
|
||||
text_chunks: list[str] = []
|
||||
urls: list[str] = []
|
||||
for block in parts:
|
||||
if not isinstance(block, dict):
|
||||
continue
|
||||
btype = block.get("type")
|
||||
if btype == "text":
|
||||
t = block.get("text")
|
||||
if isinstance(t, str):
|
||||
text_chunks.append(t)
|
||||
elif btype == "image":
|
||||
u = block.get("image")
|
||||
if isinstance(u, str) and u.startswith("data:"):
|
||||
urls.append(u)
|
||||
return "".join(text_chunks), urls
|
||||
|
|
@ -17,6 +17,8 @@ pnpm dev
|
|||
|
||||
This starts the Next.js dev server and Electron concurrently. Hot reload works — edit the web app and changes appear immediately.
|
||||
|
||||
On **Linux**, `pnpm dev` runs Electron through `scripts/electron-dev.mjs`: it sets `ELECTRON_DISABLE_SANDBOX=1` for the sandbox issue and passes **`--ozone-platform=x11`** (XWayland) unless **`SURFSENSE_ELECTRON_WAYLAND=1`** is set, so dev tends to behave closer to X11 for shortcuts and Ozone. Packaged Linux builds are unchanged.
|
||||
|
||||
## Configuration
|
||||
|
||||
Two `.env` files control the build:
|
||||
|
|
@ -43,12 +45,13 @@ cd ../surfsense_desktop
|
|||
pnpm build
|
||||
```
|
||||
|
||||
**Step 3** — Package into a distributable:
|
||||
**Step 3** — Package into a distributable (after steps 1–2):
|
||||
|
||||
```bash
|
||||
pnpm dist:mac # macOS (.dmg + .zip)
|
||||
pnpm dist:win # Windows (.exe)
|
||||
pnpm dist:linux # Linux (.deb + .AppImage)
|
||||
pnpm pack:dir # optional: unpacked app only → release/… (run that binary yourself)
|
||||
```
|
||||
|
||||
**Step 4** — Find the output:
|
||||
|
|
|
|||
|
|
@ -49,8 +49,8 @@ mac:
|
|||
hardenedRuntime: false
|
||||
gatekeeperAssess: false
|
||||
extendInfo:
|
||||
NSAccessibilityUsageDescription: "SurfSense uses accessibility features to insert suggestions into the active application."
|
||||
NSScreenCaptureUsageDescription: "SurfSense uses screen capture to analyze your screen and provide context-aware writing suggestions."
|
||||
NSAccessibilityUsageDescription: "SurfSense uses accessibility features to bring the app to the foreground and interact with the active application when you use desktop assists."
|
||||
NSScreenCaptureUsageDescription: "SurfSense uses screen capture so you can attach a selected region to chat (Screenshot Assist) or capture the full screen from the composer."
|
||||
NSAppleEventsUsageDescription: "SurfSense uses Apple Events to interact with the active application."
|
||||
target:
|
||||
- target: dmg
|
||||
|
|
@ -81,4 +81,5 @@ linux:
|
|||
Categories: Utility;Office;
|
||||
target:
|
||||
- deb
|
||||
- rpm
|
||||
- AppImage
|
||||
|
|
|
|||
|
|
@ -4,15 +4,16 @@
|
|||
"description": "SurfSense Desktop App",
|
||||
"main": "dist/main.js",
|
||||
"scripts": {
|
||||
"dev": "pnpm build && concurrently -k \"pnpm --dir ../surfsense_web dev\" \"wait-on http://localhost:3000 && electron .\"",
|
||||
"dev": "pnpm build && concurrently -k \"pnpm --dir ../surfsense_web dev\" \"wait-on http://localhost:3000 && node scripts/electron-dev.mjs\"",
|
||||
"build": "node scripts/build-electron.mjs",
|
||||
"pack:dir": "pnpm build && electron-builder --dir --config electron-builder.yml",
|
||||
"pack:dir:linux": "pnpm build && electron-builder --dir --linux --config electron-builder.yml -c.npmRebuild=false",
|
||||
"dist": "pnpm build && electron-builder --config electron-builder.yml",
|
||||
"dist:mac": "pnpm build && electron-builder --mac --config electron-builder.yml",
|
||||
"dist:win": "pnpm build && electron-builder --win --config electron-builder.yml",
|
||||
"dist:linux": "pnpm build && electron-builder --linux --config electron-builder.yml",
|
||||
"dist:linux": "pnpm build && electron-builder --linux --config electron-builder.yml -c.npmRebuild=false",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"postinstall": "electron-rebuild"
|
||||
"postinstall": "node scripts/postinstall-rebuild.mjs"
|
||||
},
|
||||
"homepage": "https://github.com/MODSetter/SurfSense",
|
||||
"author": {
|
||||
|
|
|
|||
|
|
@ -132,6 +132,18 @@ async function buildElectron() {
|
|||
outfile: 'dist/preload.js',
|
||||
});
|
||||
|
||||
await build({
|
||||
...shared,
|
||||
entryPoints: ['src/modules/screen-capture/screen-region-preload.ts'],
|
||||
outfile: 'dist/modules/screen-capture/screen-region-preload.js',
|
||||
});
|
||||
|
||||
await build({
|
||||
...shared,
|
||||
entryPoints: ['src/modules/screen-capture/window-picker-preload.ts'],
|
||||
outfile: 'dist/modules/screen-capture/window-picker-preload.js',
|
||||
});
|
||||
|
||||
console.log('Electron build complete');
|
||||
resolveStandaloneSymlinks();
|
||||
}
|
||||
|
|
|
|||
24
surfsense_desktop/scripts/electron-dev.mjs
Normal file
24
surfsense_desktop/scripts/electron-dev.mjs
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
/**
|
||||
* Linux dev: (1) ELECTRON_DISABLE_SANDBOX before start — setuid chrome-sandbox in node_modules.
|
||||
* (2) --ozone-platform=x11 — use X11 via XWayland so global shortcuts / GPU warnings match many
|
||||
* Linux Electron setups better than native Wayland. Set SURFSENSE_ELECTRON_WAYLAND=1 to skip (2).
|
||||
* Packaged apps are not launched through this script.
|
||||
*/
|
||||
import { spawnSync } from 'child_process';
|
||||
import { dirname, join } from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const root = join(dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const cli = join(root, 'node_modules', 'electron', 'cli.js');
|
||||
|
||||
const env = { ...process.env };
|
||||
const args = [cli, '.'];
|
||||
if (process.platform === 'linux') {
|
||||
env.ELECTRON_DISABLE_SANDBOX = '1';
|
||||
if (env.SURFSENSE_ELECTRON_WAYLAND !== '1') {
|
||||
args.push('--ozone-platform=x11');
|
||||
}
|
||||
}
|
||||
|
||||
const r = spawnSync(process.execPath, args, { cwd: root, env, stdio: 'inherit' });
|
||||
process.exit(r.status === null ? 1 : r.status ?? 0);
|
||||
25
surfsense_desktop/scripts/postinstall-rebuild.mjs
Normal file
25
surfsense_desktop/scripts/postinstall-rebuild.mjs
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
/**
|
||||
* node-mac-permissions is macOS-only; electron-rebuild would still compile it on Linux/Windows
|
||||
* (missing `make`, wrong platform). We skip rebuild there.
|
||||
*/
|
||||
import { existsSync } from 'fs';
|
||||
import { spawnSync } from 'child_process';
|
||||
import { dirname, join } from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const root = join(dirname(fileURLToPath(import.meta.url)), '..');
|
||||
|
||||
if (process.platform !== 'darwin') {
|
||||
console.log('[surfsense-desktop] Skipping electron-rebuild on non-macOS (native permissions module is darwin-only).');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const bin = join(root, 'node_modules', '.bin', 'electron-rebuild');
|
||||
|
||||
if (!existsSync(bin)) {
|
||||
console.warn('[surfsense-desktop] electron-rebuild not found in node_modules/.bin, skipping.');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const result = spawnSync(bin, [], { cwd: root, stdio: 'inherit' });
|
||||
process.exit(result.status === null ? 1 : result.status);
|
||||
|
|
@ -11,12 +11,13 @@ export const IPC_CHANNELS = {
|
|||
REQUEST_ACCESSIBILITY: 'request-accessibility',
|
||||
REQUEST_SCREEN_RECORDING: 'request-screen-recording',
|
||||
RESTART_APP: 'restart-app',
|
||||
// Autocomplete
|
||||
AUTOCOMPLETE_CONTEXT: 'autocomplete-context',
|
||||
ACCEPT_SUGGESTION: 'accept-suggestion',
|
||||
DISMISS_SUGGESTION: 'dismiss-suggestion',
|
||||
SET_AUTOCOMPLETE_ENABLED: 'set-autocomplete-enabled',
|
||||
GET_AUTOCOMPLETE_ENABLED: 'get-autocomplete-enabled',
|
||||
CAPTURE_FULL_SCREEN: 'capture-full-screen',
|
||||
SCREEN_REGION_SUBMIT: 'screen-region:submit',
|
||||
SCREEN_REGION_CANCEL: 'screen-region:cancel',
|
||||
WINDOW_PICK_LIST: 'window-pick:list',
|
||||
WINDOW_PICK_SUBMIT: 'window-pick:submit',
|
||||
WINDOW_PICK_CANCEL: 'window-pick:cancel',
|
||||
CHAT_SCREEN_CAPTURE: 'chat:screen-capture',
|
||||
// Folder sync channels
|
||||
FOLDER_SYNC_SELECT_FOLDER: 'folder-sync:select-folder',
|
||||
FOLDER_SYNC_ADD_FOLDER: 'folder-sync:add-folder',
|
||||
|
|
|
|||
|
|
@ -2,10 +2,12 @@ import { app, ipcMain, shell } from 'electron';
|
|||
import { IPC_CHANNELS } from './channels';
|
||||
import {
|
||||
getPermissionsStatus,
|
||||
hasScreenRecordingPermission,
|
||||
requestAccessibility,
|
||||
requestScreenRecording,
|
||||
restartApp,
|
||||
} from '../modules/permissions';
|
||||
import { pickOpenWindowCapture } from '../modules/screen-capture';
|
||||
import {
|
||||
selectFolder,
|
||||
addWatchedFolder,
|
||||
|
|
@ -27,8 +29,7 @@ import { getShortcuts, setShortcuts, type ShortcutConfig } from '../modules/shor
|
|||
import { getAutoLaunchState, setAutoLaunch } from '../modules/auto-launch';
|
||||
import { getActiveSearchSpaceId, setActiveSearchSpaceId } from '../modules/active-search-space';
|
||||
import { reregisterQuickAsk } from '../modules/quick-ask';
|
||||
import { reregisterAutocomplete } from '../modules/autocomplete';
|
||||
import { reregisterGeneralAssist } from '../modules/tray';
|
||||
import { reregisterGeneralAssist, reregisterScreenshotAssist } from '../modules/tray';
|
||||
import {
|
||||
getDistinctId,
|
||||
getMachineId,
|
||||
|
|
@ -85,6 +86,15 @@ export function registerIpcHandlers(): void {
|
|||
restartApp();
|
||||
});
|
||||
|
||||
ipcMain.handle(IPC_CHANNELS.CAPTURE_FULL_SCREEN, async () => {
|
||||
if (!hasScreenRecordingPermission()) {
|
||||
requestScreenRecording();
|
||||
return null;
|
||||
}
|
||||
const picked = await pickOpenWindowCapture();
|
||||
return picked?.dataUrl ?? null;
|
||||
});
|
||||
|
||||
// Folder sync handlers
|
||||
ipcMain.handle(IPC_CHANNELS.FOLDER_SYNC_SELECT_FOLDER, () => selectFolder());
|
||||
|
||||
|
|
@ -192,8 +202,8 @@ export function registerIpcHandlers(): void {
|
|||
ipcMain.handle(IPC_CHANNELS.SET_SHORTCUTS, async (_event, config: Partial<ShortcutConfig>) => {
|
||||
const updated = await setShortcuts(config);
|
||||
if (config.generalAssist) await reregisterGeneralAssist();
|
||||
if (config.screenshotAssist) await reregisterScreenshotAssist();
|
||||
if (config.quickAsk) await reregisterQuickAsk();
|
||||
if (config.autocomplete) await reregisterAutocomplete();
|
||||
trackEvent('desktop_shortcut_updated', {
|
||||
keys: Object.keys(config),
|
||||
});
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ import { setupDeepLinks, handlePendingDeepLink, hasPendingDeepLink } from './mod
|
|||
import { setupAutoUpdater } from './modules/auto-updater';
|
||||
import { setupMenu } from './modules/menu';
|
||||
import { registerQuickAsk, unregisterQuickAsk } from './modules/quick-ask';
|
||||
import { registerAutocomplete, unregisterAutocomplete } from './modules/autocomplete';
|
||||
import { registerFolderWatcher, unregisterFolderWatcher } from './modules/folder-watcher';
|
||||
import { registerIpcHandlers } from './ipc/handlers';
|
||||
import { createTray, destroyTray } from './modules/tray';
|
||||
|
|
@ -60,7 +59,6 @@ app.whenReady().then(async () => {
|
|||
}
|
||||
|
||||
await registerQuickAsk();
|
||||
await registerAutocomplete();
|
||||
registerFolderWatcher();
|
||||
setupAutoUpdater();
|
||||
|
||||
|
|
@ -94,7 +92,6 @@ app.on('will-quit', async (e) => {
|
|||
didCleanup = true;
|
||||
e.preventDefault();
|
||||
unregisterQuickAsk();
|
||||
unregisterAutocomplete();
|
||||
unregisterFolderWatcher();
|
||||
destroyTray();
|
||||
await shutdownAnalytics();
|
||||
|
|
|
|||
|
|
@ -1,143 +0,0 @@
|
|||
import { clipboard, globalShortcut, ipcMain, screen } from 'electron';
|
||||
import { IPC_CHANNELS } from '../../ipc/channels';
|
||||
import { getFrontmostApp, getWindowTitle, hasAccessibilityPermission, simulatePaste } from '../platform';
|
||||
import { hasScreenRecordingPermission, requestAccessibility, requestScreenRecording } from '../permissions';
|
||||
import { captureScreen } from './screenshot';
|
||||
import { createSuggestionWindow, destroySuggestion, getSuggestionWindow } from './suggestion-window';
|
||||
import { getShortcuts } from '../shortcuts';
|
||||
import { getActiveSearchSpaceId } from '../active-search-space';
|
||||
import { trackEvent } from '../analytics';
|
||||
|
||||
let currentShortcut = '';
|
||||
let autocompleteEnabled = true;
|
||||
let savedClipboard = '';
|
||||
let sourceApp = '';
|
||||
|
||||
function isSurfSenseWindow(): boolean {
|
||||
const app = getFrontmostApp();
|
||||
return app === 'Electron' || app === 'SurfSense' || app === 'surfsense-desktop';
|
||||
}
|
||||
|
||||
async function triggerAutocomplete(): Promise<void> {
|
||||
if (!autocompleteEnabled) return;
|
||||
if (isSurfSenseWindow()) return;
|
||||
|
||||
if (!hasScreenRecordingPermission()) {
|
||||
requestScreenRecording();
|
||||
return;
|
||||
}
|
||||
|
||||
sourceApp = getFrontmostApp();
|
||||
const windowTitle = getWindowTitle();
|
||||
savedClipboard = clipboard.readText();
|
||||
|
||||
const screenshot = await captureScreen();
|
||||
if (!screenshot) {
|
||||
console.error('[autocomplete] Screenshot capture failed');
|
||||
return;
|
||||
}
|
||||
|
||||
const searchSpaceId = await getActiveSearchSpaceId();
|
||||
if (!searchSpaceId) {
|
||||
console.warn('[autocomplete] No active search space. Select a search space first.');
|
||||
return;
|
||||
}
|
||||
trackEvent('desktop_autocomplete_triggered', { search_space_id: searchSpaceId });
|
||||
const cursor = screen.getCursorScreenPoint();
|
||||
const win = createSuggestionWindow(cursor.x, cursor.y);
|
||||
|
||||
win.webContents.once('did-finish-load', () => {
|
||||
const sw = getSuggestionWindow();
|
||||
setTimeout(() => {
|
||||
if (sw && !sw.isDestroyed()) {
|
||||
sw.webContents.send(IPC_CHANNELS.AUTOCOMPLETE_CONTEXT, {
|
||||
screenshot,
|
||||
searchSpaceId,
|
||||
appName: sourceApp,
|
||||
windowTitle,
|
||||
});
|
||||
}
|
||||
}, 300);
|
||||
});
|
||||
}
|
||||
|
||||
async function acceptAndInject(text: string): Promise<void> {
|
||||
if (!sourceApp) return;
|
||||
|
||||
if (!hasAccessibilityPermission()) {
|
||||
requestAccessibility();
|
||||
return;
|
||||
}
|
||||
|
||||
clipboard.writeText(text);
|
||||
destroySuggestion();
|
||||
|
||||
try {
|
||||
await new Promise((r) => setTimeout(r, 50));
|
||||
simulatePaste();
|
||||
await new Promise((r) => setTimeout(r, 100));
|
||||
clipboard.writeText(savedClipboard);
|
||||
} catch {
|
||||
clipboard.writeText(savedClipboard);
|
||||
}
|
||||
}
|
||||
|
||||
let ipcRegistered = false;
|
||||
|
||||
function registerIpcHandlers(): void {
|
||||
if (ipcRegistered) return;
|
||||
ipcRegistered = true;
|
||||
|
||||
ipcMain.handle(IPC_CHANNELS.ACCEPT_SUGGESTION, async (_event, text: string) => {
|
||||
trackEvent('desktop_autocomplete_accepted');
|
||||
await acceptAndInject(text);
|
||||
});
|
||||
ipcMain.handle(IPC_CHANNELS.DISMISS_SUGGESTION, () => {
|
||||
trackEvent('desktop_autocomplete_dismissed');
|
||||
destroySuggestion();
|
||||
});
|
||||
ipcMain.handle(IPC_CHANNELS.SET_AUTOCOMPLETE_ENABLED, (_event, enabled: boolean) => {
|
||||
autocompleteEnabled = enabled;
|
||||
if (!enabled) {
|
||||
destroySuggestion();
|
||||
}
|
||||
});
|
||||
ipcMain.handle(IPC_CHANNELS.GET_AUTOCOMPLETE_ENABLED, () => autocompleteEnabled);
|
||||
}
|
||||
|
||||
function autocompleteHandler(): void {
|
||||
const sw = getSuggestionWindow();
|
||||
if (sw && !sw.isDestroyed()) {
|
||||
destroySuggestion();
|
||||
return;
|
||||
}
|
||||
triggerAutocomplete();
|
||||
}
|
||||
|
||||
async function registerShortcut(): Promise<void> {
|
||||
const shortcuts = await getShortcuts();
|
||||
currentShortcut = shortcuts.autocomplete;
|
||||
|
||||
const ok = globalShortcut.register(currentShortcut, autocompleteHandler);
|
||||
|
||||
if (!ok) {
|
||||
console.error(`[autocomplete] Failed to register shortcut ${currentShortcut}`);
|
||||
} else {
|
||||
console.log(`[autocomplete] Registered shortcut ${currentShortcut}`);
|
||||
}
|
||||
}
|
||||
|
||||
export async function registerAutocomplete(): Promise<void> {
|
||||
registerIpcHandlers();
|
||||
await registerShortcut();
|
||||
}
|
||||
|
||||
export function unregisterAutocomplete(): void {
|
||||
if (currentShortcut) globalShortcut.unregister(currentShortcut);
|
||||
destroySuggestion();
|
||||
}
|
||||
|
||||
export async function reregisterAutocomplete(): Promise<void> {
|
||||
unregisterAutocomplete();
|
||||
await registerShortcut();
|
||||
}
|
||||
|
|
@ -1,27 +0,0 @@
|
|||
import { desktopCapturer, screen } from 'electron';
|
||||
|
||||
/**
|
||||
* Captures the primary display as a base64-encoded PNG data URL.
|
||||
* Uses the display's actual size for full-resolution capture.
|
||||
*/
|
||||
export async function captureScreen(): Promise<string | null> {
|
||||
try {
|
||||
const primaryDisplay = screen.getPrimaryDisplay();
|
||||
const { width, height } = primaryDisplay.size;
|
||||
|
||||
const sources = await desktopCapturer.getSources({
|
||||
types: ['screen'],
|
||||
thumbnailSize: { width, height },
|
||||
});
|
||||
|
||||
if (!sources.length) {
|
||||
console.error('[screenshot] No screen sources found');
|
||||
return null;
|
||||
}
|
||||
|
||||
return sources[0].thumbnail.toDataURL();
|
||||
} catch (err) {
|
||||
console.error('[screenshot] Failed to capture screen:', err);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,112 +0,0 @@
|
|||
import { BrowserWindow, screen, shell } from 'electron';
|
||||
import path from 'path';
|
||||
import { getServerPort } from '../server';
|
||||
|
||||
const TOOLTIP_WIDTH = 420;
|
||||
const TOOLTIP_HEIGHT = 38;
|
||||
const MAX_HEIGHT = 400;
|
||||
|
||||
let suggestionWindow: BrowserWindow | null = null;
|
||||
let resizeTimer: ReturnType<typeof setInterval> | null = null;
|
||||
let cursorOrigin = { x: 0, y: 0 };
|
||||
|
||||
const CURSOR_GAP = 20;
|
||||
|
||||
function positionOnScreen(cursorX: number, cursorY: number, w: number, h: number): { x: number; y: number } {
|
||||
const display = screen.getDisplayNearestPoint({ x: cursorX, y: cursorY });
|
||||
const { x: dx, y: dy, width: dw, height: dh } = display.workArea;
|
||||
|
||||
const x = Math.max(dx, Math.min(cursorX, dx + dw - w));
|
||||
|
||||
const spaceBelow = (dy + dh) - (cursorY + CURSOR_GAP);
|
||||
const y = spaceBelow >= h
|
||||
? cursorY + CURSOR_GAP
|
||||
: cursorY - h - CURSOR_GAP;
|
||||
|
||||
return { x, y: Math.max(dy, y) };
|
||||
}
|
||||
|
||||
function stopResizePolling(): void {
|
||||
if (resizeTimer) { clearInterval(resizeTimer); resizeTimer = null; }
|
||||
}
|
||||
|
||||
function startResizePolling(win: BrowserWindow): void {
|
||||
stopResizePolling();
|
||||
let lastH = 0;
|
||||
resizeTimer = setInterval(async () => {
|
||||
if (!win || win.isDestroyed()) { stopResizePolling(); return; }
|
||||
try {
|
||||
const h: number = await win.webContents.executeJavaScript(
|
||||
`document.body.scrollHeight`
|
||||
);
|
||||
if (h > 0 && h !== lastH) {
|
||||
lastH = h;
|
||||
const clamped = Math.min(h, MAX_HEIGHT);
|
||||
const pos = positionOnScreen(cursorOrigin.x, cursorOrigin.y, TOOLTIP_WIDTH, clamped);
|
||||
win.setBounds({ x: pos.x, y: pos.y, width: TOOLTIP_WIDTH, height: clamped });
|
||||
}
|
||||
} catch {}
|
||||
}, 150);
|
||||
}
|
||||
|
||||
export function getSuggestionWindow(): BrowserWindow | null {
|
||||
return suggestionWindow;
|
||||
}
|
||||
|
||||
export function destroySuggestion(): void {
|
||||
stopResizePolling();
|
||||
if (suggestionWindow && !suggestionWindow.isDestroyed()) {
|
||||
suggestionWindow.close();
|
||||
}
|
||||
suggestionWindow = null;
|
||||
}
|
||||
|
||||
export function createSuggestionWindow(x: number, y: number): BrowserWindow {
|
||||
destroySuggestion();
|
||||
cursorOrigin = { x, y };
|
||||
|
||||
const pos = positionOnScreen(x, y, TOOLTIP_WIDTH, TOOLTIP_HEIGHT);
|
||||
|
||||
suggestionWindow = new BrowserWindow({
|
||||
width: TOOLTIP_WIDTH,
|
||||
height: TOOLTIP_HEIGHT,
|
||||
x: pos.x,
|
||||
y: pos.y,
|
||||
frame: false,
|
||||
transparent: true,
|
||||
focusable: false,
|
||||
alwaysOnTop: true,
|
||||
skipTaskbar: true,
|
||||
hasShadow: true,
|
||||
type: 'panel',
|
||||
webPreferences: {
|
||||
preload: path.join(__dirname, 'preload.js'),
|
||||
contextIsolation: true,
|
||||
nodeIntegration: false,
|
||||
sandbox: true,
|
||||
},
|
||||
show: false,
|
||||
});
|
||||
|
||||
suggestionWindow.loadURL(`http://localhost:${getServerPort()}/desktop/suggestion?t=${Date.now()}`);
|
||||
|
||||
suggestionWindow.once('ready-to-show', () => {
|
||||
suggestionWindow?.showInactive();
|
||||
if (suggestionWindow) startResizePolling(suggestionWindow);
|
||||
});
|
||||
|
||||
suggestionWindow.webContents.setWindowOpenHandler(({ url }) => {
|
||||
if (url.startsWith('http://localhost')) {
|
||||
return { action: 'allow' };
|
||||
}
|
||||
shell.openExternal(url);
|
||||
return { action: 'deny' };
|
||||
});
|
||||
|
||||
suggestionWindow.on('closed', () => {
|
||||
stopResizePolling();
|
||||
suggestionWindow = null;
|
||||
});
|
||||
|
||||
return suggestionWindow;
|
||||
}
|
||||
5
surfsense_desktop/src/modules/general-assist.ts
Normal file
5
surfsense_desktop/src/modules/general-assist.ts
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
import { showMainWindow } from './window';
|
||||
|
||||
export function runGeneralAssistShortcut(): void {
|
||||
showMainWindow('shortcut');
|
||||
}
|
||||
7
surfsense_desktop/src/modules/screen-capture/index.ts
Normal file
7
surfsense_desktop/src/modules/screen-capture/index.ts
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
/**
|
||||
* Window capture for Screenshot Assist and chat fullscreen: single-session
|
||||
* desktopCapturer, region overlay, and shortcut entry point.
|
||||
*/
|
||||
export { pickOpenWindowCapture, type PickedWindowResult } from './window-picker';
|
||||
export { pickScreenRegion, captureCurrentDisplayDataUrl } from './screen-region-picker';
|
||||
export { runScreenshotAssistShortcut } from './screenshot-assist';
|
||||
|
|
@ -0,0 +1,335 @@
|
|||
import { BrowserWindow, desktopCapturer, nativeImage, screen } from 'electron';
|
||||
import path from 'path';
|
||||
import { IPC_CHANNELS } from '../../ipc/channels';
|
||||
function fitNativeImageToWorkArea(img: Electron.NativeImage, display: Electron.Display): Electron.NativeImage {
|
||||
const wa = display.workArea;
|
||||
const { width: iw, height: ih } = img.getSize();
|
||||
const scale = Math.min(1, wa.width / iw, wa.height / ih);
|
||||
if (scale >= 1) return img;
|
||||
return img.resize({
|
||||
width: Math.max(1, Math.floor(iw * scale)),
|
||||
height: Math.max(1, Math.floor(ih * scale)),
|
||||
quality: 'best',
|
||||
});
|
||||
}
|
||||
|
||||
// One getSources per pick; overlay and final crop share that bitmap (avoids a second portal session, e.g. Wayland).
|
||||
|
||||
let pickInProgress = false;
|
||||
|
||||
type DisplayCaptureSnapshot = {
|
||||
dataUrl: string;
|
||||
width: number;
|
||||
height: number;
|
||||
};
|
||||
|
||||
async function captureDisplaySnapshot(display: Electron.Display): Promise<DisplayCaptureSnapshot | null> {
|
||||
try {
|
||||
const sf = display.scaleFactor || 1;
|
||||
const tw = Math.max(1, Math.round(display.size.width * sf));
|
||||
const th = Math.max(1, Math.round(display.size.height * sf));
|
||||
const sources = await desktopCapturer.getSources({
|
||||
types: ['screen'],
|
||||
thumbnailSize: { width: tw, height: th },
|
||||
});
|
||||
if (!sources.length) return null;
|
||||
const idStr = String(display.id);
|
||||
let chosen =
|
||||
sources.find((s) => s.display_id === idStr) ||
|
||||
sources.find((s) => s.display_id && s.display_id === idStr) ||
|
||||
null;
|
||||
if (!chosen && screen.getPrimaryDisplay().id === display.id) {
|
||||
chosen = sources[0];
|
||||
}
|
||||
if (!chosen) chosen = sources[0];
|
||||
const dataUrl = chosen.thumbnail.toDataURL();
|
||||
const { width, height } = chosen.thumbnail.getSize();
|
||||
return { dataUrl, width, height };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function captureCurrentDisplayDataUrl(): Promise<string | null> {
|
||||
const display = screen.getDisplayNearestPoint(screen.getCursorScreenPoint());
|
||||
const snapshot = await captureDisplaySnapshot(display);
|
||||
return snapshot?.dataUrl ?? null;
|
||||
}
|
||||
|
||||
function buildInjectScript(dataUrl: string, iw: number, ih: number): string {
|
||||
return `(() => {
|
||||
const api = window.surfsenseScreenRegion;
|
||||
if (!api) return;
|
||||
const dataUrl = ${JSON.stringify(dataUrl)};
|
||||
const iw = ${iw};
|
||||
const ih = ${ih};
|
||||
document.body.style.margin = '0';
|
||||
document.body.style.overflow = 'hidden';
|
||||
document.body.style.background = '#000';
|
||||
const img = document.createElement('img');
|
||||
img.draggable = false;
|
||||
img.src = dataUrl;
|
||||
img.style.cssText = 'position:fixed;inset:0;width:100vw;height:100vh;object-fit:fill;user-select:none;pointer-events:none;';
|
||||
const veil = document.createElement('div');
|
||||
veil.style.cssText = 'position:fixed;inset:0;cursor:crosshair;background:rgba(0,0,0,0.15);';
|
||||
const sel = document.createElement('div');
|
||||
sel.style.cssText = 'position:fixed;border:2px solid #38bdf8;box-shadow:0 0 0 9999px rgba(0,0,0,0.45);display:none;pointer-events:none;z-index:2;';
|
||||
document.body.appendChild(img);
|
||||
document.body.appendChild(veil);
|
||||
document.body.appendChild(sel);
|
||||
let ax = 0, ay = 0, dragging = false;
|
||||
function show(x0, y0, x1, y1) {
|
||||
const l = Math.min(x0, x1), t = Math.min(y0, y1);
|
||||
const w = Math.abs(x1 - x0), h = Math.abs(y1 - y0);
|
||||
if (w < 2 || h < 2) { sel.style.display = 'none'; return; }
|
||||
sel.style.display = 'block';
|
||||
sel.style.left = l + 'px';
|
||||
sel.style.top = t + 'px';
|
||||
sel.style.width = w + 'px';
|
||||
sel.style.height = h + 'px';
|
||||
}
|
||||
function mapRect(l, t, w, h) {
|
||||
const vw = window.innerWidth, vh = window.innerHeight;
|
||||
const sx = Math.round((l / vw) * iw);
|
||||
const sy = Math.round((t / vh) * ih);
|
||||
const sw = Math.max(1, Math.round((w / vw) * iw));
|
||||
const sh = Math.max(1, Math.round((h / vh) * ih));
|
||||
const cx = Math.min(Math.max(0, sx), iw - 1);
|
||||
const cy = Math.min(Math.max(0, sy), ih - 1);
|
||||
const cw = Math.min(sw, iw - cx);
|
||||
const ch = Math.min(sh, ih - cy);
|
||||
return { x: cx, y: cy, width: cw, height: ch };
|
||||
}
|
||||
function endDrag(clientX, clientY, pointerId) {
|
||||
if (!dragging) return;
|
||||
dragging = false;
|
||||
if (typeof pointerId === 'number' && pointerId >= 0) {
|
||||
try { veil.releasePointerCapture(pointerId); } catch (_) {}
|
||||
}
|
||||
const l = Math.min(ax, clientX), t = Math.min(ay, clientY);
|
||||
const w = Math.abs(clientX - ax), h = Math.abs(clientY - ay);
|
||||
if (w < 4 || h < 4) { sel.style.display = 'none'; return; }
|
||||
api.submit(mapRect(l, t, w, h));
|
||||
}
|
||||
veil.addEventListener('pointerdown', (e) => {
|
||||
if (e.button !== 0) return;
|
||||
try { veil.setPointerCapture(e.pointerId); } catch (_) {}
|
||||
dragging = true;
|
||||
ax = e.clientX; ay = e.clientY;
|
||||
show(ax, ay, ax, ay);
|
||||
});
|
||||
veil.addEventListener('pointermove', (e) => {
|
||||
if (!dragging) return;
|
||||
show(ax, ay, e.clientX, e.clientY);
|
||||
});
|
||||
veil.addEventListener('pointerup', (e) => {
|
||||
endDrag(e.clientX, e.clientY, e.pointerId);
|
||||
});
|
||||
window.addEventListener('pointerup', (e) => {
|
||||
endDrag(e.clientX, e.clientY, e.pointerId);
|
||||
});
|
||||
document.addEventListener(
|
||||
'mouseup',
|
||||
(e) => {
|
||||
endDrag(e.clientX, e.clientY, -1);
|
||||
},
|
||||
true
|
||||
);
|
||||
veil.addEventListener('pointercancel', (e) => {
|
||||
if (!dragging) return;
|
||||
dragging = false;
|
||||
try { veil.releasePointerCapture(e.pointerId); } catch (_) {}
|
||||
sel.style.display = 'none';
|
||||
});
|
||||
window.addEventListener('keydown', (e) => {
|
||||
if (e.key === 'Escape') { api.cancel(); return; }
|
||||
if (e.key === 'Enter' && sel.style.display === 'block') {
|
||||
const l = parseFloat(sel.style.left), t = parseFloat(sel.style.top);
|
||||
const w = parseFloat(sel.style.width), h = parseFloat(sel.style.height);
|
||||
if (w >= 4 && h >= 4) api.submit(mapRect(l, t, w, h));
|
||||
}
|
||||
});
|
||||
})();`;
|
||||
}
|
||||
|
||||
export function pickScreenRegion(opts?: { windowDataUrl?: string }): Promise<string | null> {
|
||||
if (pickInProgress) return Promise.resolve(null);
|
||||
pickInProgress = true;
|
||||
|
||||
return new Promise((resolve) => {
|
||||
const display = screen.getDisplayNearestPoint(screen.getCursorScreenPoint());
|
||||
let settled = false;
|
||||
let overlay: BrowserWindow | null = null;
|
||||
/** webContents for listener removal after `BrowserWindow` may already be destroyed. */
|
||||
let overlayWc: Electron.WebContents | null = null;
|
||||
|
||||
const cleanupListeners = () => {
|
||||
const wc = overlayWc;
|
||||
overlayWc = null;
|
||||
if (!wc || wc.isDestroyed()) return;
|
||||
wc.removeListener('before-input-event', onBeforeInput);
|
||||
wc.ipc.removeListener(IPC_CHANNELS.SCREEN_REGION_SUBMIT, onSubmit);
|
||||
wc.ipc.removeListener(IPC_CHANNELS.SCREEN_REGION_CANCEL, onCancel);
|
||||
};
|
||||
|
||||
const finish = (result: string | null) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
pickInProgress = false;
|
||||
cleanupListeners();
|
||||
if (overlay && !overlay.isDestroyed()) {
|
||||
overlay.removeAllListeners('closed');
|
||||
overlay.close();
|
||||
}
|
||||
overlay = null;
|
||||
resolve(result);
|
||||
};
|
||||
|
||||
let snapshot: DisplayCaptureSnapshot | null = null;
|
||||
let cropSource: Electron.NativeImage | null = null;
|
||||
|
||||
const onSubmit = (
|
||||
_event: Electron.IpcMainEvent,
|
||||
rect: { x: number; y: number; width: number; height: number }
|
||||
) => {
|
||||
if (settled || !overlay || overlay.isDestroyed()) return;
|
||||
if (!rect || rect.width < 1 || rect.height < 1) {
|
||||
finish(null);
|
||||
return;
|
||||
}
|
||||
if (!snapshot || !cropSource) {
|
||||
finish(null);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const iw = snapshot.width;
|
||||
const ih = snapshot.height;
|
||||
const { width: cw, height: ch } = cropSource.getSize();
|
||||
const scaleX = cw / iw;
|
||||
const scaleY = ch / ih;
|
||||
const ox = Math.floor(rect.x * scaleX);
|
||||
const oy = Math.floor(rect.y * scaleY);
|
||||
const ow = Math.min(Math.floor(rect.width * scaleX), cw - ox);
|
||||
const oh = Math.min(Math.floor(rect.height * scaleY), ch - oy);
|
||||
const cropped = cropSource.crop({
|
||||
x: ox,
|
||||
y: oy,
|
||||
width: Math.max(1, ow),
|
||||
height: Math.max(1, oh),
|
||||
});
|
||||
finish(cropped.toDataURL());
|
||||
} catch {
|
||||
finish(null);
|
||||
}
|
||||
};
|
||||
|
||||
const onCancel = (_event: Electron.IpcMainEvent) => {
|
||||
if (settled || !overlay || overlay.isDestroyed()) return;
|
||||
finish(null);
|
||||
};
|
||||
|
||||
const onBeforeInput = (_event: Electron.Event, input: Electron.Input) => {
|
||||
if (input.type === 'keyDown' && input.key === 'Escape') {
|
||||
finish(null);
|
||||
}
|
||||
};
|
||||
|
||||
const openOverlay = (
|
||||
cap: DisplayCaptureSnapshot,
|
||||
crop: Electron.NativeImage,
|
||||
bounds: { x: number; y: number; width: number; height: number }
|
||||
) => {
|
||||
snapshot = cap;
|
||||
cropSource = crop;
|
||||
|
||||
overlay = new BrowserWindow({
|
||||
x: bounds.x,
|
||||
y: bounds.y,
|
||||
width: bounds.width,
|
||||
height: bounds.height,
|
||||
frame: false,
|
||||
transparent: true,
|
||||
fullscreenable: false,
|
||||
skipTaskbar: true,
|
||||
alwaysOnTop: true,
|
||||
focusable: true,
|
||||
show: false,
|
||||
autoHideMenuBar: true,
|
||||
backgroundColor: '#00000000',
|
||||
webPreferences: {
|
||||
preload: path.join(__dirname, 'modules', 'screen-capture', 'screen-region-preload.js'),
|
||||
contextIsolation: true,
|
||||
nodeIntegration: false,
|
||||
sandbox: true,
|
||||
},
|
||||
});
|
||||
|
||||
overlayWc = overlay.webContents;
|
||||
overlayWc.on('before-input-event', onBeforeInput);
|
||||
overlayWc.ipc.on(IPC_CHANNELS.SCREEN_REGION_SUBMIT, onSubmit);
|
||||
overlayWc.ipc.on(IPC_CHANNELS.SCREEN_REGION_CANCEL, onCancel);
|
||||
|
||||
overlay.setIgnoreMouseEvents(false);
|
||||
overlay.loadURL(
|
||||
'data:text/html;charset=utf-8,' +
|
||||
encodeURIComponent('<!doctype html><html><head><meta charset="utf-8"/></head><body></body></html>')
|
||||
);
|
||||
|
||||
overlay.on('closed', () => {
|
||||
if (!settled) finish(null);
|
||||
});
|
||||
|
||||
overlay.webContents.once('did-finish-load', () => {
|
||||
if (!overlay || overlay.isDestroyed()) return;
|
||||
overlay.webContents
|
||||
.executeJavaScript(buildInjectScript(cap.dataUrl, cap.width, cap.height), true)
|
||||
.then(() => {
|
||||
overlay?.show();
|
||||
overlay?.focus();
|
||||
})
|
||||
.catch(() => {
|
||||
finish(null);
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
void (async () => {
|
||||
try {
|
||||
if (opts?.windowDataUrl) {
|
||||
const fullRes = nativeImage.createFromDataURL(opts.windowDataUrl);
|
||||
if (fullRes.isEmpty()) {
|
||||
finish(null);
|
||||
return;
|
||||
}
|
||||
const fitted = fitNativeImageToWorkArea(fullRes, display);
|
||||
const fw = fitted.getSize().width;
|
||||
const fh = fitted.getSize().height;
|
||||
const wa = display.workArea;
|
||||
const x = wa.x + Math.floor((wa.width - fw) / 2);
|
||||
const y = wa.y + Math.floor((wa.height - fh) / 2);
|
||||
openOverlay(
|
||||
{ dataUrl: fitted.toDataURL(), width: fw, height: fh },
|
||||
fullRes,
|
||||
{ x, y, width: fw, height: fh }
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const cap = await captureDisplaySnapshot(display);
|
||||
if (!cap) {
|
||||
finish(null);
|
||||
return;
|
||||
}
|
||||
const crop = nativeImage.createFromDataURL(cap.dataUrl);
|
||||
openOverlay(cap, crop, {
|
||||
x: display.bounds.x,
|
||||
y: display.bounds.y,
|
||||
width: display.bounds.width,
|
||||
height: display.bounds.height,
|
||||
});
|
||||
} catch {
|
||||
finish(null);
|
||||
}
|
||||
})();
|
||||
});
|
||||
}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
import { contextBridge, ipcRenderer } from 'electron';
|
||||
import { IPC_CHANNELS } from '../../ipc/channels';
|
||||
|
||||
contextBridge.exposeInMainWorld('surfsenseScreenRegion', {
|
||||
submit: (rect: { x: number; y: number; width: number; height: number }) => {
|
||||
ipcRenderer.send(IPC_CHANNELS.SCREEN_REGION_SUBMIT, rect);
|
||||
},
|
||||
cancel: () => {
|
||||
ipcRenderer.send(IPC_CHANNELS.SCREEN_REGION_CANCEL);
|
||||
},
|
||||
});
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
import { IPC_CHANNELS } from '../../ipc/channels';
|
||||
import { trackEvent } from '../analytics';
|
||||
import { pickScreenRegion } from './screen-region-picker';
|
||||
import { pickOpenWindowCapture } from './window-picker';
|
||||
import { getMainWindow, showMainWindow } from '../window';
|
||||
import { hasScreenRecordingPermission, requestScreenRecording } from '../permissions';
|
||||
|
||||
export async function runScreenshotAssistShortcut(): Promise<void> {
|
||||
if (!hasScreenRecordingPermission()) {
|
||||
requestScreenRecording();
|
||||
return;
|
||||
}
|
||||
|
||||
const picked = await pickOpenWindowCapture();
|
||||
if (!picked) return;
|
||||
|
||||
const url = await pickScreenRegion({ windowDataUrl: picked.dataUrl });
|
||||
if (!url) return;
|
||||
|
||||
showMainWindow('shortcut');
|
||||
const mw = getMainWindow();
|
||||
if (mw && !mw.isDestroyed()) {
|
||||
mw.webContents.send(IPC_CHANNELS.CHAT_SCREEN_CAPTURE, url);
|
||||
trackEvent('desktop_screenshot_assist_region_to_chat', {});
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
import { contextBridge, ipcRenderer } from 'electron';
|
||||
import { IPC_CHANNELS } from '../../ipc/channels';
|
||||
|
||||
contextBridge.exposeInMainWorld('surfsenseWindowPick', {
|
||||
list: () =>
|
||||
ipcRenderer.invoke(IPC_CHANNELS.WINDOW_PICK_LIST) as Promise<
|
||||
{ id: string; name: string; thumbDataUrl: string }[]
|
||||
>,
|
||||
submit: (sourceId: string) => {
|
||||
ipcRenderer.send(IPC_CHANNELS.WINDOW_PICK_SUBMIT, sourceId);
|
||||
},
|
||||
cancel: () => {
|
||||
ipcRenderer.send(IPC_CHANNELS.WINDOW_PICK_CANCEL);
|
||||
},
|
||||
});
|
||||
244
surfsense_desktop/src/modules/screen-capture/window-picker.ts
Normal file
244
surfsense_desktop/src/modules/screen-capture/window-picker.ts
Normal file
|
|
@ -0,0 +1,244 @@
|
|||
import { BrowserWindow, desktopCapturer, ipcMain, screen } from 'electron';
|
||||
import path from 'path';
|
||||
import { IPC_CHANNELS } from '../../ipc/channels';
|
||||
|
||||
let pickInProgress = false;
|
||||
|
||||
const PREVIEW_THUMB = { width: 280, height: 180 } as const;
|
||||
|
||||
function maxCaptureThumbSize(): { width: number; height: number } {
|
||||
const d = screen.getPrimaryDisplay();
|
||||
const sf = d.scaleFactor || 1;
|
||||
const w = Math.min(3840, Math.max(1280, Math.round(d.size.width * sf)));
|
||||
const h = Math.min(2160, Math.max(720, Math.round(d.size.height * sf)));
|
||||
return { width: w, height: h };
|
||||
}
|
||||
|
||||
function isDesktopWindowSourceId(s: string): boolean {
|
||||
return typeof s === 'string' && s.startsWith('window:');
|
||||
}
|
||||
|
||||
export type PickedWindowResult = {
|
||||
sourceId: string;
|
||||
/** Same pixels as the one `desktopCapturer` snapshot (max thumbnail size). */
|
||||
dataUrl: string;
|
||||
};
|
||||
|
||||
function buildPickerInjectScript(): string {
|
||||
return `(async function () {
|
||||
const api = window.surfsenseWindowPick;
|
||||
if (!api) return;
|
||||
const items = await api.list();
|
||||
document.body.style.cssText =
|
||||
'margin:0;font-family:system-ui,-apple-system,sans-serif;background:#0f172a;color:#e2e8f0;min-height:100vh;padding:16px;box-sizing:border-box;';
|
||||
const top = document.createElement('div');
|
||||
top.style.cssText =
|
||||
'display:flex;justify-content:space-between;align-items:center;margin-bottom:12px;flex-wrap:wrap;gap:8px;';
|
||||
const t = document.createElement('strong');
|
||||
t.textContent = 'Open windows';
|
||||
const hint = document.createElement('span');
|
||||
hint.style.cssText = 'opacity:0.75;font-size:13px;';
|
||||
hint.textContent = 'Click a window · Esc to cancel';
|
||||
top.appendChild(t);
|
||||
top.appendChild(hint);
|
||||
document.body.appendChild(top);
|
||||
if (!items || !items.length) {
|
||||
const p = document.createElement('p');
|
||||
p.style.cssText = 'line-height:1.5;max-width:42rem;';
|
||||
p.textContent =
|
||||
'No windows were returned by the system. On Linux, allow screen capture when prompted. If other apps are open, try again.';
|
||||
document.body.appendChild(p);
|
||||
return;
|
||||
}
|
||||
const grid = document.createElement('div');
|
||||
grid.style.cssText =
|
||||
'display:grid;grid-template-columns:repeat(auto-fill,minmax(200px,1fr));gap:12px;max-height:calc(100vh - 88px);overflow:auto;padding-bottom:8px;';
|
||||
for (const it of items) {
|
||||
const card = document.createElement('button');
|
||||
card.type = 'button';
|
||||
card.style.cssText =
|
||||
'text-align:left;background:#1e293b;border:1px solid #334155;border-radius:8px;padding:8px;cursor:pointer;color:inherit;';
|
||||
card.addEventListener('mouseenter', function () {
|
||||
card.style.borderColor = '#38bdf8';
|
||||
});
|
||||
card.addEventListener('mouseleave', function () {
|
||||
card.style.borderColor = '#334155';
|
||||
});
|
||||
const img = document.createElement('img');
|
||||
img.alt = '';
|
||||
img.src =
|
||||
it.thumbDataUrl ||
|
||||
'data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///ywAAAAAAQABAAACAUwAOw==';
|
||||
img.style.cssText =
|
||||
'width:100%;height:100px;object-fit:cover;border-radius:4px;background:#000;display:block;';
|
||||
const cap = document.createElement('div');
|
||||
cap.textContent = it.name || '(untitled)';
|
||||
cap.style.cssText =
|
||||
'margin-top:6px;font-size:12px;line-height:1.35;overflow:hidden;text-overflow:ellipsis;display:-webkit-box;-webkit-line-clamp:2;-webkit-box-orient:vertical;';
|
||||
card.appendChild(img);
|
||||
card.appendChild(cap);
|
||||
card.addEventListener('click', function () {
|
||||
api.submit(it.id);
|
||||
});
|
||||
grid.appendChild(card);
|
||||
}
|
||||
document.body.appendChild(grid);
|
||||
window.addEventListener('keydown', function (e) {
|
||||
if (e.key === 'Escape') api.cancel();
|
||||
});
|
||||
})();`;
|
||||
}
|
||||
|
||||
/**
|
||||
* One OS / Chromium capture session: `getSources` runs once (important on Wayland /
|
||||
* PipeWire so the portal is not opened again for the same flow). Opens our grid to
|
||||
* choose a window; resolves with the chosen snapshot for region or full-frame use.
|
||||
*/
|
||||
export function pickOpenWindowCapture(): Promise<PickedWindowResult | null> {
|
||||
if (pickInProgress) return Promise.resolve(null);
|
||||
pickInProgress = true;
|
||||
|
||||
return new Promise((resolve) => {
|
||||
let settled = false;
|
||||
let picker: BrowserWindow | null = null;
|
||||
let pickerWc: Electron.WebContents | null = null;
|
||||
/** Filled once before the grid runs — reused for list + final image (no second getSources). */
|
||||
let sessionSources: Electron.DesktopCapturerSource[] = [];
|
||||
|
||||
const finish = (result: PickedWindowResult | null) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
pickInProgress = false;
|
||||
ipcMain.removeHandler(IPC_CHANNELS.WINDOW_PICK_LIST);
|
||||
const wc = pickerWc;
|
||||
pickerWc = null;
|
||||
if (wc && !wc.isDestroyed()) {
|
||||
wc.removeListener('before-input-event', onBeforeInput);
|
||||
wc.ipc.removeListener(IPC_CHANNELS.WINDOW_PICK_SUBMIT, onSubmit);
|
||||
wc.ipc.removeListener(IPC_CHANNELS.WINDOW_PICK_CANCEL, onCancel);
|
||||
}
|
||||
if (picker && !picker.isDestroyed()) {
|
||||
picker.removeAllListeners('closed');
|
||||
picker.close();
|
||||
}
|
||||
picker = null;
|
||||
resolve(result);
|
||||
};
|
||||
|
||||
const onSubmit = (_event: Electron.IpcMainEvent, sourceId: string) => {
|
||||
if (settled || !picker || picker.isDestroyed()) return;
|
||||
if (!isDesktopWindowSourceId(sourceId)) {
|
||||
finish(null);
|
||||
return;
|
||||
}
|
||||
const hit = sessionSources.find((s) => s.id === sourceId);
|
||||
if (!hit || hit.thumbnail.isEmpty()) {
|
||||
finish(null);
|
||||
return;
|
||||
}
|
||||
finish({ sourceId, dataUrl: hit.thumbnail.toDataURL() });
|
||||
};
|
||||
|
||||
const onCancel = () => {
|
||||
if (settled || !picker || picker.isDestroyed()) return;
|
||||
finish(null);
|
||||
};
|
||||
|
||||
const onBeforeInput = (_event: Electron.Event, input: Electron.Input) => {
|
||||
if (input.type === 'keyDown' && input.key === 'Escape') {
|
||||
finish(null);
|
||||
}
|
||||
};
|
||||
|
||||
ipcMain.handle(IPC_CHANNELS.WINDOW_PICK_LIST, async () => {
|
||||
return sessionSources.map((s, i) => {
|
||||
let thumbDataUrl = '';
|
||||
if (!s.thumbnail.isEmpty()) {
|
||||
try {
|
||||
const sm = s.thumbnail.resize({
|
||||
width: PREVIEW_THUMB.width,
|
||||
height: PREVIEW_THUMB.height,
|
||||
quality: 'good',
|
||||
});
|
||||
thumbDataUrl = sm.toDataURL();
|
||||
} catch {
|
||||
thumbDataUrl = s.thumbnail.toDataURL();
|
||||
}
|
||||
}
|
||||
return {
|
||||
id: s.id,
|
||||
name: (s.name || '').trim() || `Window ${i + 1}`,
|
||||
thumbDataUrl,
|
||||
};
|
||||
});
|
||||
});
|
||||
|
||||
picker = new BrowserWindow({
|
||||
width: 760,
|
||||
height: 560,
|
||||
show: false,
|
||||
center: true,
|
||||
autoHideMenuBar: true,
|
||||
title: 'SurfSense — choose window',
|
||||
webPreferences: {
|
||||
preload: path.join(__dirname, 'modules', 'screen-capture', 'window-picker-preload.js'),
|
||||
contextIsolation: true,
|
||||
nodeIntegration: false,
|
||||
sandbox: true,
|
||||
},
|
||||
});
|
||||
|
||||
pickerWc = picker.webContents;
|
||||
|
||||
pickerWc.on('before-input-event', onBeforeInput);
|
||||
pickerWc.ipc.on(IPC_CHANNELS.WINDOW_PICK_SUBMIT, onSubmit);
|
||||
pickerWc.ipc.on(IPC_CHANNELS.WINDOW_PICK_CANCEL, onCancel);
|
||||
|
||||
picker.on('closed', () => {
|
||||
if (!settled) finish(null);
|
||||
});
|
||||
|
||||
picker
|
||||
.loadURL(
|
||||
'data:text/html;charset=utf-8,' +
|
||||
encodeURIComponent('<!doctype html><html><head><meta charset="utf-8"/></head><body></body></html>')
|
||||
)
|
||||
.catch(() => finish(null));
|
||||
|
||||
picker.webContents.once('did-finish-load', () => {
|
||||
void (async () => {
|
||||
if (!picker || picker.isDestroyed()) return;
|
||||
let selfId = '';
|
||||
try {
|
||||
selfId = picker.getMediaSourceId();
|
||||
} catch {
|
||||
selfId = '';
|
||||
}
|
||||
try {
|
||||
const { width, height } = maxCaptureThumbSize();
|
||||
const sources = await desktopCapturer.getSources({
|
||||
types: ['window'],
|
||||
thumbnailSize: { width, height },
|
||||
fetchWindowIcons: false,
|
||||
});
|
||||
sessionSources = sources.filter((s) => !(selfId && s.id === selfId));
|
||||
} catch {
|
||||
sessionSources = [];
|
||||
}
|
||||
if (sessionSources.length === 1) {
|
||||
const only = sessionSources[0];
|
||||
if (!only.thumbnail.isEmpty()) {
|
||||
finish({ sourceId: only.id, dataUrl: only.thumbnail.toDataURL() });
|
||||
return;
|
||||
}
|
||||
}
|
||||
try {
|
||||
await picker.webContents.executeJavaScript(buildPickerInjectScript(), true);
|
||||
if (!picker.isDestroyed()) picker.show();
|
||||
} catch {
|
||||
finish(null);
|
||||
}
|
||||
})();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
|
@ -1,13 +1,13 @@
|
|||
export interface ShortcutConfig {
|
||||
generalAssist: string;
|
||||
quickAsk: string;
|
||||
autocomplete: string;
|
||||
screenshotAssist: string;
|
||||
}
|
||||
|
||||
const DEFAULTS: ShortcutConfig = {
|
||||
generalAssist: 'CommandOrControl+Shift+S',
|
||||
quickAsk: 'CommandOrControl+Alt+S',
|
||||
autocomplete: 'CommandOrControl+Shift+Space',
|
||||
screenshotAssist: 'CommandOrControl+Shift+Space',
|
||||
};
|
||||
|
||||
const STORE_KEY = 'shortcuts';
|
||||
|
|
@ -27,14 +27,30 @@ async function getStore() {
|
|||
|
||||
export async function getShortcuts(): Promise<ShortcutConfig> {
|
||||
const s = await getStore();
|
||||
const stored = s.get(STORE_KEY) as Partial<ShortcutConfig> | undefined;
|
||||
return { ...DEFAULTS, ...stored };
|
||||
const raw = (s.get(STORE_KEY) as Record<string, string> | undefined) ?? {};
|
||||
const legacyAutocomplete = raw.autocomplete;
|
||||
const { autocomplete: _drop, ...rest } = raw;
|
||||
let merged: ShortcutConfig = { ...DEFAULTS, ...rest };
|
||||
if (
|
||||
typeof legacyAutocomplete === 'string' &&
|
||||
legacyAutocomplete.length > 0 &&
|
||||
!('screenshotAssist' in raw)
|
||||
) {
|
||||
merged = { ...merged, screenshotAssist: legacyAutocomplete };
|
||||
s.set(STORE_KEY, {
|
||||
generalAssist: merged.generalAssist,
|
||||
quickAsk: merged.quickAsk,
|
||||
screenshotAssist: merged.screenshotAssist,
|
||||
});
|
||||
}
|
||||
return merged;
|
||||
}
|
||||
|
||||
export async function setShortcuts(config: Partial<ShortcutConfig>): Promise<ShortcutConfig> {
|
||||
const s = await getStore();
|
||||
const current = (s.get(STORE_KEY) as ShortcutConfig) ?? DEFAULTS;
|
||||
const merged = { ...current, ...config };
|
||||
const raw = (s.get(STORE_KEY) as Record<string, string> | undefined) ?? {};
|
||||
const { autocomplete: _drop, ...current } = raw;
|
||||
const merged = { ...DEFAULTS, ...current, ...config };
|
||||
s.set(STORE_KEY, merged);
|
||||
return merged;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,13 +1,16 @@
|
|||
import { app, globalShortcut, Menu, nativeImage, Tray } from 'electron';
|
||||
import { app, globalShortcut, Menu, nativeImage, Tray, type NativeImage } from 'electron';
|
||||
import path from 'path';
|
||||
import { getMainWindow, createMainWindow } from './window';
|
||||
import { runGeneralAssistShortcut } from './general-assist';
|
||||
import { runScreenshotAssistShortcut } from './screen-capture';
|
||||
import { showMainWindow } from './window';
|
||||
import { getShortcuts } from './shortcuts';
|
||||
import { trackEvent } from './analytics';
|
||||
|
||||
let tray: Tray | null = null;
|
||||
let currentShortcut: string | null = null;
|
||||
let registeredGeneralAssist: string | null = null;
|
||||
let registeredScreenshotAssist: string | null = null;
|
||||
|
||||
function getTrayIcon(): nativeImage {
|
||||
function getTrayIcon(): NativeImage {
|
||||
const iconName = process.platform === 'win32' ? 'icon.ico' : 'icon.png';
|
||||
const iconPath = app.isPackaged
|
||||
? path.join(process.resourcesPath, 'assets', iconName)
|
||||
|
|
@ -16,34 +19,29 @@ function getTrayIcon(): nativeImage {
|
|||
return img.resize({ width: 16, height: 16 });
|
||||
}
|
||||
|
||||
function showMainWindow(source: 'tray_click' | 'tray_menu' | 'shortcut' = 'tray_click'): void {
|
||||
const existing = getMainWindow();
|
||||
const reopened = !existing || existing.isDestroyed();
|
||||
if (reopened) {
|
||||
createMainWindow('/dashboard');
|
||||
} else {
|
||||
existing.show();
|
||||
existing.focus();
|
||||
function registerOne(
|
||||
previous: string | null,
|
||||
accelerator: string,
|
||||
onFire: () => void | Promise<void>,
|
||||
label: string
|
||||
): string | null {
|
||||
if (previous) {
|
||||
globalShortcut.unregister(previous);
|
||||
}
|
||||
trackEvent('desktop_main_window_shown', { source, reopened });
|
||||
}
|
||||
|
||||
function registerShortcut(accelerator: string): void {
|
||||
if (currentShortcut) {
|
||||
globalShortcut.unregister(currentShortcut);
|
||||
currentShortcut = null;
|
||||
}
|
||||
if (!accelerator) return;
|
||||
if (!accelerator) return null;
|
||||
try {
|
||||
const ok = globalShortcut.register(accelerator, () => showMainWindow('shortcut'));
|
||||
const ok = globalShortcut.register(accelerator, () => {
|
||||
void Promise.resolve(onFire());
|
||||
});
|
||||
if (ok) {
|
||||
currentShortcut = accelerator;
|
||||
} else {
|
||||
console.warn(`[tray] Failed to register General Assist shortcut: ${accelerator}`);
|
||||
console.log(`[hotkeys] Register ${label} ${accelerator}: OK`);
|
||||
return accelerator;
|
||||
}
|
||||
console.warn(`[hotkeys] Register ${label} ${accelerator}: FAILED (OS or another app may own this chord)`);
|
||||
} catch (err) {
|
||||
console.error(`[tray] Error registering General Assist shortcut:`, err);
|
||||
console.error(`[tray] Error registering ${label} shortcut:`, err);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export async function createTray(): Promise<void> {
|
||||
|
|
@ -68,18 +66,48 @@ export async function createTray(): Promise<void> {
|
|||
tray.on('double-click', () => showMainWindow('tray_click'));
|
||||
|
||||
const shortcuts = await getShortcuts();
|
||||
registerShortcut(shortcuts.generalAssist);
|
||||
registeredGeneralAssist = registerOne(
|
||||
null,
|
||||
shortcuts.generalAssist,
|
||||
runGeneralAssistShortcut,
|
||||
'General Assist'
|
||||
);
|
||||
registeredScreenshotAssist = registerOne(
|
||||
null,
|
||||
shortcuts.screenshotAssist,
|
||||
runScreenshotAssistShortcut,
|
||||
'Screenshot Assist'
|
||||
);
|
||||
}
|
||||
|
||||
export async function reregisterGeneralAssist(): Promise<void> {
|
||||
const shortcuts = await getShortcuts();
|
||||
registerShortcut(shortcuts.generalAssist);
|
||||
registeredGeneralAssist = registerOne(
|
||||
registeredGeneralAssist,
|
||||
shortcuts.generalAssist,
|
||||
runGeneralAssistShortcut,
|
||||
'General Assist'
|
||||
);
|
||||
}
|
||||
|
||||
export async function reregisterScreenshotAssist(): Promise<void> {
|
||||
const shortcuts = await getShortcuts();
|
||||
registeredScreenshotAssist = registerOne(
|
||||
registeredScreenshotAssist,
|
||||
shortcuts.screenshotAssist,
|
||||
runScreenshotAssistShortcut,
|
||||
'Screenshot Assist'
|
||||
);
|
||||
}
|
||||
|
||||
export function destroyTray(): void {
|
||||
if (currentShortcut) {
|
||||
globalShortcut.unregister(currentShortcut);
|
||||
currentShortcut = null;
|
||||
if (registeredGeneralAssist) {
|
||||
globalShortcut.unregister(registeredGeneralAssist);
|
||||
registeredGeneralAssist = null;
|
||||
}
|
||||
if (registeredScreenshotAssist) {
|
||||
globalShortcut.unregister(registeredScreenshotAssist);
|
||||
registeredScreenshotAssist = null;
|
||||
}
|
||||
tray?.destroy();
|
||||
tray = null;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import { app, BrowserWindow, shell, session } from 'electron';
|
||||
import path from 'path';
|
||||
import { trackEvent } from './analytics';
|
||||
import { showErrorDialog } from './errors';
|
||||
import { getServerPort } from './server';
|
||||
import { setActiveSearchSpaceId } from './active-search-space';
|
||||
|
|
@ -93,3 +94,15 @@ export function createMainWindow(initialPath = '/dashboard'): BrowserWindow {
|
|||
|
||||
return mainWindow;
|
||||
}
|
||||
|
||||
export function showMainWindow(source: 'tray_click' | 'tray_menu' | 'shortcut' = 'tray_click'): void {
|
||||
const existing = getMainWindow();
|
||||
const reopened = !existing || existing.isDestroyed();
|
||||
if (reopened) {
|
||||
createMainWindow('/dashboard');
|
||||
} else {
|
||||
existing.show();
|
||||
existing.focus();
|
||||
}
|
||||
trackEvent('desktop_main_window_shown', { source, reopened });
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,6 +17,13 @@ contextBridge.exposeInMainWorld('electronAPI', {
|
|||
ipcRenderer.removeListener(IPC_CHANNELS.DEEP_LINK, listener);
|
||||
};
|
||||
},
|
||||
onChatScreenCapture: (callback: (dataUrl: string) => void) => {
|
||||
const listener = (_event: unknown, dataUrl: string) => callback(dataUrl);
|
||||
ipcRenderer.on(IPC_CHANNELS.CHAT_SCREEN_CAPTURE, listener);
|
||||
return () => {
|
||||
ipcRenderer.removeListener(IPC_CHANNELS.CHAT_SCREEN_CAPTURE, listener);
|
||||
};
|
||||
},
|
||||
getQuickAskText: () => ipcRenderer.invoke(IPC_CHANNELS.QUICK_ASK_TEXT),
|
||||
setQuickAskMode: (mode: string) => ipcRenderer.invoke(IPC_CHANNELS.SET_QUICK_ASK_MODE, mode),
|
||||
getQuickAskMode: () => ipcRenderer.invoke(IPC_CHANNELS.GET_QUICK_ASK_MODE),
|
||||
|
|
@ -25,20 +32,8 @@ contextBridge.exposeInMainWorld('electronAPI', {
|
|||
getPermissionsStatus: () => ipcRenderer.invoke(IPC_CHANNELS.GET_PERMISSIONS_STATUS),
|
||||
requestAccessibility: () => ipcRenderer.invoke(IPC_CHANNELS.REQUEST_ACCESSIBILITY),
|
||||
requestScreenRecording: () => ipcRenderer.invoke(IPC_CHANNELS.REQUEST_SCREEN_RECORDING),
|
||||
captureFullScreen: () => ipcRenderer.invoke(IPC_CHANNELS.CAPTURE_FULL_SCREEN),
|
||||
restartApp: () => ipcRenderer.invoke(IPC_CHANNELS.RESTART_APP),
|
||||
// Autocomplete
|
||||
onAutocompleteContext: (callback: (data: { screenshot: string; searchSpaceId?: string; appName?: string; windowTitle?: string }) => void) => {
|
||||
const listener = (_event: unknown, data: { screenshot: string; searchSpaceId?: string; appName?: string; windowTitle?: string }) => callback(data);
|
||||
ipcRenderer.on(IPC_CHANNELS.AUTOCOMPLETE_CONTEXT, listener);
|
||||
return () => {
|
||||
ipcRenderer.removeListener(IPC_CHANNELS.AUTOCOMPLETE_CONTEXT, listener);
|
||||
};
|
||||
},
|
||||
acceptSuggestion: (text: string) => ipcRenderer.invoke(IPC_CHANNELS.ACCEPT_SUGGESTION, text),
|
||||
dismissSuggestion: () => ipcRenderer.invoke(IPC_CHANNELS.DISMISS_SUGGESTION),
|
||||
setAutocompleteEnabled: (enabled: boolean) => ipcRenderer.invoke(IPC_CHANNELS.SET_AUTOCOMPLETE_ENABLED, enabled),
|
||||
getAutocompleteEnabled: () => ipcRenderer.invoke(IPC_CHANNELS.GET_AUTOCOMPLETE_ENABLED),
|
||||
|
||||
// Folder sync
|
||||
selectFolder: () => ipcRenderer.invoke(IPC_CHANNELS.FOLDER_SYNC_SELECT_FOLDER),
|
||||
addWatchedFolder: (config: any) => ipcRenderer.invoke(IPC_CHANNELS.FOLDER_SYNC_ADD_FOLDER, config),
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import { useTranslations } from "next-intl";
|
|||
import type React from "react";
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { toast } from "sonner";
|
||||
import { pendingUserImageDataUrlsAtom } from "@/atoms/chat/pending-user-images.atom";
|
||||
import { myAccessAtom } from "@/atoms/members/members-query.atoms";
|
||||
import { updateLLMPreferencesMutationAtom } from "@/atoms/new-llm-config/new-llm-config-mutation.atoms";
|
||||
import {
|
||||
|
|
@ -33,6 +34,7 @@ export function DashboardClientLayout({
|
|||
const pathname = usePathname();
|
||||
const { search_space_id } = useParams();
|
||||
const setActiveSearchSpaceIdState = useSetAtom(activeSearchSpaceIdAtom);
|
||||
const setPendingUserImageUrls = useSetAtom(pendingUserImageDataUrlsAtom);
|
||||
|
||||
const {
|
||||
data: preferences = {},
|
||||
|
|
@ -142,6 +144,14 @@ export function DashboardClientLayout({
|
|||
|
||||
const electronAPI = useElectronAPI();
|
||||
|
||||
useEffect(() => {
|
||||
if (!electronAPI?.onChatScreenCapture) return;
|
||||
return electronAPI.onChatScreenCapture((dataUrl: string) => {
|
||||
if (typeof dataUrl !== "string" || !dataUrl.startsWith("data:image/")) return;
|
||||
setPendingUserImageUrls((prev) => [...prev, dataUrl]);
|
||||
});
|
||||
}, [electronAPI, setPendingUserImageUrls]);
|
||||
|
||||
useEffect(() => {
|
||||
const activeSeacrhSpaceId =
|
||||
typeof search_space_id === "string"
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ import {
|
|||
messageDocumentsMapAtom,
|
||||
sidebarSelectedDocumentsAtom,
|
||||
} from "@/atoms/chat/mentioned-documents.atom";
|
||||
import { pendingUserImageDataUrlsAtom } from "@/atoms/chat/pending-user-images.atom";
|
||||
import {
|
||||
clearPlanOwnerRegistry,
|
||||
// extractWriteTodosFromContent,
|
||||
|
|
@ -76,6 +77,10 @@ import {
|
|||
type ThreadListResponse,
|
||||
type ThreadRecord,
|
||||
} from "@/lib/chat/thread-persistence";
|
||||
import {
|
||||
extractUserTurnForNewChatApi,
|
||||
type NewChatUserImagePayload,
|
||||
} from "@/lib/chat/user-turn-api-parts";
|
||||
import { NotFoundError } from "@/lib/error";
|
||||
import {
|
||||
trackChatCreated,
|
||||
|
|
@ -231,6 +236,8 @@ export default function NewChatPage() {
|
|||
const updateChatTabTitle = useSetAtom(updateChatTabTitleAtom);
|
||||
const removeChatTab = useSetAtom(removeChatTabAtom);
|
||||
const setAgentCreatedDocuments = useSetAtom(agentCreatedDocumentsAtom);
|
||||
const pendingUserImageUrls = useAtomValue(pendingUserImageDataUrlsAtom);
|
||||
const setPendingUserImageUrls = useSetAtom(pendingUserImageDataUrlsAtom);
|
||||
|
||||
// Get current user for author info in shared chats
|
||||
const { data: currentUser } = useAtomValue(currentUserAtom);
|
||||
|
|
@ -494,18 +501,12 @@ export default function NewChatPage() {
|
|||
abortControllerRef.current = null;
|
||||
}
|
||||
|
||||
// Extract user query text from content parts
|
||||
let userQuery = "";
|
||||
for (const part of message.content) {
|
||||
if (part.type === "text") {
|
||||
userQuery += part.text;
|
||||
}
|
||||
}
|
||||
const urlsSnapshot = [...pendingUserImageUrls];
|
||||
const { userQuery, userImages } = extractUserTurnForNewChatApi(message, urlsSnapshot);
|
||||
|
||||
if (!userQuery.trim()) return;
|
||||
if (!userQuery.trim() && userImages.length === 0) return;
|
||||
|
||||
// Check if podcast is already generating
|
||||
if (isPodcastGenerating() && looksLikePodcastRequest(userQuery)) {
|
||||
if (userQuery.trim() && isPodcastGenerating() && looksLikePodcastRequest(userQuery)) {
|
||||
toast.warning("A podcast is already being generated.");
|
||||
return;
|
||||
}
|
||||
|
|
@ -545,6 +546,10 @@ export default function NewChatPage() {
|
|||
}
|
||||
}
|
||||
|
||||
if (urlsSnapshot.length > 0) {
|
||||
setPendingUserImageUrls((prev) => prev.filter((u) => !urlsSnapshot.includes(u)));
|
||||
}
|
||||
|
||||
// Add user message to state
|
||||
const userMsgId = `msg-user-${Date.now()}`;
|
||||
|
||||
|
|
@ -560,10 +565,27 @@ export default function NewChatPage() {
|
|||
}
|
||||
: undefined;
|
||||
|
||||
const existingImageUrls = new Set(
|
||||
message.content
|
||||
.filter(
|
||||
(p): p is { type: "image"; image: string } =>
|
||||
typeof p === "object" &&
|
||||
p !== null &&
|
||||
"type" in p &&
|
||||
p.type === "image" &&
|
||||
"image" in p
|
||||
)
|
||||
.map((p) => p.image)
|
||||
);
|
||||
const extraImageParts = urlsSnapshot
|
||||
.filter((u) => !existingImageUrls.has(u))
|
||||
.map((image) => ({ type: "image" as const, image }));
|
||||
const userDisplayContent = [...message.content, ...extraImageParts];
|
||||
|
||||
const userMessage: ThreadMessageLike = {
|
||||
id: userMsgId,
|
||||
role: "user",
|
||||
content: message.content,
|
||||
content: userDisplayContent,
|
||||
createdAt: new Date(),
|
||||
metadata: authorMetadata,
|
||||
};
|
||||
|
|
@ -571,7 +593,7 @@ export default function NewChatPage() {
|
|||
|
||||
// Track message sent
|
||||
trackChatMessageSent(searchSpaceId, currentThreadId, {
|
||||
hasAttachments: false,
|
||||
hasAttachments: userImages.length > 0,
|
||||
hasMentionedDocuments:
|
||||
mentionedDocumentIds.surfsense_doc_ids.length > 0 ||
|
||||
mentionedDocumentIds.document_ids.length > 0,
|
||||
|
|
@ -596,7 +618,7 @@ export default function NewChatPage() {
|
|||
}));
|
||||
}
|
||||
|
||||
const persistContent: unknown[] = [...message.content];
|
||||
const persistContent: unknown[] = [...userDisplayContent];
|
||||
|
||||
if (allMentionedDocs.length > 0) {
|
||||
persistContent.push({
|
||||
|
|
@ -710,6 +732,7 @@ export default function NewChatPage() {
|
|||
? mentionedDocumentIds.surfsense_doc_ids
|
||||
: undefined,
|
||||
disabled_tools: disabledTools.length > 0 ? disabledTools : undefined,
|
||||
...(userImages.length > 0 ? { user_images: userImages } : {}),
|
||||
}),
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
|
@ -981,6 +1004,9 @@ export default function NewChatPage() {
|
|||
disabledTools,
|
||||
updateChatTabTitle,
|
||||
tokenUsageStore,
|
||||
pendingUserImageUrls,
|
||||
setPendingUserImageUrls,
|
||||
toolsWithUI,
|
||||
]
|
||||
);
|
||||
|
||||
|
|
@ -1246,7 +1272,7 @@ export default function NewChatPage() {
|
|||
abortControllerRef.current = null;
|
||||
}
|
||||
},
|
||||
[pendingInterrupt, messages, searchSpaceId, tokenUsageStore]
|
||||
[pendingInterrupt, messages, searchSpaceId, tokenUsageStore, toolsWithUI]
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
|
|
@ -1314,15 +1340,24 @@ export default function NewChatPage() {
|
|||
* Handle regeneration (edit or reload) by calling the regenerate endpoint
|
||||
* and streaming the response. This rewinds the LangGraph checkpointer state.
|
||||
*
|
||||
* @param newUserQuery - The new user query (for edit). Pass null/undefined for reload.
|
||||
* @param newUserQuery - `null` = reload with same turn from the server. A string = edit
|
||||
* (including an empty string when the edited turn is images-only); pass `editExtras` for images/content.
|
||||
*/
|
||||
const handleRegenerate = useCallback(
|
||||
async (newUserQuery?: string | null) => {
|
||||
async (
|
||||
newUserQuery: string | null,
|
||||
editExtras?: {
|
||||
userMessageContent: ThreadMessageLike["content"];
|
||||
userImages: NewChatUserImagePayload[];
|
||||
}
|
||||
) => {
|
||||
if (!threadId) {
|
||||
toast.error("Cannot regenerate: no active chat thread");
|
||||
return;
|
||||
}
|
||||
|
||||
const isEdit = newUserQuery !== null;
|
||||
|
||||
// Abort any previous streaming request
|
||||
if (abortControllerRef.current) {
|
||||
abortControllerRef.current.abort();
|
||||
|
|
@ -1336,11 +1371,11 @@ export default function NewChatPage() {
|
|||
}
|
||||
|
||||
// Extract the original user query BEFORE removing messages (for reload mode)
|
||||
let userQueryToDisplay = newUserQuery;
|
||||
let userQueryToDisplay: string | undefined;
|
||||
let originalUserMessageContent: ThreadMessageLike["content"] | null = null;
|
||||
let originalUserMessageMetadata: ThreadMessageLike["metadata"] | undefined;
|
||||
|
||||
if (!newUserQuery) {
|
||||
if (!isEdit) {
|
||||
// Reload mode - find and preserve the last user message content
|
||||
const lastUserMessage = [...messages].reverse().find((m) => m.role === "user");
|
||||
if (lastUserMessage) {
|
||||
|
|
@ -1354,6 +1389,8 @@ export default function NewChatPage() {
|
|||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
userQueryToDisplay = newUserQuery;
|
||||
}
|
||||
|
||||
// Remove the last two messages (user + assistant) from the UI immediately
|
||||
|
|
@ -1389,11 +1426,11 @@ export default function NewChatPage() {
|
|||
const userMessage: ThreadMessageLike = {
|
||||
id: userMsgId,
|
||||
role: "user",
|
||||
content: newUserQuery
|
||||
? [{ type: "text", text: newUserQuery }]
|
||||
content: isEdit
|
||||
? (editExtras?.userMessageContent ?? [{ type: "text", text: newUserQuery ?? "" }])
|
||||
: originalUserMessageContent || [{ type: "text", text: userQueryToDisplay || "" }],
|
||||
createdAt: new Date(),
|
||||
metadata: newUserQuery ? undefined : originalUserMessageMetadata,
|
||||
metadata: isEdit ? undefined : originalUserMessageMetadata,
|
||||
};
|
||||
setMessages((prev) => [...prev, userMessage]);
|
||||
|
||||
|
|
@ -1410,20 +1447,24 @@ export default function NewChatPage() {
|
|||
|
||||
try {
|
||||
const selection = await getAgentFilesystemSelection(searchSpaceId);
|
||||
const requestBody: Record<string, unknown> = {
|
||||
search_space_id: searchSpaceId,
|
||||
user_query: newUserQuery,
|
||||
disabled_tools: disabledTools.length > 0 ? disabledTools : undefined,
|
||||
filesystem_mode: selection.filesystem_mode,
|
||||
client_platform: selection.client_platform,
|
||||
local_filesystem_mounts: selection.local_filesystem_mounts,
|
||||
};
|
||||
if (isEdit) {
|
||||
requestBody.user_images = editExtras?.userImages ?? [];
|
||||
}
|
||||
const response = await fetch(getRegenerateUrl(threadId), {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
Authorization: `Bearer ${token}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
search_space_id: searchSpaceId,
|
||||
user_query: newUserQuery || null,
|
||||
disabled_tools: disabledTools.length > 0 ? disabledTools : undefined,
|
||||
filesystem_mode: selection.filesystem_mode,
|
||||
client_platform: selection.client_platform,
|
||||
local_filesystem_mounts: selection.local_filesystem_mounts,
|
||||
}),
|
||||
body: JSON.stringify(requestBody),
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
|
|
@ -1513,8 +1554,8 @@ export default function NewChatPage() {
|
|||
if (contentParts.length > 0) {
|
||||
try {
|
||||
// Persist user message (for both edit and reload modes, since backend deleted it)
|
||||
const userContentToPersist = newUserQuery
|
||||
? [{ type: "text", text: newUserQuery }]
|
||||
const userContentToPersist = isEdit
|
||||
? (editExtras?.userMessageContent ?? [{ type: "text", text: newUserQuery ?? "" }])
|
||||
: originalUserMessageContent || [{ type: "text", text: userQueryToDisplay || "" }];
|
||||
|
||||
const savedUserMessage = await appendMessage(threadId, {
|
||||
|
|
@ -1573,27 +1614,21 @@ export default function NewChatPage() {
|
|||
abortControllerRef.current = null;
|
||||
}
|
||||
},
|
||||
[threadId, searchSpaceId, messages, disabledTools, tokenUsageStore]
|
||||
[threadId, searchSpaceId, messages, disabledTools, tokenUsageStore, toolsWithUI]
|
||||
);
|
||||
|
||||
// Handle editing a message - truncates history and regenerates with new query
|
||||
const onEdit = useCallback(
|
||||
async (message: AppendMessage) => {
|
||||
// Extract the new user query from the message content
|
||||
let newUserQuery = "";
|
||||
for (const part of message.content) {
|
||||
if (part.type === "text") {
|
||||
newUserQuery += part.text;
|
||||
}
|
||||
}
|
||||
|
||||
if (!newUserQuery.trim()) {
|
||||
const { userQuery, userImages } = extractUserTurnForNewChatApi(message, []);
|
||||
const queryForApi = userQuery.trim();
|
||||
if (!queryForApi && userImages.length === 0) {
|
||||
toast.error("Cannot edit with empty message");
|
||||
return;
|
||||
}
|
||||
|
||||
// Call regenerate with the new query
|
||||
await handleRegenerate(newUserQuery.trim());
|
||||
const userMessageContent = message.content as unknown as ThreadMessageLike["content"];
|
||||
await handleRegenerate(queryForApi, { userMessageContent, userImages });
|
||||
},
|
||||
[handleRegenerate]
|
||||
);
|
||||
|
|
|
|||
|
|
@ -20,7 +20,6 @@ import { searchSpacesApiService } from "@/lib/apis/search-spaces-api.service";
|
|||
export function DesktopContent() {
|
||||
const api = useElectronAPI();
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [enabled, setEnabled] = useState(true);
|
||||
|
||||
const [searchSpaces, setSearchSpaces] = useState<SearchSpace[]>([]);
|
||||
const [activeSpaceId, setActiveSpaceId] = useState<string | null>(null);
|
||||
|
|
@ -41,14 +40,12 @@ export function DesktopContent() {
|
|||
setAutoLaunchSupported(hasAutoLaunchApi);
|
||||
|
||||
Promise.all([
|
||||
api.getAutocompleteEnabled(),
|
||||
api.getActiveSearchSpace?.() ?? Promise.resolve(null),
|
||||
searchSpacesApiService.getSearchSpaces(),
|
||||
hasAutoLaunchApi ? api.getAutoLaunch() : Promise.resolve(null),
|
||||
])
|
||||
.then(([autoEnabled, spaceId, spaces, autoLaunch]) => {
|
||||
.then(([spaceId, spaces, autoLaunch]) => {
|
||||
if (!mounted) return;
|
||||
setEnabled(autoEnabled);
|
||||
setActiveSpaceId(spaceId);
|
||||
if (spaces) setSearchSpaces(spaces);
|
||||
if (autoLaunch) {
|
||||
|
|
@ -86,11 +83,6 @@ export function DesktopContent() {
|
|||
);
|
||||
}
|
||||
|
||||
const handleToggle = async (checked: boolean) => {
|
||||
setEnabled(checked);
|
||||
await api.setAutocompleteEnabled(checked);
|
||||
};
|
||||
|
||||
const handleAutoLaunchToggle = async (checked: boolean) => {
|
||||
if (!autoLaunchSupported || !api.setAutoLaunch) {
|
||||
toast.error("Please update the desktop app to configure launch on startup");
|
||||
|
|
@ -133,13 +125,12 @@ export function DesktopContent() {
|
|||
|
||||
return (
|
||||
<div className="space-y-4 md:space-y-6">
|
||||
{/* Default Search Space */}
|
||||
<Card>
|
||||
<CardHeader className="px-3 md:px-6 pt-3 md:pt-6 pb-2 md:pb-3">
|
||||
<CardTitle className="text-base md:text-lg">Default Search Space</CardTitle>
|
||||
<CardDescription className="text-xs md:text-sm">
|
||||
Choose which search space General Assist, Quick Assist, and Extreme Assist operate
|
||||
against.
|
||||
Choose which search space General Assist, Screenshot Assist, and Quick Assist use by
|
||||
default.
|
||||
</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent className="px-3 md:px-6 pb-3 md:pb-6">
|
||||
|
|
@ -164,7 +155,6 @@ export function DesktopContent() {
|
|||
</CardContent>
|
||||
</Card>
|
||||
|
||||
{/* Launch on Startup */}
|
||||
<Card>
|
||||
<CardHeader className="px-3 md:px-6 pt-3 md:pt-6 pb-2 md:pb-3">
|
||||
<CardTitle className="text-base md:text-lg flex items-center gap-2">
|
||||
|
|
@ -215,29 +205,6 @@ export function DesktopContent() {
|
|||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
{/* Extreme Assist Toggle */}
|
||||
<Card>
|
||||
<CardHeader className="px-3 md:px-6 pt-3 md:pt-6 pb-2 md:pb-3">
|
||||
<CardTitle className="text-base md:text-lg">Extreme Assist</CardTitle>
|
||||
<CardDescription className="text-xs md:text-sm">
|
||||
Get inline writing suggestions powered by your knowledge base as you type in any app.
|
||||
</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent className="px-3 md:px-6 pb-3 md:pb-6">
|
||||
<div className="flex items-center justify-between rounded-lg border p-4">
|
||||
<div className="space-y-0.5">
|
||||
<Label htmlFor="autocomplete-toggle" className="text-sm font-medium cursor-pointer">
|
||||
Enable Extreme Assist
|
||||
</Label>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Show suggestions while typing in other applications.
|
||||
</p>
|
||||
</div>
|
||||
<Switch id="autocomplete-toggle" checked={enabled} onCheckedChange={handleToggle} />
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
"use client";
|
||||
|
||||
import { BrainCog, Rocket, RotateCcw, Zap } from "lucide-react";
|
||||
import { Crop, Rocket, RotateCcw, Zap } from "lucide-react";
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
import { toast } from "sonner";
|
||||
import { DEFAULT_SHORTCUTS, keyEventToAccelerator } from "@/components/desktop/shortcut-recorder";
|
||||
|
|
@ -9,13 +9,13 @@ import { ShortcutKbd } from "@/components/ui/shortcut-kbd";
|
|||
import { Spinner } from "@/components/ui/spinner";
|
||||
import { useElectronAPI } from "@/hooks/use-platform";
|
||||
|
||||
type ShortcutKey = "generalAssist" | "quickAsk" | "autocomplete";
|
||||
type ShortcutKey = "generalAssist" | "quickAsk" | "screenshotAssist";
|
||||
type ShortcutMap = typeof DEFAULT_SHORTCUTS;
|
||||
|
||||
const HOTKEY_ROWS: Array<{ key: ShortcutKey; label: string; icon: React.ElementType }> = [
|
||||
{ key: "generalAssist", label: "General Assist", icon: Rocket },
|
||||
{ key: "screenshotAssist", label: "Screenshot Assist", icon: Crop },
|
||||
{ key: "quickAsk", label: "Quick Assist", icon: Zap },
|
||||
{ key: "autocomplete", label: "Extreme Assist", icon: BrainCog },
|
||||
];
|
||||
|
||||
function acceleratorToKeys(accel: string, isMac: boolean): string[] {
|
||||
|
|
@ -160,10 +160,7 @@ export function DesktopShortcutsContent() {
|
|||
);
|
||||
}
|
||||
|
||||
const updateShortcut = (
|
||||
key: "generalAssist" | "quickAsk" | "autocomplete",
|
||||
accelerator: string
|
||||
) => {
|
||||
const updateShortcut = (key: ShortcutKey, accelerator: string) => {
|
||||
setShortcuts((prev) => {
|
||||
const updated = { ...prev, [key]: accelerator };
|
||||
api.setShortcuts?.({ [key]: accelerator }).catch(() => {
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
import { IconBrandGoogleFilled } from "@tabler/icons-react";
|
||||
import { useAtom } from "jotai";
|
||||
import { BrainCog, Eye, EyeOff, Rocket, RotateCcw, Zap } from "lucide-react";
|
||||
import { Crop, Eye, EyeOff, Rocket, RotateCcw, Zap } from "lucide-react";
|
||||
import Image from "next/image";
|
||||
import { useRouter } from "next/navigation";
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
|
|
@ -21,7 +21,7 @@ import { setBearerToken } from "@/lib/auth-utils";
|
|||
import { AUTH_TYPE, BACKEND_URL } from "@/lib/env-config";
|
||||
|
||||
const isGoogleAuth = AUTH_TYPE === "GOOGLE";
|
||||
type ShortcutKey = "generalAssist" | "quickAsk" | "autocomplete";
|
||||
type ShortcutKey = "generalAssist" | "quickAsk" | "screenshotAssist";
|
||||
type ShortcutMap = typeof DEFAULT_SHORTCUTS;
|
||||
|
||||
const HOTKEY_ROWS: Array<{
|
||||
|
|
@ -36,18 +36,18 @@ const HOTKEY_ROWS: Array<{
|
|||
description: "Launch SurfSense instantly from any application",
|
||||
icon: Rocket,
|
||||
},
|
||||
{
|
||||
key: "screenshotAssist",
|
||||
label: "Screenshot Assist",
|
||||
description: "Draw a region on screen to attach that capture to chat",
|
||||
icon: Crop,
|
||||
},
|
||||
{
|
||||
key: "quickAsk",
|
||||
label: "Quick Assist",
|
||||
description: "Select text anywhere, then ask AI to explain, rewrite, or act on it",
|
||||
icon: Zap,
|
||||
},
|
||||
{
|
||||
key: "autocomplete",
|
||||
label: "Extreme Assist",
|
||||
description: "AI drafts text using your screen context and knowledge base",
|
||||
icon: BrainCog,
|
||||
},
|
||||
];
|
||||
|
||||
function acceleratorToKeys(accel: string, isMac: boolean): string[] {
|
||||
|
|
@ -187,7 +187,7 @@ export default function DesktopLoginPage() {
|
|||
}, [api]);
|
||||
|
||||
const updateShortcut = useCallback(
|
||||
(key: "generalAssist" | "quickAsk" | "autocomplete", accelerator: string) => {
|
||||
(key: ShortcutKey, accelerator: string) => {
|
||||
setShortcuts((prev) => {
|
||||
const updated = { ...prev, [key]: accelerator };
|
||||
api?.setShortcuts?.({ [key]: accelerator }).catch(() => {
|
||||
|
|
@ -201,7 +201,7 @@ export default function DesktopLoginPage() {
|
|||
);
|
||||
|
||||
const resetShortcut = useCallback(
|
||||
(key: "generalAssist" | "quickAsk" | "autocomplete") => {
|
||||
(key: ShortcutKey) => {
|
||||
updateShortcut(key, DEFAULT_SHORTCUTS[key]);
|
||||
},
|
||||
[updateShortcut]
|
||||
|
|
|
|||
|
|
@ -19,14 +19,15 @@ const STEPS = [
|
|||
id: "screen-recording",
|
||||
title: "Screen Recording",
|
||||
description:
|
||||
"Lets SurfSense capture your screen to understand context and provide smart writing suggestions.",
|
||||
"Lets SurfSense capture a region of your screen, full display, or browser (where supported) to attach to chat in Screenshot Assist, or to capture the full display from the composer.",
|
||||
action: "requestScreenRecording",
|
||||
field: "screenRecording" as const,
|
||||
},
|
||||
{
|
||||
id: "accessibility",
|
||||
title: "Accessibility",
|
||||
description: "Lets SurfSense insert suggestions seamlessly, right where you\u2019re typing.",
|
||||
description:
|
||||
"Lets SurfSense bring the app to the foreground and work with the active application (for example Quick Assist) when you use desktop shortcuts.",
|
||||
action: "requestAccessibility",
|
||||
field: "accessibility" as const,
|
||||
},
|
||||
|
|
@ -131,7 +132,8 @@ export default function DesktopPermissionsPage() {
|
|||
<div className="space-y-1">
|
||||
<h1 className="text-2xl font-semibold tracking-tight">System Permissions</h1>
|
||||
<p className="text-sm text-muted-foreground">
|
||||
SurfSense needs two macOS permissions to provide context-aware writing suggestions.
|
||||
SurfSense needs two macOS permissions for Screenshot Assist and for desktop features that
|
||||
require focusing the app or the active application.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -1,9 +0,0 @@
|
|||
import "./suggestion.css";
|
||||
|
||||
export const metadata = {
|
||||
title: "SurfSense Suggestion",
|
||||
};
|
||||
|
||||
export default function SuggestionLayout({ children }: { children: React.ReactNode }) {
|
||||
return <div className="suggestion-body">{children}</div>;
|
||||
}
|
||||
|
|
@ -1,384 +0,0 @@
|
|||
"use client";
|
||||
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { useElectronAPI } from "@/hooks/use-platform";
|
||||
import { ensureTokensFromElectron, getBearerToken } from "@/lib/auth-utils";
|
||||
|
||||
type SSEEvent =
|
||||
| { type: "text-delta"; id: string; delta: string }
|
||||
| { type: "text-start"; id: string }
|
||||
| { type: "text-end"; id: string }
|
||||
| { type: "start"; messageId: string }
|
||||
| { type: "finish" }
|
||||
| { type: "error"; errorText: string }
|
||||
| {
|
||||
type: "data-thinking-step";
|
||||
data: { id: string; title: string; status: string; items: string[] };
|
||||
}
|
||||
| {
|
||||
type: "data-suggestions";
|
||||
data: { options: string[] };
|
||||
};
|
||||
|
||||
interface AgentStep {
|
||||
id: string;
|
||||
title: string;
|
||||
status: string;
|
||||
items: string[];
|
||||
}
|
||||
|
||||
type FriendlyError = { message: string; isSetup?: boolean };
|
||||
|
||||
function friendlyError(raw: string | number): FriendlyError {
|
||||
if (typeof raw === "number") {
|
||||
if (raw === 401) return { message: "Please sign in to use suggestions." };
|
||||
if (raw === 403) return { message: "You don\u2019t have permission for this." };
|
||||
if (raw === 404) return { message: "Suggestion service not found. Is the backend running?" };
|
||||
if (raw >= 500) return { message: "Something went wrong on the server. Try again." };
|
||||
return { message: "Something went wrong. Try again." };
|
||||
}
|
||||
const lower = raw.toLowerCase();
|
||||
if (lower.includes("not authenticated") || lower.includes("unauthorized"))
|
||||
return { message: "Please sign in to use suggestions." };
|
||||
if (lower.includes("no vision llm configured") || lower.includes("no llm configured"))
|
||||
return {
|
||||
message: "Configure a vision-capable model (e.g. GPT-4o, Gemini) to enable autocomplete.",
|
||||
isSetup: true,
|
||||
};
|
||||
if (lower.includes("does not support vision"))
|
||||
return {
|
||||
message: "The selected model doesn\u2019t support vision. Choose a vision-capable model.",
|
||||
isSetup: true,
|
||||
};
|
||||
if (lower.includes("fetch") || lower.includes("network") || lower.includes("econnrefused"))
|
||||
return { message: "Can\u2019t reach the server. Check your connection." };
|
||||
return { message: "Something went wrong. Try again." };
|
||||
}
|
||||
|
||||
const AUTO_DISMISS_MS = 3000;
|
||||
|
||||
function StepIcon({ status }: { status: string }) {
|
||||
if (status === "complete") {
|
||||
return (
|
||||
<svg
|
||||
className="step-icon step-icon-done"
|
||||
viewBox="0 0 16 16"
|
||||
fill="none"
|
||||
aria-label="Step complete"
|
||||
>
|
||||
<circle cx="8" cy="8" r="7" stroke="#4ade80" strokeWidth="1.5" />
|
||||
<path
|
||||
d="M5 8.5l2 2 4-4.5"
|
||||
stroke="#4ade80"
|
||||
strokeWidth="1.5"
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
/>
|
||||
</svg>
|
||||
);
|
||||
}
|
||||
return <span className="step-spinner" />;
|
||||
}
|
||||
|
||||
export default function SuggestionPage() {
|
||||
const api = useElectronAPI();
|
||||
const [options, setOptions] = useState<string[]>([]);
|
||||
const [isLoading, setIsLoading] = useState(true);
|
||||
const [error, setError] = useState<FriendlyError | null>(null);
|
||||
const [steps, setSteps] = useState<AgentStep[]>([]);
|
||||
const [expandedOption, setExpandedOption] = useState<number | null>(null);
|
||||
const abortRef = useRef<AbortController | null>(null);
|
||||
|
||||
const isDesktop = !!api?.onAutocompleteContext;
|
||||
|
||||
useEffect(() => {
|
||||
if (!api?.onAutocompleteContext) {
|
||||
setIsLoading(false);
|
||||
}
|
||||
}, [api]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!error || error.isSetup) return;
|
||||
const timer = setTimeout(() => {
|
||||
api?.dismissSuggestion?.();
|
||||
}, AUTO_DISMISS_MS);
|
||||
return () => clearTimeout(timer);
|
||||
}, [error, api]);
|
||||
|
||||
useEffect(() => {
|
||||
if (isLoading || error || options.length > 0) return;
|
||||
const timer = setTimeout(() => {
|
||||
api?.dismissSuggestion?.();
|
||||
}, AUTO_DISMISS_MS);
|
||||
return () => clearTimeout(timer);
|
||||
}, [isLoading, error, options, api]);
|
||||
|
||||
const fetchSuggestion = useCallback(
|
||||
async (screenshot: string, searchSpaceId: string, appName?: string, windowTitle?: string) => {
|
||||
abortRef.current?.abort();
|
||||
const controller = new AbortController();
|
||||
abortRef.current = controller;
|
||||
|
||||
setIsLoading(true);
|
||||
setOptions([]);
|
||||
setError(null);
|
||||
setSteps([]);
|
||||
setExpandedOption(null);
|
||||
|
||||
let token = getBearerToken();
|
||||
if (!token) {
|
||||
await ensureTokensFromElectron();
|
||||
token = getBearerToken();
|
||||
}
|
||||
if (!token) {
|
||||
setError(friendlyError("not authenticated"));
|
||||
setIsLoading(false);
|
||||
return;
|
||||
}
|
||||
|
||||
const backendUrl = process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL || "http://localhost:8000";
|
||||
|
||||
try {
|
||||
const response = await fetch(`${backendUrl}/api/v1/autocomplete/vision/stream`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${token}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
screenshot,
|
||||
search_space_id: parseInt(searchSpaceId, 10),
|
||||
app_name: appName || "",
|
||||
window_title: windowTitle || "",
|
||||
}),
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
setError(friendlyError(response.status));
|
||||
setIsLoading(false);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!response.body) {
|
||||
setError(friendlyError("network error"));
|
||||
setIsLoading(false);
|
||||
return;
|
||||
}
|
||||
|
||||
const reader = response.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let buffer = "";
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
|
||||
buffer += decoder.decode(value, { stream: true });
|
||||
const events = buffer.split(/\r?\n\r?\n/);
|
||||
buffer = events.pop() || "";
|
||||
|
||||
for (const event of events) {
|
||||
const lines = event.split(/\r?\n/);
|
||||
for (const line of lines) {
|
||||
if (!line.startsWith("data: ")) continue;
|
||||
const data = line.slice(6).trim();
|
||||
if (!data || data === "[DONE]") continue;
|
||||
|
||||
try {
|
||||
const parsed: SSEEvent = JSON.parse(data);
|
||||
if (parsed.type === "data-suggestions") {
|
||||
setOptions(parsed.data.options);
|
||||
} else if (parsed.type === "error") {
|
||||
setError(friendlyError(parsed.errorText));
|
||||
} else if (parsed.type === "data-thinking-step") {
|
||||
const { id, title, status, items } = parsed.data;
|
||||
setSteps((prev) => {
|
||||
const existing = prev.findIndex((s) => s.id === id);
|
||||
if (existing >= 0) {
|
||||
const updated = [...prev];
|
||||
updated[existing] = { id, title, status, items };
|
||||
return updated;
|
||||
}
|
||||
return [...prev, { id, title, status, items }];
|
||||
});
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
if (err instanceof DOMException && err.name === "AbortError") return;
|
||||
setError(friendlyError("network error"));
|
||||
} finally {
|
||||
setIsLoading(false);
|
||||
}
|
||||
},
|
||||
[]
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
if (!api?.onAutocompleteContext) return;
|
||||
|
||||
const cleanup = api.onAutocompleteContext((data) => {
|
||||
const searchSpaceId = data.searchSpaceId || "1";
|
||||
if (data.screenshot) {
|
||||
fetchSuggestion(data.screenshot, searchSpaceId, data.appName, data.windowTitle);
|
||||
}
|
||||
});
|
||||
|
||||
return cleanup;
|
||||
}, [fetchSuggestion, api]);
|
||||
|
||||
if (!isDesktop) {
|
||||
return (
|
||||
<div className="suggestion-tooltip">
|
||||
<span className="suggestion-error-text">
|
||||
This page is only available in the SurfSense desktop app.
|
||||
</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (error) {
|
||||
if (error.isSetup) {
|
||||
return (
|
||||
<div className="suggestion-tooltip suggestion-setup">
|
||||
<div className="setup-icon">
|
||||
<svg viewBox="0 0 24 24" fill="none" width="28" height="28" aria-hidden="true">
|
||||
<path
|
||||
d="M1 12C1 12 5 4 12 4C19 4 23 12 23 12C23 12 19 20 12 20C5 20 1 12 1 12Z"
|
||||
stroke="#a78bfa"
|
||||
strokeWidth="1.5"
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
/>
|
||||
<circle
|
||||
cx="12"
|
||||
cy="12"
|
||||
r="3"
|
||||
stroke="#a78bfa"
|
||||
strokeWidth="1.5"
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
/>
|
||||
</svg>
|
||||
</div>
|
||||
<div className="setup-content">
|
||||
<span className="setup-title">Vision Model Required</span>
|
||||
<span className="setup-message">{error.message}</span>
|
||||
<span className="setup-hint">Settings → Vision Models</span>
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
className="setup-dismiss"
|
||||
onClick={() => api?.dismissSuggestion?.()}
|
||||
>
|
||||
✕
|
||||
</button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
return (
|
||||
<div className="suggestion-tooltip suggestion-error">
|
||||
<span className="suggestion-error-text">{error.message}</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const showLoading = isLoading && options.length === 0;
|
||||
|
||||
if (showLoading) {
|
||||
return (
|
||||
<div className="suggestion-tooltip">
|
||||
<div className="agent-activity">
|
||||
{steps.length === 0 && (
|
||||
<div className="activity-initial">
|
||||
<span className="step-spinner" />
|
||||
<span className="activity-label">Preparing…</span>
|
||||
</div>
|
||||
)}
|
||||
{steps.length > 0 && (
|
||||
<div className="activity-steps">
|
||||
{steps.map((step) => (
|
||||
<div key={step.id} className="activity-step">
|
||||
<StepIcon status={step.status} />
|
||||
<span className="step-label">
|
||||
{step.title}
|
||||
{step.items.length > 0 && (
|
||||
<span className="step-detail"> · {step.items[0]}</span>
|
||||
)}
|
||||
</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const handleSelect = (text: string) => {
|
||||
api?.acceptSuggestion?.(text);
|
||||
};
|
||||
|
||||
const handleDismiss = () => {
|
||||
api?.dismissSuggestion?.();
|
||||
};
|
||||
|
||||
const TRUNCATE_LENGTH = 120;
|
||||
|
||||
if (options.length === 0) {
|
||||
return (
|
||||
<div className="suggestion-tooltip suggestion-error">
|
||||
<span className="suggestion-error-text">No suggestions available.</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="suggestion-tooltip">
|
||||
<div className="suggestion-options">
|
||||
{options.map((option, index) => {
|
||||
const isExpanded = expandedOption === index;
|
||||
const needsTruncation = option.length > TRUNCATE_LENGTH;
|
||||
const displayText =
|
||||
needsTruncation && !isExpanded ? option.slice(0, TRUNCATE_LENGTH) + "…" : option;
|
||||
|
||||
return (
|
||||
<button
|
||||
type="button"
|
||||
key={index}
|
||||
className="suggestion-option"
|
||||
onClick={() => handleSelect(option)}
|
||||
>
|
||||
<span className="option-number">{index + 1}</span>
|
||||
<span className="option-text">{displayText}</span>
|
||||
{needsTruncation && (
|
||||
<button
|
||||
type="button"
|
||||
className="option-expand"
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
setExpandedOption(isExpanded ? null : index);
|
||||
}}
|
||||
>
|
||||
{isExpanded ? "less" : "more"}
|
||||
</button>
|
||||
)}
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
<div className="suggestion-actions">
|
||||
<button
|
||||
type="button"
|
||||
className="suggestion-btn suggestion-btn-dismiss"
|
||||
onClick={handleDismiss}
|
||||
>
|
||||
Dismiss
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -1,352 +0,0 @@
|
|||
html:has(.suggestion-body),
|
||||
body:has(.suggestion-body) {
|
||||
margin: 0 !important;
|
||||
padding: 0 !important;
|
||||
background: transparent !important;
|
||||
overflow: hidden !important;
|
||||
height: auto !important;
|
||||
width: 100% !important;
|
||||
}
|
||||
|
||||
.suggestion-body {
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
background: transparent;
|
||||
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
||||
-webkit-font-smoothing: antialiased;
|
||||
user-select: none;
|
||||
-webkit-app-region: no-drag;
|
||||
}
|
||||
|
||||
.suggestion-tooltip {
|
||||
box-sizing: border-box;
|
||||
background: #1e1e1e;
|
||||
border: 1px solid #3c3c3c;
|
||||
border-radius: 8px;
|
||||
padding: 8px 12px;
|
||||
margin: 4px;
|
||||
max-width: 400px;
|
||||
/* MAX_HEIGHT in suggestion-window.ts is 400px. Subtract 8px for margin
|
||||
(4px * 2) so the tooltip + margin fits within the Electron window.
|
||||
box-sizing: border-box ensures padding + border are included. */
|
||||
max-height: 392px;
|
||||
box-shadow: 0 4px 16px rgba(0, 0, 0, 0.5);
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.suggestion-text {
|
||||
color: #d4d4d4;
|
||||
font-size: 13px;
|
||||
line-height: 1.45;
|
||||
margin: 0 0 6px 0;
|
||||
word-wrap: break-word;
|
||||
white-space: pre-wrap;
|
||||
overflow-y: auto;
|
||||
flex: 1 1 auto;
|
||||
min-height: 0;
|
||||
}
|
||||
|
||||
.suggestion-text::-webkit-scrollbar {
|
||||
width: 5px;
|
||||
}
|
||||
|
||||
.suggestion-text::-webkit-scrollbar-track {
|
||||
background: transparent;
|
||||
}
|
||||
|
||||
.suggestion-text::-webkit-scrollbar-thumb {
|
||||
background: #555;
|
||||
border-radius: 3px;
|
||||
}
|
||||
|
||||
.suggestion-text::-webkit-scrollbar-thumb:hover {
|
||||
background: #777;
|
||||
}
|
||||
|
||||
.suggestion-actions {
|
||||
display: flex;
|
||||
justify-content: flex-end;
|
||||
gap: 4px;
|
||||
border-top: 1px solid #2a2a2a;
|
||||
padding-top: 6px;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.suggestion-btn {
|
||||
padding: 2px 8px;
|
||||
border-radius: 3px;
|
||||
border: 1px solid #3c3c3c;
|
||||
font-family: inherit;
|
||||
font-size: 10px;
|
||||
font-weight: 500;
|
||||
cursor: pointer;
|
||||
line-height: 16px;
|
||||
transition:
|
||||
background 0.15s,
|
||||
border-color 0.15s;
|
||||
}
|
||||
|
||||
.suggestion-btn-accept {
|
||||
background: #2563eb;
|
||||
border-color: #3b82f6;
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.suggestion-btn-accept:hover {
|
||||
background: #1d4ed8;
|
||||
}
|
||||
|
||||
.suggestion-btn-dismiss {
|
||||
background: #2a2a2a;
|
||||
color: #999;
|
||||
}
|
||||
|
||||
.suggestion-btn-dismiss:hover {
|
||||
background: #333;
|
||||
color: #ccc;
|
||||
}
|
||||
|
||||
.suggestion-error {
|
||||
border-color: #5c2626;
|
||||
}
|
||||
|
||||
.suggestion-error-text {
|
||||
color: #f48771;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
/* --- Setup prompt (vision model not configured) --- */
|
||||
|
||||
.suggestion-setup {
|
||||
display: flex;
|
||||
flex-direction: row;
|
||||
align-items: flex-start;
|
||||
gap: 10px;
|
||||
border-color: #3b2d6b;
|
||||
padding: 10px 14px;
|
||||
}
|
||||
|
||||
.setup-icon {
|
||||
flex-shrink: 0;
|
||||
margin-top: 1px;
|
||||
}
|
||||
|
||||
.setup-content {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 3px;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.setup-title {
|
||||
font-size: 13px;
|
||||
font-weight: 600;
|
||||
color: #c4b5fd;
|
||||
}
|
||||
|
||||
.setup-message {
|
||||
font-size: 11.5px;
|
||||
color: #a1a1aa;
|
||||
line-height: 1.4;
|
||||
}
|
||||
|
||||
.setup-hint {
|
||||
font-size: 10.5px;
|
||||
color: #7c6dac;
|
||||
margin-top: 2px;
|
||||
}
|
||||
|
||||
.setup-dismiss {
|
||||
flex-shrink: 0;
|
||||
align-self: flex-start;
|
||||
background: none;
|
||||
border: none;
|
||||
color: #6b6b7b;
|
||||
font-size: 14px;
|
||||
cursor: pointer;
|
||||
padding: 2px 4px;
|
||||
line-height: 1;
|
||||
border-radius: 4px;
|
||||
transition:
|
||||
color 0.15s,
|
||||
background 0.15s;
|
||||
}
|
||||
|
||||
.setup-dismiss:hover {
|
||||
color: #c4b5fd;
|
||||
background: rgba(124, 109, 172, 0.15);
|
||||
}
|
||||
|
||||
/* --- Agent activity indicator --- */
|
||||
|
||||
.agent-activity {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 4px;
|
||||
overflow-y: auto;
|
||||
max-height: 340px;
|
||||
}
|
||||
|
||||
.agent-activity::-webkit-scrollbar {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.activity-initial {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
padding: 2px 0;
|
||||
}
|
||||
|
||||
.activity-label {
|
||||
color: #a1a1aa;
|
||||
font-size: 12px;
|
||||
white-space: nowrap;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
|
||||
.activity-steps {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 3px;
|
||||
}
|
||||
|
||||
.activity-step {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
min-height: 18px;
|
||||
}
|
||||
|
||||
.step-label {
|
||||
color: #d4d4d4;
|
||||
font-size: 12px;
|
||||
white-space: nowrap;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
|
||||
.step-detail {
|
||||
color: #71717a;
|
||||
font-size: 11px;
|
||||
}
|
||||
|
||||
/* Spinner (in_progress) */
|
||||
.step-spinner {
|
||||
width: 14px;
|
||||
height: 14px;
|
||||
flex-shrink: 0;
|
||||
border: 1.5px solid #3f3f46;
|
||||
border-top-color: #a78bfa;
|
||||
border-radius: 50%;
|
||||
animation: step-spin 0.7s linear infinite;
|
||||
}
|
||||
|
||||
/* Checkmark icon (complete) */
|
||||
.step-icon {
|
||||
width: 14px;
|
||||
height: 14px;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
@keyframes step-spin {
|
||||
to {
|
||||
transform: rotate(360deg);
|
||||
}
|
||||
}
|
||||
|
||||
/* --- Suggestion option cards --- */
|
||||
|
||||
.suggestion-options {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 4px;
|
||||
overflow-y: auto;
|
||||
flex: 1 1 auto;
|
||||
min-height: 0;
|
||||
margin-bottom: 6px;
|
||||
}
|
||||
|
||||
.suggestion-options::-webkit-scrollbar {
|
||||
width: 5px;
|
||||
}
|
||||
|
||||
.suggestion-options::-webkit-scrollbar-track {
|
||||
background: transparent;
|
||||
}
|
||||
|
||||
.suggestion-options::-webkit-scrollbar-thumb {
|
||||
background: #555;
|
||||
border-radius: 3px;
|
||||
}
|
||||
|
||||
.suggestion-option {
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
gap: 8px;
|
||||
padding: 6px 8px;
|
||||
border-radius: 5px;
|
||||
border: 1px solid #333;
|
||||
background: #262626;
|
||||
cursor: pointer;
|
||||
text-align: left;
|
||||
font-family: inherit;
|
||||
transition:
|
||||
background 0.15s,
|
||||
border-color 0.15s;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.suggestion-option:hover {
|
||||
background: #2a2d3a;
|
||||
border-color: #3b82f6;
|
||||
}
|
||||
|
||||
.option-number {
|
||||
flex-shrink: 0;
|
||||
width: 18px;
|
||||
height: 18px;
|
||||
border-radius: 50%;
|
||||
background: #3f3f46;
|
||||
color: #d4d4d4;
|
||||
font-size: 10px;
|
||||
font-weight: 600;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
margin-top: 1px;
|
||||
}
|
||||
|
||||
.suggestion-option:hover .option-number {
|
||||
background: #2563eb;
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.option-text {
|
||||
color: #d4d4d4;
|
||||
font-size: 12px;
|
||||
line-height: 1.45;
|
||||
word-wrap: break-word;
|
||||
white-space: pre-wrap;
|
||||
flex: 1 1 auto;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.option-expand {
|
||||
flex-shrink: 0;
|
||||
background: none;
|
||||
border: none;
|
||||
color: #71717a;
|
||||
font-size: 10px;
|
||||
cursor: pointer;
|
||||
padding: 0 2px;
|
||||
font-family: inherit;
|
||||
margin-top: 1px;
|
||||
}
|
||||
|
||||
.option-expand:hover {
|
||||
color: #a1a1aa;
|
||||
}
|
||||
3
surfsense_web/atoms/chat/pending-user-images.atom.ts
Normal file
3
surfsense_web/atoms/chat/pending-user-images.atom.ts
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
import { atom } from "jotai";
|
||||
|
||||
export const pendingUserImageDataUrlsAtom = atom<string[]>([]);
|
||||
|
|
@ -12,6 +12,7 @@ import {
|
|||
AlertCircle,
|
||||
ArrowDownIcon,
|
||||
ArrowUpIcon,
|
||||
Camera,
|
||||
ChevronDown,
|
||||
ChevronUp,
|
||||
Clipboard,
|
||||
|
|
@ -40,6 +41,7 @@ import {
|
|||
mentionedDocumentsAtom,
|
||||
sidebarSelectedDocumentsAtom,
|
||||
} from "@/atoms/chat/mentioned-documents.atom";
|
||||
import { pendingUserImageDataUrlsAtom } from "@/atoms/chat/pending-user-images.atom";
|
||||
import { connectorDialogOpenAtom } from "@/atoms/connector-dialog/connector-dialog.atoms";
|
||||
import { connectorsAtom } from "@/atoms/connectors/connector-query.atoms";
|
||||
import { documentsSidebarOpenAtom } from "@/atoms/documents/ui.atoms";
|
||||
|
|
@ -89,6 +91,7 @@ import { useBatchCommentsPreload } from "@/hooks/use-comments";
|
|||
import { useCommentsSync } from "@/hooks/use-comments-sync";
|
||||
import { useMediaQuery } from "@/hooks/use-media-query";
|
||||
import { useElectronAPI } from "@/hooks/use-platform";
|
||||
import { captureDisplayToPngDataUrl } from "@/lib/chat/display-media-capture";
|
||||
import { SLIDEOUT_PANEL_OPENED_EVENT } from "@/lib/layout-events";
|
||||
import { cn } from "@/lib/utils";
|
||||
|
||||
|
|
@ -295,6 +298,32 @@ const ConnectToolsBanner: FC<{ isThreadEmpty: boolean }> = ({ isThreadEmpty }) =
|
|||
);
|
||||
};
|
||||
|
||||
const PendingScreenImageStrip: FC = () => {
|
||||
const [urls, setUrls] = useAtom(pendingUserImageDataUrlsAtom);
|
||||
if (urls.length === 0) return null;
|
||||
return (
|
||||
<div className="mx-3 mt-2 flex flex-wrap gap-2">
|
||||
{urls.map((url, index) => (
|
||||
<div
|
||||
key={url}
|
||||
className="group relative h-14 w-14 shrink-0 overflow-hidden rounded-md border border-border/50 bg-muted"
|
||||
>
|
||||
{/* biome-ignore lint/performance/noImgElement: data URL thumbnails from capture */}
|
||||
<img src={url} alt="" className="size-full object-cover" draggable={false} />
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setUrls((prev) => prev.filter((_, i) => i !== index))}
|
||||
className="absolute right-0.5 top-0.5 flex size-5 items-center justify-center rounded-full bg-background/90 text-muted-foreground shadow-sm transition-opacity hover:text-foreground sm:opacity-0 sm:group-hover:opacity-100"
|
||||
aria-label="Remove screenshot"
|
||||
>
|
||||
<X className="size-3" />
|
||||
</button>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
const ClipboardChip: FC<{ text: string; onDismiss: () => void }> = ({ text, onDismiss }) => {
|
||||
const [expanded, setExpanded] = useState(false);
|
||||
const isLong = text.length > 120;
|
||||
|
|
@ -702,6 +731,7 @@ const Composer: FC = () => {
|
|||
</div>
|
||||
)}
|
||||
<div className="aui-composer-attachment-dropzone flex w-full flex-col overflow-hidden rounded-2xl border-input bg-muted pt-2 outline-none transition-shadow">
|
||||
<PendingScreenImageStrip />
|
||||
{clipboardInitialText && (
|
||||
<ClipboardChip
|
||||
text={clipboardInitialText}
|
||||
|
|
@ -761,11 +791,23 @@ const ComposerAction: FC<ComposerActionProps> = ({ isBlockedByOtherUser = false
|
|||
},
|
||||
[]
|
||||
);
|
||||
const pendingScreenImages = useAtomValue(pendingUserImageDataUrlsAtom);
|
||||
const setPendingScreenImages = useSetAtom(pendingUserImageDataUrlsAtom);
|
||||
const electronAPI = useElectronAPI();
|
||||
|
||||
const isComposerTextEmpty = useAuiState(({ composer }) => {
|
||||
const text = composer.text?.trim() || "";
|
||||
return text.length === 0;
|
||||
});
|
||||
const isComposerEmpty = isComposerTextEmpty && mentionedDocuments.length === 0;
|
||||
const isComposerEmpty =
|
||||
isComposerTextEmpty && mentionedDocuments.length === 0 && pendingScreenImages.length === 0;
|
||||
|
||||
const handleScreenCapture = useCallback(async () => {
|
||||
const url = electronAPI?.captureFullScreen
|
||||
? await electronAPI.captureFullScreen()
|
||||
: await captureDisplayToPngDataUrl();
|
||||
if (url) setPendingScreenImages((prev) => [...prev, url]);
|
||||
}, [electronAPI, setPendingScreenImages]);
|
||||
|
||||
const { data: userConfigs } = useAtomValue(newLLMConfigsAtom);
|
||||
const { data: globalConfigs } = useAtomValue(globalNewLLMConfigsAtom);
|
||||
|
|
@ -1201,6 +1243,17 @@ const ComposerAction: FC<ComposerActionProps> = ({ isBlockedByOtherUser = false
|
|||
</div>
|
||||
)}
|
||||
<div className="flex items-center gap-2">
|
||||
<TooltipIconButton
|
||||
tooltip="Capture screen"
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
className="size-8 rounded-full"
|
||||
aria-label="Capture screen"
|
||||
onClick={() => void handleScreenCapture()}
|
||||
>
|
||||
<Camera className="size-4" />
|
||||
</TooltipIconButton>
|
||||
<AuiIf condition={({ thread }) => !thread.isRunning}>
|
||||
<ComposerPrimitive.Send asChild disabled={isSendDisabled}>
|
||||
<TooltipIconButton
|
||||
|
|
@ -1210,7 +1263,7 @@ const ComposerAction: FC<ComposerActionProps> = ({ isBlockedByOtherUser = false
|
|||
: !hasModelConfigured
|
||||
? "Please select a model from the header to start chatting"
|
||||
: isComposerEmpty
|
||||
? "Enter a message to send"
|
||||
? "Enter a message or add a screenshot to send"
|
||||
: "Send message"
|
||||
}
|
||||
side="bottom"
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ export function acceleratorToDisplay(accel: string): string[] {
|
|||
export const DEFAULT_SHORTCUTS = {
|
||||
generalAssist: "CommandOrControl+Shift+S",
|
||||
quickAsk: "CommandOrControl+Alt+S",
|
||||
autocomplete: "CommandOrControl+Shift+Space",
|
||||
screenshotAssist: "CommandOrControl+Shift+Space",
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -63,10 +63,10 @@ const TAB_ITEMS = [
|
|||
featured: true,
|
||||
},
|
||||
{
|
||||
title: "Extreme Assist",
|
||||
title: "Screenshot Assist",
|
||||
description:
|
||||
"Get inline writing suggestions powered by your knowledge base as you type in any app.",
|
||||
src: "/homepage/hero_tutorial/extreme_assist.mp4",
|
||||
"Use a global shortcut to select a region on your screen and attach it to your chat message.",
|
||||
src: "/homepage/hero_tutorial/screenshot_assist.mp4",
|
||||
featured: true,
|
||||
},
|
||||
{
|
||||
|
|
|
|||
|
|
@ -67,6 +67,9 @@ const DesktopShortcutsContent = dynamic(
|
|||
import(
|
||||
"@/app/dashboard/[search_space_id]/user-settings/components/DesktopShortcutsContent"
|
||||
).then((m) => ({ default: m.DesktopShortcutsContent })),
|
||||
import(
|
||||
"@/app/dashboard/[search_space_id]/user-settings/components/DesktopShortcutsContent"
|
||||
).then((m) => ({ default: m.DesktopShortcutsContent })),
|
||||
{ ssr: false }
|
||||
);
|
||||
const MemoryContent = dynamic(
|
||||
|
|
|
|||
121
surfsense_web/content/docs/connectors/baidu-search.mdx
Normal file
121
surfsense_web/content/docs/connectors/baidu-search.mdx
Normal file
|
|
@ -0,0 +1,121 @@
|
|||
---
|
||||
title: Baidu Search
|
||||
description: Search the Chinese web with Baidu AI Search in SurfSense
|
||||
---
|
||||
|
||||
# Baidu Search Integration Setup Guide
|
||||
|
||||
This guide walks you through connecting Baidu AI Search to SurfSense for Chinese web search and AI-powered research.
|
||||
|
||||
## How it works
|
||||
|
||||
The Baidu Search connector uses Baidu AI Search through Qianfan AppBuilder's intelligent search generation API. It is a live search connector: SurfSense queries Baidu when the assistant needs current web results instead of periodically indexing content into your knowledge base.
|
||||
|
||||
- Baidu Search is best for Simplified Chinese queries and China-focused web content.
|
||||
- Results are merged with SurfSense's other configured web search engines.
|
||||
- The connector returns Baidu references as sources that can be cited in chat responses.
|
||||
|
||||
---
|
||||
|
||||
## Authorization
|
||||
|
||||
<Callout type="info" title="API Key Required">
|
||||
You need a Baidu Qianfan AppBuilder API key to use this connector. The key is encrypted and stored securely by SurfSense.
|
||||
</Callout>
|
||||
|
||||
### Step 1: Get Your Baidu AI Search API Key
|
||||
|
||||
1. Open the [Baidu AI Search product page](https://cloud.baidu.com/product/ai-search.html) and sign in with your Baidu Cloud account.
|
||||
2. Open Qianfan AppBuilder or the AI Search console from Baidu Cloud.
|
||||
3. Create or select an application that has access to Baidu AI Search.
|
||||
4. Generate an API key for the application.
|
||||
5. Copy the API key. SurfSense uses it as the `BAIDU_API_KEY` connector setting.
|
||||
|
||||
<Callout type="warn">
|
||||
Keep this key private. Do not paste it into chat messages, issue reports, screenshots, or public repositories.
|
||||
</Callout>
|
||||
|
||||
---
|
||||
|
||||
## Connecting to SurfSense
|
||||
|
||||
1. Navigate to **Connectors** → **Add Connector** → **Baidu Search**.
|
||||
2. Fill in the required fields:
|
||||
|
||||
| Field | Description | Example |
|
||||
|-------|-------------|---------|
|
||||
| **Connector Name** | A friendly name to identify this connector | `Baidu Search` |
|
||||
| **Baidu AppBuilder API Key** | Your Qianfan AppBuilder API key | `bce-v3/...` |
|
||||
|
||||
3. Click **Connect** to save the connector.
|
||||
4. Ask a current Chinese web query in chat, such as `今天中国人工智能行业有什么重要新闻？`.
|
||||
|
||||
### Optional Advanced Settings
|
||||
|
||||
SurfSense stores advanced Baidu options in the connector config. If your deployment exposes these fields, use the following values:
|
||||
|
||||
| Setting | Description | Default |
|
||||
|---------|-------------|---------|
|
||||
| `BAIDU_MODEL` | The model Baidu AI Search uses for answer generation | `ernie-3.5-8k` |
|
||||
| `BAIDU_SEARCH_SOURCE` | Baidu search source version | `baidu_search_v2` |
|
||||
| `BAIDU_ENABLE_DEEP_SEARCH` | Enables Baidu's deeper search mode when supported by your account | `false` |
|
||||
|
||||
SurfSense calls Baidu's intelligent search generation endpoint:
|
||||
|
||||
```text
|
||||
POST https://qianfan.baidubce.com/v2/ai_search/chat/completions
|
||||
```
|
||||
|
||||
For request and response details, see Baidu's [intelligent search generation API documentation](https://cloud.baidu.com/doc/qianfan/s/Omh4su4s0).
|
||||
|
||||
---
|
||||
|
||||
## When to Use Baidu Search
|
||||
|
||||
| Use Case | Why Baidu Search Helps |
|
||||
|----------|------------------------|
|
||||
| Chinese news and current events | Better coverage for China-focused sources |
|
||||
| Chinese company, product, or policy research | More local web results than global search engines alone |
|
||||
| Mandarin-language fact finding | Native Chinese search and summarization behavior |
|
||||
| Cross-checking web search | Adds another source alongside SearXNG, Tavily, or Linkup |
|
||||
|
||||
<Callout type="info" title="Live Search Connector">
|
||||
Baidu Search does not create indexed documents in your knowledge base. It runs when the assistant calls web search, then returns live sources for that answer.
|
||||
</Callout>
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
**No Baidu results appear**
|
||||
|
||||
- Confirm the Baidu Search connector is active in the current search space.
|
||||
- Try a Chinese query with clear search intent, for example `百度智能云千帆 AppBuilder 最新功能`.
|
||||
- Check whether other web search engines are returning results. If none are, review the general [Web Search](/docs/how-to/web-search) setup.
|
||||
|
||||
**Authentication failed**
|
||||
|
||||
- Verify that the API key was copied from Qianfan AppBuilder, not another Baidu Cloud product.
|
||||
- Regenerate the API key if it was rotated, expired, or copied with extra whitespace.
|
||||
- Make sure the related application has access to Baidu AI Search.
|
||||
|
||||
**Requests time out**
|
||||
|
||||
- Baidu AI Search can take longer than ordinary keyword search because it performs search and summarization.
|
||||
- Retry with a narrower query.
|
||||
- If you self-host SurfSense, verify that the backend container can reach `qianfan.baidubce.com`.
|
||||
|
||||
**Results are not relevant**
|
||||
|
||||
- Use Chinese keywords for China-focused topics.
|
||||
- Include entity names, dates, or locations in the query.
|
||||
- Compare with SearXNG or another configured live search connector for broader coverage.
|
||||
|
||||
---
|
||||
|
||||
## Verification Checklist
|
||||
|
||||
- The Baidu Search connector appears in your connector list.
|
||||
- A Chinese current-events query triggers web search in chat.
|
||||
- Chat responses include Baidu-backed sources with titles and URLs.
|
||||
- Invalid API keys fail without breaking other configured search engines.
|
||||
|
|
@ -83,6 +83,11 @@ Connect SurfSense to your favorite tools and services. Browse the available inte
|
|||
description="Connect your GitHub repositories to SurfSense"
|
||||
href="/docs/connectors/github"
|
||||
/>
|
||||
<Card
|
||||
title="Baidu Search"
|
||||
description="Search the Chinese web with Baidu AI Search"
|
||||
href="/docs/connectors/baidu-search"
|
||||
/>
|
||||
<Card
|
||||
title="Luma"
|
||||
description="Connect your Luma events to SurfSense"
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@
|
|||
"airtable",
|
||||
"clickup",
|
||||
"github",
|
||||
"baidu-search",
|
||||
"luma",
|
||||
"circleback",
|
||||
"elasticsearch",
|
||||
|
|
|
|||
|
|
@ -7,6 +7,8 @@ description: How SurfSense web search works and how to configure it for producti
|
|||
|
||||
SurfSense uses [SearXNG](https://docs.searxng.org/) as a bundled meta-search engine to provide web search across all search spaces. SearXNG aggregates results from multiple search engines (Google, DuckDuckGo, Brave, Bing, and more) without requiring any API keys.
|
||||
|
||||
You can also add live search connectors such as Baidu Search, Tavily, and Linkup to a search space. When those connectors are active, SurfSense queries them in parallel with SearXNG and merges the results before passing sources to the assistant.
|
||||
|
||||
## How It Works
|
||||
|
||||
When a user triggers a web search in SurfSense:
|
||||
|
|
@ -14,10 +16,25 @@ When a user triggers a web search in SurfSense:
|
|||
1. The backend sends a query to the bundled SearXNG instance via its JSON API
|
||||
2. SearXNG fans out the query to all enabled search engines simultaneously
|
||||
3. Results are aggregated, deduplicated, and ranked by engine weight
|
||||
4. The backend receives merged results and presents them to the user
|
||||
4. If the current search space has live search connectors, the backend queries them in parallel
|
||||
5. The backend deduplicates the merged results and presents them to the user
|
||||
|
||||
SearXNG runs as a Docker container alongside the backend. It is never exposed to the internet. Only the backend communicates with it over the internal Docker network.
|
||||
|
||||
## Live Search Connectors
|
||||
|
||||
Live search connectors are optional API-backed search providers configured per search space. They are useful when you need a specialized index, authenticated search API, or stronger regional coverage.
|
||||
|
||||
| Connector | Best For | Setup |
|
||||
|-----------|----------|-------|
|
||||
| Baidu Search | Chinese web search and China-focused current information | [Baidu Search connector](/docs/connectors/baidu-search) |
|
||||
| Tavily | General web research through Tavily's search API | Add the Tavily connector from the Connectors dashboard |
|
||||
| Linkup | General web search through Linkup's search API | Add the Linkup connector from the Connectors dashboard |
|
||||
|
||||
<Callout type="info" title="Search Space Scoped">
|
||||
Live search connectors only run for the search space where they are configured. They do not replace SearXNG globally.
|
||||
</Callout>
|
||||
|
||||
## Docker Setup
|
||||
|
||||
SearXNG is included in both `docker-compose.yml` and `docker-compose.dev.yml` and works out of the box with no configuration needed.
|
||||
|
|
|
|||
120
surfsense_web/lib/chat/display-media-capture.ts
Normal file
120
surfsense_web/lib/chat/display-media-capture.ts
Normal file
|
|
@ -0,0 +1,120 @@
|
|||
/** `getDisplayMedia` → single PNG frame (data URL). */
|
||||
function getImageCaptureCtor():
|
||||
| (new (
|
||||
track: MediaStreamTrack
|
||||
) => { grabFrame: () => Promise<ImageBitmap> })
|
||||
| undefined {
|
||||
if (typeof window === "undefined") return undefined;
|
||||
const IC = (
|
||||
window as unknown as {
|
||||
ImageCapture?: new (track: MediaStreamTrack) => { grabFrame: () => Promise<ImageBitmap> };
|
||||
}
|
||||
).ImageCapture;
|
||||
return typeof IC === "function" ? IC : undefined;
|
||||
}
|
||||
|
||||
function stopAllTracks(stream: MediaStream): void {
|
||||
for (const t of stream.getTracks()) {
|
||||
t.stop();
|
||||
}
|
||||
}
|
||||
|
||||
async function captureTrackToPngDataUrl(
|
||||
track: MediaStreamTrack,
|
||||
stream: MediaStream
|
||||
): Promise<string | null> {
|
||||
const ImageCtor = getImageCaptureCtor();
|
||||
if (ImageCtor !== undefined) {
|
||||
try {
|
||||
const ic = new ImageCtor(track);
|
||||
const bitmap = await ic.grabFrame();
|
||||
try {
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = bitmap.width;
|
||||
canvas.height = bitmap.height;
|
||||
const ctx = canvas.getContext("2d");
|
||||
if (!ctx) {
|
||||
stopAllTracks(stream);
|
||||
return null;
|
||||
}
|
||||
ctx.drawImage(bitmap, 0, 0);
|
||||
stopAllTracks(stream);
|
||||
return canvas.toDataURL("image/png");
|
||||
} finally {
|
||||
if ("close" in bitmap && typeof bitmap.close === "function") {
|
||||
bitmap.close();
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
/* fall through to <video> */
|
||||
}
|
||||
}
|
||||
|
||||
const videoEl = document.createElement("video");
|
||||
videoEl.srcObject = stream;
|
||||
videoEl.muted = true;
|
||||
const haveCurrentData = 2;
|
||||
const dataReady = new Promise<void>((resolve) => {
|
||||
if (videoEl.readyState >= haveCurrentData) {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
videoEl.addEventListener("loadeddata", () => resolve(), { once: true });
|
||||
});
|
||||
await videoEl.play();
|
||||
await Promise.race([
|
||||
dataReady,
|
||||
new Promise<void>((resolve) => {
|
||||
setTimeout(resolve, 500);
|
||||
}),
|
||||
]);
|
||||
const w = videoEl.videoWidth;
|
||||
const h = videoEl.videoHeight;
|
||||
if (!w || !h) {
|
||||
stopAllTracks(stream);
|
||||
return null;
|
||||
}
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = w;
|
||||
canvas.height = h;
|
||||
const ctx = canvas.getContext("2d");
|
||||
if (!ctx) {
|
||||
stopAllTracks(stream);
|
||||
return null;
|
||||
}
|
||||
ctx.drawImage(videoEl, 0, 0);
|
||||
stopAllTracks(stream);
|
||||
return canvas.toDataURL("image/png");
|
||||
}
|
||||
|
||||
export async function captureDisplayToPngDataUrl(): Promise<string | null> {
|
||||
if (typeof navigator === "undefined" || !navigator.mediaDevices?.getDisplayMedia) {
|
||||
return null;
|
||||
}
|
||||
let stream: MediaStream | null = null;
|
||||
try {
|
||||
stream = await navigator.mediaDevices.getDisplayMedia({
|
||||
video: { frameRate: { ideal: 1, max: 5 } },
|
||||
audio: false,
|
||||
selfBrowserSurface: "exclude",
|
||||
} as Parameters<MediaDevices["getDisplayMedia"]>[0]);
|
||||
|
||||
const track = stream.getVideoTracks()[0];
|
||||
if (!track) {
|
||||
stopAllTracks(stream);
|
||||
return null;
|
||||
}
|
||||
|
||||
const dataUrl = await captureTrackToPngDataUrl(track, stream);
|
||||
stream = null;
|
||||
return dataUrl;
|
||||
} catch (e) {
|
||||
if (typeof process !== "undefined" && process.env?.NODE_ENV !== "production") {
|
||||
console.warn("[captureDisplayToPngDataUrl]", e);
|
||||
}
|
||||
if (stream) {
|
||||
stopAllTracks(stream);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
56
surfsense_web/lib/chat/user-turn-api-parts.ts
Normal file
56
surfsense_web/lib/chat/user-turn-api-parts.ts
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
import type { AppendMessage } from "@assistant-ui/react";
|
||||
|
||||
const MAX_IMAGES = 4;
|
||||
|
||||
export type NewChatUserImagePayload = {
|
||||
media_type: "image/png" | "image/jpeg" | "image/webp";
|
||||
data: string;
|
||||
};
|
||||
|
||||
function dataUrlToPayload(dataUrl: string): NewChatUserImagePayload | null {
|
||||
const m = /^data:(image\/(?:png|jpeg|webp|jpg));base64,([\s\S]+)$/i.exec(dataUrl.trim());
|
||||
if (!m) return null;
|
||||
let media = m[1].toLowerCase() as string;
|
||||
if (media === "image/jpg") media = "image/jpeg";
|
||||
if (media !== "image/png" && media !== "image/jpeg" && media !== "image/webp") return null;
|
||||
const data = m[2].replace(/\s/g, "");
|
||||
if (!data) return null;
|
||||
return { media_type: media as NewChatUserImagePayload["media_type"], data };
|
||||
}
|
||||
|
||||
function collectImageDataUrlsFromParts(parts: AppendMessage["content"]): string[] {
|
||||
const out: string[] = [];
|
||||
for (const part of parts) {
|
||||
if (typeof part !== "object" || part === null || !("type" in part)) continue;
|
||||
if (part.type !== "image") continue;
|
||||
const img = "image" in part && typeof part.image === "string" ? part.image : null;
|
||||
if (img && dataUrlToPayload(img)) out.push(img);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export function extractUserTurnForNewChatApi(
|
||||
message: AppendMessage,
|
||||
extraDataUrls: readonly string[]
|
||||
): { userQuery: string; userImages: NewChatUserImagePayload[] } {
|
||||
let userQuery = "";
|
||||
for (const part of message.content) {
|
||||
if (part.type === "text") {
|
||||
userQuery += part.text;
|
||||
}
|
||||
}
|
||||
|
||||
const merged = [...extraDataUrls, ...collectImageDataUrlsFromParts(message.content)];
|
||||
const payloads: NewChatUserImagePayload[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const url of merged) {
|
||||
const p = dataUrlToPayload(url);
|
||||
if (!p) continue;
|
||||
if (seen.has(p.data)) continue;
|
||||
seen.add(p.data);
|
||||
payloads.push(p);
|
||||
if (payloads.length >= MAX_IMAGES) break;
|
||||
}
|
||||
|
||||
return { userQuery, userImages: payloads };
|
||||
}
|
||||
|
Before Width: | Height: | Size: 3.5 MiB After Width: | Height: | Size: 3.5 MiB |
29
surfsense_web/types/window.d.ts
vendored
29
surfsense_web/types/window.d.ts
vendored
|
|
@ -93,6 +93,7 @@ interface ElectronAPI {
|
|||
openExternal: (url: string) => void;
|
||||
getAppVersion: () => Promise<string>;
|
||||
onDeepLink: (callback: (url: string) => void) => () => void;
|
||||
onChatScreenCapture: (callback: (dataUrl: string) => void) => () => void;
|
||||
getQuickAskText: () => Promise<string>;
|
||||
setQuickAskMode: (mode: string) => Promise<void>;
|
||||
getQuickAskMode: () => Promise<string>;
|
||||
|
|
@ -104,20 +105,8 @@ interface ElectronAPI {
|
|||
}>;
|
||||
requestAccessibility: () => Promise<void>;
|
||||
requestScreenRecording: () => Promise<void>;
|
||||
captureFullScreen: () => Promise<string | null>;
|
||||
restartApp: () => Promise<void>;
|
||||
// Autocomplete
|
||||
onAutocompleteContext: (
|
||||
callback: (data: {
|
||||
screenshot: string;
|
||||
searchSpaceId?: string;
|
||||
appName?: string;
|
||||
windowTitle?: string;
|
||||
}) => void
|
||||
) => () => void;
|
||||
acceptSuggestion: (text: string) => Promise<void>;
|
||||
dismissSuggestion: () => Promise<void>;
|
||||
setAutocompleteEnabled: (enabled: boolean) => Promise<void>;
|
||||
getAutocompleteEnabled: () => Promise<boolean>;
|
||||
// Folder sync
|
||||
selectFolder: () => Promise<string | null>;
|
||||
addWatchedFolder: (config: WatchedFolderConfig) => Promise<WatchedFolderConfig[]>;
|
||||
|
|
@ -149,10 +138,18 @@ interface ElectronAPI {
|
|||
getAuthTokens: () => Promise<{ bearer: string; refresh: string } | null>;
|
||||
setAuthTokens: (bearer: string, refresh: string) => Promise<void>;
|
||||
// Keyboard shortcut configuration
|
||||
getShortcuts: () => Promise<{ generalAssist: string; quickAsk: string; autocomplete: string }>;
|
||||
getShortcuts: () => Promise<{
|
||||
generalAssist: string;
|
||||
quickAsk: string;
|
||||
screenshotAssist: string;
|
||||
}>;
|
||||
setShortcuts: (
|
||||
config: Partial<{ generalAssist: string; quickAsk: string; autocomplete: string }>
|
||||
) => Promise<{ generalAssist: string; quickAsk: string; autocomplete: string }>;
|
||||
config: Partial<{ generalAssist: string; quickAsk: string; screenshotAssist: string }>
|
||||
) => Promise<{
|
||||
generalAssist: string;
|
||||
quickAsk: string;
|
||||
screenshotAssist: string;
|
||||
}>;
|
||||
// Launch on system startup
|
||||
getAutoLaunch: () => Promise<{
|
||||
enabled: boolean;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue