diff --git a/.env.example b/.env.example index 8dca11ce6..894a989cc 100644 --- a/.env.example +++ b/.env.example @@ -11,6 +11,12 @@ NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE=LOCAL or GOOGLE (Default: LOCAL) NEXT_PUBLIC_ETL_SERVICE=UNSTRUCTURED or LLAMACLOUD or DOCLING (Default: DOCLING) # Backend Configuration BACKEND_PORT=8000 +# Auth type for backend login flow (Default: LOCAL) +# Set to GOOGLE if using Google OAuth +AUTH_TYPE=LOCAL +# Frontend URL used by backend for CORS allowed origins and OAuth redirects +# Must match the URL your browser uses to access the frontend +NEXT_FRONTEND_URL=http://localhost:3000 # Database Configuration POSTGRES_USER=postgres diff --git a/.vscode/launch.json b/.vscode/launch.json index ad7f04bd0..4988cc8f3 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -71,6 +71,7 @@ "app.celery_app:celery_app", "worker", "--loglevel=info", + "--queues=surfsense,surfsense.connectors,surfsense-dev,surfsense-dev.connectors", "--pool=solo" ], "console": "integratedTerminal", diff --git a/Dockerfile.allinone b/Dockerfile.allinone index 7f9f18806..e96618adc 100644 --- a/Dockerfile.allinone +++ b/Dockerfile.allinone @@ -50,6 +50,7 @@ ENV NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE=__NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYP ENV NEXT_PUBLIC_ETL_SERVICE=__NEXT_PUBLIC_ETL_SERVICE__ ENV NEXT_PUBLIC_ELECTRIC_URL=__NEXT_PUBLIC_ELECTRIC_URL__ ENV NEXT_PUBLIC_ELECTRIC_AUTH_MODE=__NEXT_PUBLIC_ELECTRIC_AUTH_MODE__ +ENV NEXT_PUBLIC_DEPLOYMENT_MODE=__NEXT_PUBLIC_DEPLOYMENT_MODE__ # Build RUN pnpm run build @@ -123,6 +124,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libpango-1.0-0 \ && rm -rf /var/lib/apt/lists/* +# Install Pandoc 3.x from GitHub (apt ships 2.9 which has broken table rendering). +RUN ARCH=$(dpkg --print-architecture) && \ + wget -qO /tmp/pandoc.deb "https://github.com/jgm/pandoc/releases/download/3.9/pandoc-3.9-1-${ARCH}.deb" && \ + dpkg -i /tmp/pandoc.deb && \ + rm /tmp/pandoc.deb + + # Install Node.js 20.x (for running frontend) RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \ && apt-get install -y nodejs \ diff --git a/README.es.md b/README.es.md index 86438eded..b3f2eb5c3 100644 --- a/README.es.md +++ b/README.es.md @@ -15,6 +15,9 @@ [English](README.md) | [Español](README.es.md) | [Português](README.pt-BR.md) | [हिन्दी](README.hi.md) | [简体中文](README.zh-CN.md) + +
+MODSetter%2FSurfSense | Trendshift
# SurfSense @@ -22,101 +25,122 @@ Conecta cualquier LLM a tus fuentes de conocimiento internas y chatea con él en SurfSense es un agente de investigación de IA altamente personalizable, conectado a fuentes externas como motores de búsqueda (SearxNG, Tavily, LinkUp), Google Drive, Slack, Microsoft Teams, Linear, Jira, ClickUp, Confluence, BookStack, Gmail, Notion, YouTube, GitHub, Discord, Airtable, Google Calendar, Luma, Circleback, Elasticsearch, Obsidian y más por venir. -
-MODSetter%2FSurfSense | Trendshift -
-# Video +# Video https://github.com/user-attachments/assets/cc0c84d3-1f2f-4f7a-b519-2ecce22310b1 - ## Ejemplo de Podcast https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7 +## Cómo usar SurfSense +### Cloud + +1. Ve a [surfsense.com](https://www.surfsense.com) e inicia sesión. + +

Login

+ +2. Conecta tus conectores y sincroniza. Activa la sincronización periódica para mantenerlos actualizados. + +

Conectores

+ +3. Mientras se indexan los datos de los conectores, sube documentos. + +

Subir Documentos

+ +4. Una vez que todo esté indexado, pregunta lo que quieras (Casos de uso): + + - Búsqueda básica y citaciones + +

Búsqueda y Citación

+ + - QNA con mención de documentos + +

QNA con Mención de Documentos

+ + - Generación de informes y exportaciones (PDF, DOCX por ahora) + +

Generación de Informes

+ + - Generación de podcasts + +

Generación de Podcasts

+ + - Generación de imágenes + +

Generación de Imágenes

+ + - Y más próximamente. + + +### Auto-Hospedado + +Ejecuta SurfSense en tu propia infraestructura para control total de datos y privacidad. + +**Inicio Rápido (Docker en un solo comando):** + +```bash +docker run -d -p 3000:3000 -p 8000:8000 -p 5133:5133 \ + -v surfsense-data:/data \ + --name surfsense \ + --restart unless-stopped \ + ghcr.io/modsetter/surfsense:latest +``` + +Después de iniciar, abre [http://localhost:3000](http://localhost:3000) en tu navegador. + +Para Docker Compose, instalación manual y otras opciones de despliegue, consulta la [documentación](https://www.surfsense.com/docs/). + +### Cómo Colaborar en Tiempo Real (Beta) + +1. Ve a la página de Gestión de Miembros y crea una invitación. + +

Invitar Miembros

+ +2. El compañero de equipo se une y ese SearchSpace se comparte. + +

Flujo de Unión por Invitación

+ +3. Haz el chat compartido. + +

Hacer Chat Compartido

+ +4. Tu equipo ahora puede chatear en tiempo real. + +

Chat en Tiempo Real

+ +5. Agrega comentarios para etiquetar a compañeros de equipo. + +

Comentarios en Tiempo Real

## Funcionalidades Principales -### 💡 **Idea**: -- Alternativa de código abierto a NotebookLM, Perplexity y Glean. Conecta cualquier LLM a tus fuentes de conocimiento internas y colabora con tu equipo en tiempo real. -### 📁 **Soporte de Múltiples Formatos de Archivo** -- Guarda contenido de tus archivos personales *(Documentos, imágenes, videos y soporta **más de 50 extensiones de archivo**)* en tu propia base de conocimiento personal. -### 🔍 **Búsqueda Potente** -- Investiga o encuentra rápidamente cualquier cosa en tu contenido guardado. -### 💬 **Chatea con tu Contenido Guardado** -- Interactúa en lenguaje natural y obtén respuestas con citas. -### 📄 **Respuestas con Citas** -- Obtén respuestas con citas como en Perplexity. -### 🧩 **Compatibilidad Universal** -- Conecta virtualmente cualquier proveedor de inferencia a través de la especificación OpenAI y LiteLLM. -### 🔔 **Privacidad y Soporte de LLM Local** -- Funciona perfectamente con LLMs locales como vLLM y Ollama. -### 🏠 **Auto-Hospedable** -- Código abierto y fácil de desplegar localmente. -### 👥 **Colaboración en Equipo con RBAC** -- Control de acceso basado en roles para los espacios de búsqueda -- Invita a miembros del equipo con roles personalizables (Propietario, Admin, Editor, Visor) -- Permisos granulares para documentos, chats, conectores y configuración -- Comparte bases de conocimiento de forma segura dentro de tu organización -- Los chats de equipo se actualizan en tiempo real y puedes "Chatear sobre el chat" en hilos de comentarios -### 🎙️ Podcasts -- Agente de generación de podcasts ultrarrápido. (Crea un podcast de 3 minutos en menos de 20 segundos.) -- Convierte tus conversaciones de chat en contenido de audio atractivo -- Soporte para proveedores TTS locales (Kokoro TTS) -- Soporte para múltiples proveedores TTS (OpenAI, Azure, Google Vertex AI) +| Funcionalidad | Descripción | +|----------------|-------------| +| Alternativa OSS | Reemplazo directo de NotebookLM, Perplexity y Glean con colaboración en equipo en tiempo real | +| 50+ Formatos de Archivo | Sube documentos, imágenes, videos vía LlamaCloud, Unstructured o Docling (local) | +| Búsqueda Híbrida | Semántica + Texto completo con Índices Jerárquicos y Reciprocal Rank Fusion | +| Respuestas con Citas | Chatea con tu base de conocimiento y obtén respuestas citadas al estilo Perplexity | +| Arquitectura de Agentes Profundos | Impulsado por [LangChain Deep Agents](https://docs.langchain.com/oss/python/deepagents/overview) con planificación, subagentes y acceso al sistema de archivos | +| Soporte Universal de LLM | 100+ LLMs, 6000+ modelos de embeddings, todos los principales rerankers vía OpenAI spec y LiteLLM | +| Privacidad Primero | Soporte completo de LLM local (vLLM, Ollama) tus datos son tuyos | +| Colaboración en Equipo | RBAC con roles de Propietario / Admin / Editor / Visor, chat en tiempo real e hilos de comentarios | +| Generación de Podcasts | Podcast de 3 min en menos de 20 segundos; múltiples proveedores TTS (OpenAI, Azure, Kokoro) | +| Extensión de Navegador | Extensión multi-navegador para guardar cualquier página web, incluyendo páginas protegidas por autenticación | +| 25+ Conectores | Motores de búsqueda, Google Drive, Slack, Teams, Jira, Notion, GitHub, Discord y [más](#fuentes-externas) | +| Auto-Hospedable | Código abierto, Docker en un solo comando o Docker Compose completo para producción | -### 🤖 **Arquitectura de Agentes Profundos** -- Impulsado por [LangChain Deep Agents](https://docs.langchain.com/oss/python/deepagents/overview) - agentes que pueden planificar, usar subagentes y aprovechar sistemas de archivos para tareas complejas. +
+Lista completa de Fuentes Externas + -### 📊 **Técnicas Avanzadas de RAG** -- Soporta más de 100 LLMs -- Soporta más de 6000 modelos de embeddings -- Soporta todos los principales rerankers (Pinecone, Cohere, Flashrank, etc.) -- Utiliza índices jerárquicos (configuración RAG de 2 niveles) -- Utiliza búsqueda híbrida (Semántica + Texto completo combinado con Reciprocal Rank Fusion) - -### ℹ️ **Fuentes Externas** -- Motores de búsqueda (Tavily, LinkUp) -- SearxNG (instancias auto-hospedadas) -- Google Drive -- Slack -- Microsoft Teams -- Linear -- Jira -- ClickUp -- Confluence -- BookStack -- Notion -- Gmail -- Videos de YouTube -- GitHub -- Discord -- Airtable -- Google Calendar -- Luma -- Circleback -- Elasticsearch -- Obsidian -- y más por venir..... - -## 📄 **Extensiones de Archivo Soportadas** - -| Servicio ETL | Formatos | Notas | -|--------------|----------|-------| -| **LlamaCloud** | 50+ formatos | Documentos, presentaciones, hojas de cálculo, imágenes | -| **Unstructured** | 34+ formatos | Formatos principales + soporte de email | -| **Docling** | Formatos principales | Procesamiento local, no requiere clave API | - -**Audio/Video** (vía servicio STT): `.mp3`, `.wav`, `.mp4`, `.webm`, etc. - -### 🔖 Extensión Multi-Navegador -- La extensión de SurfSense se puede usar para guardar cualquier página web que desees. -- Su principal uso es guardar páginas web protegidas por autenticación. +Motores de Búsqueda (Tavily, LinkUp) · SearxNG · Google Drive · Slack · Microsoft Teams · Linear · Jira · ClickUp · Confluence · BookStack · Notion · Gmail · Videos de YouTube · GitHub · Discord · Airtable · Google Calendar · Luma · Circleback · Elasticsearch · Obsidian, y más por venir. +
## SOLICITUDES DE FUNCIONES Y FUTURO @@ -126,119 +150,25 @@ https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7 ¡Únete al [Discord de SurfSense](https://discord.gg/ejRNvftDp9) y ayuda a dar forma al futuro de SurfSense! -## 🚀 Hoja de Ruta +## Hoja de Ruta ¡Mantente al día con nuestro progreso de desarrollo y próximas funcionalidades! Consulta nuestra hoja de ruta pública y contribuye con tus ideas o comentarios: -**📋 Discusión de la Hoja de Ruta:** [SurfSense 2025-2026 Roadmap: Deep Agents, Real-Time Collaboration & MCP Servers](https://github.com/MODSetter/SurfSense/discussions/565) +**Discusión de la Hoja de Ruta:** [SurfSense 2026 Roadmap](https://github.com/MODSetter/SurfSense/discussions/565) -**📊 Tablero Kanban:** [SurfSense Project Board](https://github.com/users/MODSetter/projects/3) - - -## ¿Cómo empezar? - -### Inicio Rápido con Docker 🐳 - -> [!TIP] -> Para despliegues en producción, usa la configuración completa de [Docker Compose](https://www.surfsense.com/docs/docker-installation) que ofrece más control y escalabilidad. - -**Linux/macOS:** - -```bash -docker run -d -p 3000:3000 -p 8000:8000 -p 5133:5133 \ - -v surfsense-data:/data \ - --name surfsense \ - --restart unless-stopped \ - ghcr.io/modsetter/surfsense:latest -``` - -**Windows (PowerShell):** - -```powershell -docker run -d -p 3000:3000 -p 8000:8000 -p 5133:5133 ` - -v surfsense-data:/data ` - --name surfsense ` - --restart unless-stopped ` - ghcr.io/modsetter/surfsense:latest -``` - -**Con Configuración Personalizada:** - -Puedes pasar cualquier variable de entorno usando flags `-e`: - -```bash -docker run -d -p 3000:3000 -p 8000:8000 -p 5133:5133 \ - -v surfsense-data:/data \ - -e EMBEDDING_MODEL=openai://text-embedding-ada-002 \ - -e OPENAI_API_KEY=your_openai_api_key \ - -e AUTH_TYPE=GOOGLE \ - -e GOOGLE_OAUTH_CLIENT_ID=your_google_client_id \ - -e GOOGLE_OAUTH_CLIENT_SECRET=your_google_client_secret \ - -e ETL_SERVICE=LLAMACLOUD \ - -e LLAMA_CLOUD_API_KEY=your_llama_cloud_key \ - --name surfsense \ - --restart unless-stopped \ - ghcr.io/modsetter/surfsense:latest -``` - -> [!NOTE] -> - Si despliegas detrás de un proxy inverso con HTTPS, agrega `-e BACKEND_URL=https://api.yourdomain.com` - -Después de iniciar, accede a SurfSense en: -- **Frontend**: [http://localhost:3000](http://localhost:3000) -- **API Backend**: [http://localhost:8000](http://localhost:8000) -- **Documentación API**: [http://localhost:8000/docs](http://localhost:8000/docs) -- **Electric-SQL**: [http://localhost:5133](http://localhost:5133) - -**Comandos Útiles:** - -```bash -docker logs -f surfsense # Ver logs -docker stop surfsense # Detener -docker start surfsense # Iniciar -docker rm surfsense # Eliminar (datos preservados en el volumen) -``` - -### Opciones de Instalación - -SurfSense ofrece múltiples opciones para empezar: - -1. **[SurfSense Cloud](https://www.surfsense.com/login)** - La forma más fácil de probar SurfSense sin ninguna configuración. - - No requiere instalación - - Acceso instantáneo a todas las funcionalidades - - Perfecto para empezar rápidamente - -2. **Inicio Rápido Docker (Arriba)** - Un solo comando para tener SurfSense funcionando localmente. - - Imagen todo-en-uno con PostgreSQL, Redis y todos los servicios incluidos - - Perfecto para evaluación, desarrollo y despliegues pequeños - - Datos persistidos vía volumen Docker - -3. **[Docker Compose (Producción)](https://www.surfsense.com/docs/docker-installation)** - Despliegue de stack completo con servicios separados. - - Incluye pgAdmin para gestión de base de datos a través de interfaz web - - Soporta personalización de variables de entorno vía archivo `.env` - - Opciones de despliegue flexibles (stack completo o solo servicios principales) - - Mejor para producción con escalado independiente de servicios - -4. **[Instalación Manual](https://www.surfsense.com/docs/manual-installation)** - Para usuarios que prefieren más control sobre su configuración o necesitan personalizar su despliegue. - -Las guías de Docker e instalación manual incluyen instrucciones detalladas específicas para Windows, macOS y Linux. - -Antes de la instalación auto-hospedada, asegúrate de completar los [pasos de configuración previos](https://www.surfsense.com/docs/) incluyendo: -- Configuración de autenticación (opcional - por defecto usa autenticación LOCAL) -- **Servicio ETL de Procesamiento de Archivos** (opcional - por defecto usa Docling): - - Docling (por defecto, procesamiento local, no requiere clave API, soporta PDF, documentos Office, imágenes, HTML, CSV) - - Clave API de Unstructured.io (soporta 34+ formatos) - - Clave API de LlamaIndex (análisis mejorado, soporta 50+ formatos) -- Otras claves API según sea necesario para tu caso de uso +**Tablero Kanban:** [SurfSense Project Board](https://github.com/users/MODSetter/projects/3) ## Contribuir -¡Las contribuciones son muy bienvenidas! Una contribución puede ser tan pequeña como una ⭐ o incluso encontrar y crear issues. -El ajuste fino del Backend siempre es deseado. +Todas las contribuciones son bienvenidas, desde estrellas y reportes de bugs hasta mejoras del backend. Consulta [CONTRIBUTING.md](CONTRIBUTING.md) para comenzar. -Para guías detalladas de contribución, consulta nuestro archivo [CONTRIBUTING.md](CONTRIBUTING.md). +Gracias a todos nuestros Surfers: + + + + ## Historial de Stars diff --git a/README.hi.md b/README.hi.md index 6ba70097d..595dd3799 100644 --- a/README.hi.md +++ b/README.hi.md @@ -15,6 +15,9 @@ [English](README.md) | [Español](README.es.md) | [Português](README.pt-BR.md) | [हिन्दी](README.hi.md) | [简体中文](README.zh-CN.md) + +
+MODSetter%2FSurfSense | Trendshift
# SurfSense @@ -22,101 +25,122 @@ SurfSense एक अत्यधिक अनुकूलन योग्य AI शोध एजेंट है, जो बाहरी स्रोतों से जुड़ा है जैसे सर्च इंजन (SearxNG, Tavily, LinkUp), Google Drive, Slack, Microsoft Teams, Linear, Jira, ClickUp, Confluence, BookStack, Gmail, Notion, YouTube, GitHub, Discord, Airtable, Google Calendar, Luma, Circleback, Elasticsearch, Obsidian और भी बहुत कुछ आने वाला है। -
-MODSetter%2FSurfSense | Trendshift -
-# वीडियो +# वीडियो https://github.com/user-attachments/assets/cc0c84d3-1f2f-4f7a-b519-2ecce22310b1 - ## पॉडकास्ट नमूना https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7 +## SurfSense का उपयोग कैसे करें +### Cloud + +1. [surfsense.com](https://www.surfsense.com) पर जाएं और लॉगिन करें। + +

लॉगिन

+ +2. अपने कनेक्टर जोड़ें और सिंक करें। कनेक्टर्स को अपडेट रखने के लिए आवधिक सिंकिंग सक्षम करें। + +

कनेक्टर्स

+ +3. जब तक कनेक्टर्स का डेटा इंडेक्स हो रहा है, दस्तावेज़ अपलोड करें। + +

दस्तावेज़ अपलोड करें

+ +4. सब कुछ इंडेक्स हो जाने के बाद, कुछ भी पूछें (उपयोग के मामले): + + - बेसिक सर्च और उद्धरण + +

सर्च और उद्धरण

+ + - दस्तावेज़ मेंशन QNA + +

दस्तावेज़ मेंशन QNA

+ + - रिपोर्ट जनरेशन और एक्सपोर्ट (फ़िलहाल PDF, DOCX) + +

रिपोर्ट जनरेशन

+ + - पॉडकास्ट जनरेशन + +

पॉडकास्ट जनरेशन

+ + - इमेज जनरेशन + +

इमेज जनरेशन

+ + - और भी बहुत कुछ जल्द आ रहा है। + + +### सेल्फ-होस्टेड + +पूर्ण डेटा नियंत्रण और गोपनीयता के लिए SurfSense को अपने स्वयं के बुनियादी ढांचे पर चलाएं। + +**त्वरित शुरुआत (Docker एक कमांड में):** + +```bash +docker run -d -p 3000:3000 -p 8000:8000 -p 5133:5133 \ + -v surfsense-data:/data \ + --name surfsense \ + --restart unless-stopped \ + ghcr.io/modsetter/surfsense:latest +``` + +शुरू करने के बाद, अपने ब्राउज़र में [http://localhost:3000](http://localhost:3000) खोलें। + +Docker Compose, मैनुअल इंस्टॉलेशन और अन्य डिप्लॉयमेंट विकल्पों के लिए, [डॉक्स](https://www.surfsense.com/docs/) देखें। + +### रीयल-टाइम सहयोग कैसे करें (बीटा) + +1. सदस्य प्रबंधन पेज पर जाएं और एक आमंत्रण बनाएं। + +

सदस्यों को आमंत्रित करें

+ +2. टीममेट जुड़ता है और वह SearchSpace साझा हो जाता है। + +

आमंत्रण स्वीकार प्रवाह

+ +3. चैट को साझा करें। + +

चैट साझा करें

+ +4. आपकी टीम अब रीयल-टाइम में चैट कर सकती है। + +

रीयल-टाइम चैट

+ +5. टीममेट्स को टैग करने के लिए कमेंट जोड़ें। + +

रीयल-टाइम कमेंट्स

## प्रमुख विशेषताएं -### 💡 **विचार**: -- NotebookLM, Perplexity और Glean का ओपन सोर्स विकल्प। किसी भी LLM को अपने आंतरिक ज्ञान स्रोतों से जोड़ें और अपनी टीम के साथ रीयल-टाइम में सहयोग करें। -### 📁 **कई फ़ाइल फ़ॉर्मेट अपलोड सपोर्ट** -- अपनी व्यक्तिगत फ़ाइलों *(दस्तावेज़, चित्र, वीडियो और **50+ फ़ाइल एक्सटेंशन** का समर्थन)* से सामग्री को अपने व्यक्तिगत ज्ञान आधार में सहेजें। -### 🔍 **शक्तिशाली खोज** -- अपनी सहेजी गई सामग्री में कुछ भी तुरंत खोजें या शोध करें। -### 💬 **अपनी सहेजी गई सामग्री के साथ चैट करें** -- प्राकृतिक भाषा में बातचीत करें और उद्धृत उत्तर प्राप्त करें। -### 📄 **उद्धृत उत्तर** -- Perplexity की तरह उद्धृत उत्तर प्राप्त करें। -### 🧩 **सार्वभौमिक अनुकूलता** -- OpenAI स्पेक और LiteLLM के माध्यम से लगभग किसी भी इंफ्रेंस प्रदाता को कनेक्ट करें। -### 🔔 **गोपनीयता और स्थानीय LLM सपोर्ट** -- vLLM और Ollama जैसे स्थानीय LLMs के साथ बेहतरीन काम करता है। -### 🏠 **सेल्फ-होस्ट करने योग्य** -- ओपन सोर्स और स्थानीय रूप से तैनात करना आसान। -### 👥 **RBAC के साथ टीम सहयोग** -- सर्च स्पेस के लिए भूमिका-आधारित एक्सेस नियंत्रण -- अनुकूलन योग्य भूमिकाओं (मालिक, एडमिन, संपादक, दर्शक) के साथ टीम सदस्यों को आमंत्रित करें -- दस्तावेज़ों, चैट, कनेक्टर और सेटिंग्स के लिए विस्तृत अनुमतियां -- अपने संगठन के भीतर सुरक्षित रूप से ज्ञान आधार साझा करें -- टीम चैट रीयल-टाइम में अपडेट होते हैं और कमेंट थ्रेड में "चैट के बारे में चैट" करें -### 🎙️ पॉडकास्ट -- अत्यंत तेज़ पॉडकास्ट जनरेशन एजेंट। (20 सेकंड से कम में 3 मिनट का पॉडकास्ट बनाता है।) -- अपनी चैट बातचीत को आकर्षक ऑडियो सामग्री में बदलें -- स्थानीय TTS प्रदाताओं का समर्थन (Kokoro TTS) -- कई TTS प्रदाताओं का समर्थन (OpenAI, Azure, Google Vertex AI) +| विशेषता | विवरण | +|----------|--------| +| OSS विकल्प | रीयल-टाइम टीम सहयोग के साथ NotebookLM, Perplexity और Glean का सीधा प्रतिस्थापन | +| 50+ फ़ाइल फ़ॉर्मेट | LlamaCloud, Unstructured या Docling (लोकल) के माध्यम से दस्तावेज़, चित्र, वीडियो अपलोड करें | +| हाइब्रिड सर्च | हायरार्किकल इंडाइसेस और Reciprocal Rank Fusion के साथ सिमैंटिक + फुल टेक्स्ट सर्च | +| उद्धृत उत्तर | अपने ज्ञान आधार के साथ चैट करें और Perplexity शैली के उद्धृत उत्तर पाएं | +| डीप एजेंट आर्किटेक्चर | [LangChain Deep Agents](https://docs.langchain.com/oss/python/deepagents/overview) द्वारा संचालित, योजना, सब-एजेंट और फ़ाइल सिस्टम एक्सेस | +| यूनिवर्सल LLM सपोर्ट | 100+ LLMs, 6000+ एम्बेडिंग मॉडल, सभी प्रमुख रीरैंकर्स OpenAI spec और LiteLLM के माध्यम से | +| प्राइवेसी फर्स्ट | पूर्ण लोकल LLM सपोर्ट (vLLM, Ollama) आपका डेटा आपका रहता है | +| टीम सहयोग | मालिक / एडमिन / संपादक / दर्शक भूमिकाओं के साथ RBAC, रीयल-टाइम चैट और कमेंट थ्रेड | +| पॉडकास्ट जनरेशन | 20 सेकंड से कम में 3 मिनट का पॉडकास्ट; कई TTS प्रदाता (OpenAI, Azure, Kokoro) | +| ब्राउज़र एक्सटेंशन | किसी भी वेबपेज को सहेजने के लिए क्रॉस-ब्राउज़र एक्सटेंशन, प्रमाणीकरण सुरक्षित पेज सहित | +| 25+ कनेक्टर्स | सर्च इंजन, Google Drive, Slack, Teams, Jira, Notion, GitHub, Discord और [अधिक](#बाहरी-स्रोत) | +| सेल्फ-होस्ट करने योग्य | ओपन सोर्स, Docker एक कमांड या प्रोडक्शन के लिए पूर्ण Docker Compose | -### 🤖 **डीप एजेंट आर्किटेक्चर** -- [LangChain Deep Agents](https://docs.langchain.com/oss/python/deepagents/overview) द्वारा संचालित - ऐसे एजेंट जो योजना बना सकते हैं, सब-एजेंट का उपयोग कर सकते हैं, और जटिल कार्यों के लिए फ़ाइल सिस्टम का लाभ उठा सकते हैं। +
+बाहरी स्रोतों की पूरी सूची + -### 📊 **उन्नत RAG तकनीकें** -- 100+ LLMs का समर्थन -- 6000+ एम्बेडिंग मॉडल का समर्थन -- सभी प्रमुख रीरैंकर्स का समर्थन (Pinecone, Cohere, Flashrank आदि) -- हायरार्किकल इंडाइसेस का उपयोग (2 स्तरीय RAG सेटअप) -- हाइब्रिड सर्च का उपयोग (सिमैंटिक + फुल टेक्स्ट सर्च, Reciprocal Rank Fusion के साथ) - -### ℹ️ **बाहरी स्रोत** -- सर्च इंजन (Tavily, LinkUp) -- SearxNG (सेल्फ-होस्टेड इंस्टेंस) -- Google Drive -- Slack -- Microsoft Teams -- Linear -- Jira -- ClickUp -- Confluence -- BookStack -- Notion -- Gmail -- YouTube वीडियो -- GitHub -- Discord -- Airtable -- Google Calendar -- Luma -- Circleback -- Elasticsearch -- Obsidian -- और भी बहुत कुछ आने वाला है..... - -## 📄 **समर्थित फ़ाइल एक्सटेंशन** - -| ETL सेवा | फ़ॉर्मेट | नोट्स | -|-----------|----------|-------| -| **LlamaCloud** | 50+ फ़ॉर्मेट | दस्तावेज़, प्रस्तुतियां, स्प्रेडशीट, चित्र | -| **Unstructured** | 34+ फ़ॉर्मेट | मुख्य फ़ॉर्मेट + ईमेल समर्थन | -| **Docling** | मुख्य फ़ॉर्मेट | स्थानीय प्रोसेसिंग, API कुंजी की आवश्यकता नहीं | - -**ऑडियो/वीडियो** (STT सेवा के माध्यम से): `.mp3`, `.wav`, `.mp4`, `.webm`, आदि। - -### 🔖 क्रॉस-ब्राउज़र एक्सटेंशन -- SurfSense एक्सटेंशन का उपयोग किसी भी वेबपेज को सहेजने के लिए किया जा सकता है। -- इसका मुख्य उपयोग प्रमाणीकरण द्वारा संरक्षित वेबपेजों को सहेजना है। +सर्च इंजन (Tavily, LinkUp) · SearxNG · Google Drive · Slack · Microsoft Teams · Linear · Jira · ClickUp · Confluence · BookStack · Notion · Gmail · YouTube वीडियो · GitHub · Discord · Airtable · Google Calendar · Luma · Circleback · Elasticsearch · Obsidian, और भी बहुत कुछ आने वाला है। +
## फ़ीचर अनुरोध और भविष्य @@ -126,119 +150,25 @@ https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7 [SurfSense Discord](https://discord.gg/ejRNvftDp9) में शामिल हों और SurfSense के भविष्य को आकार देने में मदद करें! -## 🚀 रोडमैप +## रोडमैप हमारे विकास की प्रगति और आने वाली सुविधाओं से अपडेट रहें! हमारा सार्वजनिक रोडमैप देखें और अपने विचार या फ़ीडबैक दें: -**📋 रोडमैप चर्चा:** [SurfSense 2025-2026 Roadmap: Deep Agents, Real-Time Collaboration & MCP Servers](https://github.com/MODSetter/SurfSense/discussions/565) +**रोडमैप चर्चा:** [SurfSense 2026 Roadmap](https://github.com/MODSetter/SurfSense/discussions/565) -**📊 कानबन बोर्ड:** [SurfSense Project Board](https://github.com/users/MODSetter/projects/3) - - -## कैसे शुरू करें? - -### Docker के साथ त्वरित शुरुआत 🐳 - -> [!TIP] -> प्रोडक्शन डिप्लॉयमेंट के लिए, पूर्ण [Docker Compose सेटअप](https://www.surfsense.com/docs/docker-installation) का उपयोग करें जो अधिक नियंत्रण और स्केलेबिलिटी प्रदान करता है। - -**Linux/macOS:** - -```bash -docker run -d -p 3000:3000 -p 8000:8000 -p 5133:5133 \ - -v surfsense-data:/data \ - --name surfsense \ - --restart unless-stopped \ - ghcr.io/modsetter/surfsense:latest -``` - -**Windows (PowerShell):** - -```powershell -docker run -d -p 3000:3000 -p 8000:8000 -p 5133:5133 ` - -v surfsense-data:/data ` - --name surfsense ` - --restart unless-stopped ` - ghcr.io/modsetter/surfsense:latest -``` - -**कस्टम कॉन्फ़िगरेशन के साथ:** - -आप `-e` फ़्लैग का उपयोग करके कोई भी एन्वायरनमेंट वेरिएबल पास कर सकते हैं: - -```bash -docker run -d -p 3000:3000 -p 8000:8000 -p 5133:5133 \ - -v surfsense-data:/data \ - -e EMBEDDING_MODEL=openai://text-embedding-ada-002 \ - -e OPENAI_API_KEY=your_openai_api_key \ - -e AUTH_TYPE=GOOGLE \ - -e GOOGLE_OAUTH_CLIENT_ID=your_google_client_id \ - -e GOOGLE_OAUTH_CLIENT_SECRET=your_google_client_secret \ - -e ETL_SERVICE=LLAMACLOUD \ - -e LLAMA_CLOUD_API_KEY=your_llama_cloud_key \ - --name surfsense \ - --restart unless-stopped \ - ghcr.io/modsetter/surfsense:latest -``` - -> [!NOTE] -> - यदि HTTPS के साथ रिवर्स प्रॉक्सी के पीछे डिप्लॉय कर रहे हैं, तो `-e BACKEND_URL=https://api.yourdomain.com` जोड़ें - -शुरू करने के बाद, SurfSense तक पहुंचें: -- **फ्रंटएंड**: [http://localhost:3000](http://localhost:3000) -- **बैकएंड API**: [http://localhost:8000](http://localhost:8000) -- **API डॉक्स**: [http://localhost:8000/docs](http://localhost:8000/docs) -- **Electric-SQL**: [http://localhost:5133](http://localhost:5133) - -**उपयोगी कमांड:** - -```bash -docker logs -f surfsense # लॉग देखें -docker stop surfsense # रोकें -docker start surfsense # शुरू करें -docker rm surfsense # हटाएं (डेटा वॉल्यूम में सुरक्षित रहता है) -``` - -### इंस्टॉलेशन विकल्प - -SurfSense शुरू करने के लिए कई विकल्प प्रदान करता है: - -1. **[SurfSense Cloud](https://www.surfsense.com/login)** - बिना किसी सेटअप के SurfSense आज़माने का सबसे आसान तरीका। - - इंस्टॉलेशन की आवश्यकता नहीं - - सभी सुविधाओं तक तत्काल पहुंच - - जल्दी शुरू करने के लिए बिल्कुल सही - -2. **Docker त्वरित शुरुआत (ऊपर)** - एक ही कमांड से SurfSense को स्थानीय रूप से चलाएं। - - PostgreSQL, Redis और सभी सेवाओं के साथ ऑल-इन-वन इमेज - - मूल्यांकन, विकास और छोटे डिप्लॉयमेंट के लिए बिल्कुल सही - - Docker वॉल्यूम के माध्यम से डेटा पर्सिस्ट - -3. **[Docker Compose (प्रोडक्शन)](https://www.surfsense.com/docs/docker-installation)** - अलग-अलग सेवाओं के साथ पूर्ण स्टैक डिप्लॉयमेंट। - - वेब UI के माध्यम से डेटाबेस प्रबंधन के लिए pgAdmin शामिल - - `.env` फ़ाइल के माध्यम से एन्वायरनमेंट वेरिएबल कस्टमाइज़ेशन का समर्थन - - लचीले डिप्लॉयमेंट विकल्प (पूर्ण स्टैक या केवल मुख्य सेवाएं) - - सेवाओं के स्वतंत्र स्केलिंग के साथ प्रोडक्शन के लिए बेहतर - -4. **[मैनुअल इंस्टॉलेशन](https://www.surfsense.com/docs/manual-installation)** - उन उपयोगकर्ताओं के लिए जो अपने सेटअप पर अधिक नियंत्रण चाहते हैं या अपने डिप्लॉयमेंट को कस्टमाइज़ करना चाहते हैं। - -Docker और मैनुअल इंस्टॉलेशन गाइड में Windows, macOS और Linux के लिए विस्तृत OS-विशिष्ट निर्देश शामिल हैं। - -सेल्फ-होस्टिंग इंस्टॉलेशन से पहले, [पूर्वापेक्षा सेटअप चरण](https://www.surfsense.com/docs/) पूरा करना सुनिश्चित करें, जिसमें शामिल हैं: -- प्रमाणीकरण सेटअप (वैकल्पिक - डिफ़ॉल्ट LOCAL प्रमाणीकरण) -- **फ़ाइल प्रोसेसिंग ETL सेवा** (वैकल्पिक - डिफ़ॉल्ट Docling): - - Docling (डिफ़ॉल्ट, स्थानीय प्रोसेसिंग, API कुंजी की आवश्यकता नहीं, PDF, Office दस्तावेज़, चित्र, HTML, CSV का समर्थन) - - Unstructured.io API कुंजी (34+ फ़ॉर्मेट का समर्थन) - - LlamaIndex API कुंजी (उन्नत पार्सिंग, 50+ फ़ॉर्मेट का समर्थन) -- आपके उपयोग के अनुसार अन्य API कुंजियां +**कानबन बोर्ड:** [SurfSense Project Board](https://github.com/users/MODSetter/projects/3) ## योगदान करें -योगदान का बहुत स्वागत है! योगदान एक ⭐ जितना छोटा हो सकता है या issues खोजना और बनाना भी। -बैकएंड की फ़ाइन-ट्यूनिंग हमेशा वांछित है। +सभी योगदान स्वागत योग्य हैं, स्टार और बग रिपोर्ट से लेकर बैकएंड सुधार तक। शुरू करने के लिए [CONTRIBUTING.md](CONTRIBUTING.md) देखें। -विस्तृत योगदान दिशानिर्देशों के लिए, कृपया हमारी [CONTRIBUTING.md](CONTRIBUTING.md) फ़ाइल देखें। +हमारे सभी Surfers को धन्यवाद: + + + + ## Star इतिहास diff --git a/README.md b/README.md index 3fb8014c5..e1c2ca256 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,9 @@ [English](README.md) | [Español](README.es.md) | [Português](README.pt-BR.md) | [हिन्दी](README.hi.md) | [简体中文](README.zh-CN.md) + +
+MODSetter%2FSurfSense | Trendshift
# SurfSense @@ -22,101 +25,122 @@ Connect any LLM to your internal knowledge sources and chat with it in real time SurfSense is a highly customizable AI research agent, connected to external sources such as Search Engines (SearxNG, Tavily, LinkUp), Google Drive, Slack, Microsoft Teams, Linear, Jira, ClickUp, Confluence, BookStack, Gmail, Notion, YouTube, GitHub, Discord, Airtable, Google Calendar, Luma, Circleback, Elasticsearch, Obsidian and more to come. -
-MODSetter%2FSurfSense | Trendshift -
# Video https://github.com/user-attachments/assets/cc0c84d3-1f2f-4f7a-b519-2ecce22310b1 - ## Podcast Sample https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7 +## How to Use SurfSense +### Cloud + +1. Go to [surfsense.com](https://www.surfsense.com) and login. + +

Login

+ +2. Connect your connectors and sync. Enable periodic syncing to keep connectors synced. + +

Connectors

+ +3. Till connectors data index, upload Documents. + +

Upload Documents

+ +4. Once everything is indexed, Ask Away (Use Cases): + + - Basic search and citation + +

Search and Citation

+ + - Document Mention QNA + +

Document Mention QNA

+ + - Report Generations and Exports (PDF, DOCX for now) + +

Report Generation

+ + - Podcast Generations + +

Podcast Generation

+ + - Image Generations + +

Image Generation

+ + - And more coming soon. + + +### Self Hosted + +Run SurfSense on your own infrastructure for full data control and privacy. + +**Quick Start (Docker one-liner):** + +```bash +docker run -d -p 3000:3000 -p 8000:8000 -p 5133:5133 \ + -v surfsense-data:/data \ + --name surfsense \ + --restart unless-stopped \ + ghcr.io/modsetter/surfsense:latest +``` + +After starting, open [http://localhost:3000](http://localhost:3000) in your browser. + +For Docker Compose, manual installation, and other deployment options, check the [docs](https://www.surfsense.com/docs/). + +### How to Realtime Collaborate (Beta) + +1. Go to Manage Members page and create an invite. + +

Invite Members

+ +2. Teammate joins and that SearchSpace becomes shared. + +

Invite Join Flow

+ +3. Make chat shared. + +

Make Chat Shared

+ +4. Your team can now chat in realtime. + +

Realtime Chat

+ +5. Add comment to tag teammates. + +

Realtime Comments

## Key Features -### 💡 **Idea**: -- Open source alternative to NotebookLM, Perplexity, and Glean. Connect any LLM to your internal knowledge sources and collaborate with your team in real time. -### 📁 **Multiple File Format Uploading Support** -- Save content from your own personal files *(Documents, images, videos and supports **50+ file extensions**)* to your own personal knowledge base . -### 🔍 **Powerful Search** -- Quickly research or find anything in your saved content . -### 💬 **Chat with your Saved Content** -- Interact in Natural Language and get cited answers. -### 📄 **Cited Answers** -- Get Cited answers just like Perplexity. -### 🧩 **Universal Compatibility** -- Connect virtually any inference provider via the OpenAI spec and LiteLLM. -### 🔔 **Privacy & Local LLM Support** -- Works Flawlessly with local LLMs like vLLM and Ollama. -### 🏠 **Self Hostable** -- Open source and easy to deploy locally. -### 👥 **Team Collaboration with RBAC** -- Role-Based Access Control for Search Spaces -- Invite team members with customizable roles (Owner, Admin, Editor, Viewer) -- Granular permissions for documents, chats, connectors, and settings -- Share knowledge bases securely within your organization -- Team chats update in real-time and "Chat about the chat" in comment threads -### 🎙️ Podcasts -- Blazingly fast podcast generation agent. (Creates a 3-minute podcast in under 20 seconds.) -- Convert your chat conversations into engaging audio content -- Support for local TTS providers (Kokoro TTS) -- Support for multiple TTS providers (OpenAI, Azure, Google Vertex AI) +| Feature | Description | +|---------|-------------| +| OSS Alternative | Drop in replacement for NotebookLM, Perplexity, and Glean with real time team collaboration | +| 50+ File Formats | Upload documents, images, videos via LlamaCloud, Unstructured, or Docling (local) | +| Hybrid Search | Semantic + Full Text Search with Hierarchical Indices and Reciprocal Rank Fusion | +| Cited Answers | Chat with your knowledge base and get Perplexity style cited responses | +| Deep Agent Architecture | Powered by [LangChain Deep Agents](https://docs.langchain.com/oss/python/deepagents/overview) planning, subagents, and file system access | +| Universal LLM Support | 100+ LLMs, 6000+ embedding models, all major rerankers via OpenAI spec & LiteLLM | +| Privacy First | Full local LLM support (vLLM, Ollama) your data stays yours | +| Team Collaboration | RBAC with Owner / Admin / Editor / Viewer roles, real time chat & comment threads | +| Podcast Generation | 3 min podcast in under 20 seconds; multiple TTS providers (OpenAI, Azure, Kokoro) | +| Browser Extension | Cross browser extension to save any webpage, including auth protected pages | +| 25+ Connectors | Search Engines, Google Drive, Slack, Teams, Jira, Notion, GitHub, Discord & [more](#external-sources) | +| Self Hostable | Open source, Docker one liner or full Docker Compose for production | -### 🤖 **Deep Agent Architecture** -- Powered by [LangChain Deep Agents](https://docs.langchain.com/oss/python/deepagents/overview) - agents that can plan, use subagents, and leverage file systems for complex tasks. +
+Full list of External Sources + -### 📊 **Advanced RAG Techniques** -- Supports 100+ LLM's -- Supports 6000+ Embedding Models. -- Supports all major Rerankers (Pinecone, Cohere, Flashrank etc) -- Uses Hierarchical Indices (2 tiered RAG setup). -- Utilizes Hybrid Search (Semantic + Full Text Search combined with Reciprocal Rank Fusion). - -### ℹ️ **External Sources** -- Search Engines (Tavily, LinkUp) -- SearxNG (self-hosted instances) -- Google Drive -- Slack -- Microsoft Teams -- Linear -- Jira -- ClickUp -- Confluence -- BookStack -- Notion -- Gmail -- Youtube Videos -- GitHub -- Discord -- Airtable -- Google Calendar -- Luma -- Circleback -- Elasticsearch -- Obsidian -- and more to come..... - -## 📄 **Supported File Extensions** - -| ETL Service | Formats | Notes | -|-------------|---------|-------| -| **LlamaCloud** | 50+ formats | Documents, presentations, spreadsheets, images | -| **Unstructured** | 34+ formats | Core formats + email support | -| **Docling** | Core formats | Local processing, no API key required | - -**Audio/Video** (via STT Service): `.mp3`, `.wav`, `.mp4`, `.webm`, etc. - -### 🔖 Cross Browser Extension -- The SurfSense extension can be used to save any webpage you like. -- Its main usecase is to save any webpages protected beyond authentication. +Search Engines (Tavily, LinkUp) · SearxNG · Google Drive · Slack · Microsoft Teams · Linear · Jira · ClickUp · Confluence · BookStack · Notion · Gmail · YouTube Videos · GitHub · Discord · Airtable · Google Calendar · Luma · Circleback · Elasticsearch · Obsidian, and more to come. +
## FEATURE REQUESTS AND FUTURE @@ -126,119 +150,25 @@ https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7 Join the [SurfSense Discord](https://discord.gg/ejRNvftDp9) and help shape the future of SurfSense! -## 🚀 Roadmap +## Roadmap Stay up to date with our development progress and upcoming features! Check out our public roadmap and contribute your ideas or feedback: -**📋 Roadmap Discussion:** [SurfSense 2025-2026 Roadmap: Deep Agents, Real-Time Collaboration & MCP Servers](https://github.com/MODSetter/SurfSense/discussions/565) +**Roadmap Discussion:** [SurfSense 2026 Roadmap](https://github.com/MODSetter/SurfSense/discussions/565) -**📊 Kanban Board:** [SurfSense Project Board](https://github.com/users/MODSetter/projects/3) +**Kanban Board:** [SurfSense Project Board](https://github.com/users/MODSetter/projects/3) -## How to get started? +## Contribute -### Quick Start with Docker 🐳 +All contributions welcome, from stars and bug reports to backend improvements. See [CONTRIBUTING.md](CONTRIBUTING.md) to get started. -> [!TIP] -> For production deployments, use the full [Docker Compose setup](https://www.surfsense.com/docs/docker-installation) which offers more control and scalability. +Thanks to all our Surfers: -**Linux/macOS:** - -```bash -docker run -d -p 3000:3000 -p 8000:8000 -p 5133:5133 \ - -v surfsense-data:/data \ - --name surfsense \ - --restart unless-stopped \ - ghcr.io/modsetter/surfsense:latest -``` - -**Windows (PowerShell):** - -```powershell -docker run -d -p 3000:3000 -p 8000:8000 -p 5133:5133 ` - -v surfsense-data:/data ` - --name surfsense ` - --restart unless-stopped ` - ghcr.io/modsetter/surfsense:latest -``` - -**With Custom Configuration:** - -You can pass any environment variable using `-e` flags: - -```bash -docker run -d -p 3000:3000 -p 8000:8000 -p 5133:5133 \ - -v surfsense-data:/data \ - -e EMBEDDING_MODEL=openai://text-embedding-ada-002 \ - -e OPENAI_API_KEY=your_openai_api_key \ - -e AUTH_TYPE=GOOGLE \ - -e GOOGLE_OAUTH_CLIENT_ID=your_google_client_id \ - -e GOOGLE_OAUTH_CLIENT_SECRET=your_google_client_secret \ - -e ETL_SERVICE=LLAMACLOUD \ - -e LLAMA_CLOUD_API_KEY=your_llama_cloud_key \ - --name surfsense \ - --restart unless-stopped \ - ghcr.io/modsetter/surfsense:latest -``` - -> [!NOTE] -> - If deploying behind a reverse proxy with HTTPS, add `-e BACKEND_URL=https://api.yourdomain.com` - -After starting, access SurfSense at: -- **Frontend**: [http://localhost:3000](http://localhost:3000) -- **Backend API**: [http://localhost:8000](http://localhost:8000) -- **API Docs**: [http://localhost:8000/docs](http://localhost:8000/docs) -- **Electric-SQL**: [http://localhost:5133](http://localhost:5133) - -**Useful Commands:** - -```bash -docker logs -f surfsense # View logs -docker stop surfsense # Stop -docker start surfsense # Start -docker rm surfsense # Remove (data preserved in volume) -``` - -### Installation Options - -SurfSense provides multiple options to get started: - -1. **[SurfSense Cloud](https://www.surfsense.com/login)** - The easiest way to try SurfSense without any setup. - - No installation required - - Instant access to all features - - Perfect for getting started quickly - -2. **Quick Start Docker (Above)** - Single command to get SurfSense running locally. - - All-in-one image with PostgreSQL, Redis, and all services bundled - - Perfect for evaluation, development, and small deployments - - Data persisted via Docker volume - -3. **[Docker Compose (Production)](https://www.surfsense.com/docs/docker-installation)** - Full stack deployment with separate services. - - Includes pgAdmin for database management through a web UI - - Supports environment variable customization via `.env` file - - Flexible deployment options (full stack or core services only) - - Better for production with separate scaling of services - -4. **[Manual Installation](https://www.surfsense.com/docs/manual-installation)** - For users who prefer more control over their setup or need to customize their deployment. - -Docker and manual installation guides include detailed OS-specific instructions for Windows, macOS, and Linux. - -Before self-hosting installation, make sure to complete the [prerequisite setup steps](https://www.surfsense.com/docs/) including: -- Auth setup (optional - defaults to LOCAL auth) -- **File Processing ETL Service** (optional - defaults to Docling): - - Docling (default, local processing, no API key required, supports PDF, Office docs, images, HTML, CSV) - - Unstructured.io API key (supports 34+ formats) - - LlamaIndex API key (enhanced parsing, supports 50+ formats) -- Other API keys as needed for your use case - - -## Contribute - -Contributions are very welcome! A contribution can be as small as a ⭐ or even finding and creating issues. -Fine-tuning the Backend is always desired. - -For detailed contribution guidelines, please see our [CONTRIBUTING.md](CONTRIBUTING.md) file. + + + ## Star History diff --git a/README.pt-BR.md b/README.pt-BR.md index f57c28664..50016b4e6 100644 --- a/README.pt-BR.md +++ b/README.pt-BR.md @@ -15,6 +15,9 @@ [English](README.md) | [Español](README.es.md) | [Português](README.pt-BR.md) | [हिन्दी](README.hi.md) | [简体中文](README.zh-CN.md) + +
+MODSetter%2FSurfSense | Trendshift
# SurfSense @@ -22,101 +25,122 @@ Conecte qualquer LLM às suas fontes de conhecimento internas e converse com ele SurfSense é um agente de pesquisa de IA altamente personalizável, conectado a fontes externas como mecanismos de busca (SearxNG, Tavily, LinkUp), Google Drive, Slack, Microsoft Teams, Linear, Jira, ClickUp, Confluence, BookStack, Gmail, Notion, YouTube, GitHub, Discord, Airtable, Google Calendar, Luma, Circleback, Elasticsearch, Obsidian e mais por vir. -
-MODSetter%2FSurfSense | Trendshift -
-# Vídeo +# Vídeo https://github.com/user-attachments/assets/cc0c84d3-1f2f-4f7a-b519-2ecce22310b1 - ## Exemplo de Podcast https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7 +## Como Usar o SurfSense +### Cloud + +1. Acesse [surfsense.com](https://www.surfsense.com) e faça login. + +

Login

+ +2. Conecte seus conectores e sincronize. Ative a sincronização periódica para manter os conectores atualizados. + +

Conectores

+ +3. Enquanto os dados dos conectores são indexados, faça upload de documentos. + +

Upload de Documentos

+ +4. Quando tudo estiver indexado, pergunte o que quiser (Casos de uso): + + - Busca básica e citações + +

Busca e Citação

+ + - QNA com menção de documentos + +

QNA com Menção de Documentos

+ + - Geração de relatórios e exportações (PDF, DOCX por enquanto) + +

Geração de Relatórios

+ + - Geração de podcasts + +

Geração de Podcasts

+ + - Geração de imagens + +

Geração de Imagens

+ + - E mais em breve. + + +### Auto-Hospedado + +Execute o SurfSense na sua própria infraestrutura para controle total de dados e privacidade. + +**Início Rápido (Docker em um único comando):** + +```bash +docker run -d -p 3000:3000 -p 8000:8000 -p 5133:5133 \ + -v surfsense-data:/data \ + --name surfsense \ + --restart unless-stopped \ + ghcr.io/modsetter/surfsense:latest +``` + +Após iniciar, abra [http://localhost:3000](http://localhost:3000) no seu navegador. + +Para Docker Compose, instalação manual e outras opções de implantação, consulte a [documentação](https://www.surfsense.com/docs/). + +### Como Colaborar em Tempo Real (Beta) + +1. Acesse a página de Gerenciar Membros e crie um convite. + +

Convidar Membros

+ +2. O colega aceita e aquele SearchSpace se torna compartilhado. + +

Fluxo de Entrada por Convite

+ +3. Torne o chat compartilhado. + +

Tornar Chat Compartilhado

+ +4. Sua equipe agora pode conversar em tempo real. + +

Chat em Tempo Real

+ +5. Adicione comentários para marcar colegas de equipe. + +

Comentários em Tempo Real

## Funcionalidades Principais -### 💡 **Ideia**: -- Alternativa de código aberto ao NotebookLM, Perplexity e Glean. Conecte qualquer LLM às suas fontes de conhecimento internas e colabore com sua equipe em tempo real. -### 📁 **Suporte a Múltiplos Formatos de Arquivo** -- Salve conteúdo dos seus arquivos pessoais *(Documentos, imagens, vídeos e suporta **mais de 50 extensões de arquivo**)* na sua própria base de conhecimento pessoal. -### 🔍 **Pesquisa Poderosa** -- Pesquise ou encontre rapidamente qualquer coisa no seu conteúdo salvo. -### 💬 **Converse com seu Conteúdo Salvo** -- Interaja em linguagem natural e obtenha respostas com citações. -### 📄 **Respostas com Citações** -- Obtenha respostas com citações como no Perplexity. -### 🧩 **Compatibilidade Universal** -- Conecte virtualmente qualquer provedor de inferência via especificação OpenAI e LiteLLM. -### 🔔 **Privacidade e Suporte a LLM Local** -- Funciona perfeitamente com LLMs locais como vLLM e Ollama. -### 🏠 **Auto-Hospedável** -- Código aberto e fácil de implantar localmente. -### 👥 **Colaboração em Equipe com RBAC** -- Controle de acesso baseado em funções para Espaços de Pesquisa -- Convide membros da equipe com funções personalizáveis (Proprietário, Admin, Editor, Visualizador) -- Permissões granulares para documentos, chats, conectores e configurações -- Compartilhe bases de conhecimento com segurança dentro da sua organização -- Chats de equipe atualizam em tempo real e "Converse sobre o chat" em threads de comentários -### 🎙️ Podcasts -- Agente de geração de podcasts ultrarrápido. (Cria um podcast de 3 minutos em menos de 20 segundos.) -- Converta suas conversas de chat em conteúdo de áudio envolvente -- Suporte para provedores TTS locais (Kokoro TTS) -- Suporte para múltiplos provedores TTS (OpenAI, Azure, Google Vertex AI) +| Funcionalidade | Descrição | +|----------------|-----------| +| Alternativa OSS | Substituto direto do NotebookLM, Perplexity e Glean com colaboração em equipe em tempo real | +| 50+ Formatos de Arquivo | Faça upload de documentos, imagens, vídeos via LlamaCloud, Unstructured ou Docling (local) | +| Busca Híbrida | Semântica + Texto completo com Índices Hierárquicos e Reciprocal Rank Fusion | +| Respostas com Citações | Converse com sua base de conhecimento e obtenha respostas citadas no estilo Perplexity | +| Arquitetura de Agentes Profundos | Alimentado por [LangChain Deep Agents](https://docs.langchain.com/oss/python/deepagents/overview) com planejamento, subagentes e acesso ao sistema de arquivos | +| Suporte Universal de LLM | 100+ LLMs, 6000+ modelos de embeddings, todos os principais rerankers via OpenAI spec e LiteLLM | +| Privacidade em Primeiro Lugar | Suporte completo a LLM local (vLLM, Ollama) seus dados ficam com você | +| Colaboração em Equipe | RBAC com papéis de Proprietário / Admin / Editor / Visualizador, chat em tempo real e threads de comentários | +| Geração de Podcasts | Podcast de 3 min em menos de 20 segundos; múltiplos provedores TTS (OpenAI, Azure, Kokoro) | +| Extensão de Navegador | Extensão multi-navegador para salvar qualquer página web, incluindo páginas protegidas por autenticação | +| 25+ Conectores | Mecanismos de busca, Google Drive, Slack, Teams, Jira, Notion, GitHub, Discord e [mais](#fontes-externas) | +| Auto-Hospedável | Código aberto, Docker em um único comando ou Docker Compose completo para produção | -### 🤖 **Arquitetura de Agentes Profundos** -- Alimentado por [LangChain Deep Agents](https://docs.langchain.com/oss/python/deepagents/overview) - agentes que podem planejar, usar subagentes e aproveitar sistemas de arquivos para tarefas complexas. +
+Lista completa de Fontes Externas + -### 📊 **Técnicas Avançadas de RAG** -- Suporta mais de 100 LLMs -- Suporta mais de 6000 modelos de embeddings -- Suporta todos os principais rerankers (Pinecone, Cohere, Flashrank, etc.) -- Utiliza índices hierárquicos (configuração RAG de 2 níveis) -- Utiliza busca híbrida (Semântica + Texto completo combinado com Reciprocal Rank Fusion) - -### ℹ️ **Fontes Externas** -- Mecanismos de busca (Tavily, LinkUp) -- SearxNG (instâncias auto-hospedadas) -- Google Drive -- Slack -- Microsoft Teams -- Linear -- Jira -- ClickUp -- Confluence -- BookStack -- Notion -- Gmail -- Vídeos do YouTube -- GitHub -- Discord -- Airtable -- Google Calendar -- Luma -- Circleback -- Elasticsearch -- Obsidian -- e mais por vir..... - -## 📄 **Extensões de Arquivo Suportadas** - -| Serviço ETL | Formatos | Notas | -|-------------|----------|-------| -| **LlamaCloud** | 50+ formatos | Documentos, apresentações, planilhas, imagens | -| **Unstructured** | 34+ formatos | Formatos principais + suporte a e-mail | -| **Docling** | Formatos principais | Processamento local, sem necessidade de chave API | - -**Áudio/Vídeo** (via serviço STT): `.mp3`, `.wav`, `.mp4`, `.webm`, etc. - -### 🔖 Extensão Multi-Navegador -- A extensão do SurfSense pode ser usada para salvar qualquer página web que você desejar. -- Seu principal uso é salvar páginas web protegidas por autenticação. +Mecanismos de Busca (Tavily, LinkUp) · SearxNG · Google Drive · Slack · Microsoft Teams · Linear · Jira · ClickUp · Confluence · BookStack · Notion · Gmail · Vídeos do YouTube · GitHub · Discord · Airtable · Google Calendar · Luma · Circleback · Elasticsearch · Obsidian, e mais por vir. +
## SOLICITAÇÕES DE FUNCIONALIDADES E FUTURO @@ -126,119 +150,25 @@ https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7 Junte-se ao [Discord do SurfSense](https://discord.gg/ejRNvftDp9) e ajude a moldar o futuro do SurfSense! -## 🚀 Roadmap +## Roadmap Fique atualizado com nosso progresso de desenvolvimento e próximas funcionalidades! Confira nosso roadmap público e contribua com suas ideias ou feedback: -**📋 Discussão do Roadmap:** [SurfSense 2025-2026 Roadmap: Deep Agents, Real-Time Collaboration & MCP Servers](https://github.com/MODSetter/SurfSense/discussions/565) +**Discussão do Roadmap:** [SurfSense 2026 Roadmap](https://github.com/MODSetter/SurfSense/discussions/565) -**📊 Quadro Kanban:** [SurfSense Project Board](https://github.com/users/MODSetter/projects/3) - - -## Como começar? - -### Início Rápido com Docker 🐳 - -> [!TIP] -> Para implantações em produção, use a configuração completa do [Docker Compose](https://www.surfsense.com/docs/docker-installation) que oferece mais controle e escalabilidade. - -**Linux/macOS:** - -```bash -docker run -d -p 3000:3000 -p 8000:8000 -p 5133:5133 \ - -v surfsense-data:/data \ - --name surfsense \ - --restart unless-stopped \ - ghcr.io/modsetter/surfsense:latest -``` - -**Windows (PowerShell):** - -```powershell -docker run -d -p 3000:3000 -p 8000:8000 -p 5133:5133 ` - -v surfsense-data:/data ` - --name surfsense ` - --restart unless-stopped ` - ghcr.io/modsetter/surfsense:latest -``` - -**Com Configuração Personalizada:** - -Você pode passar qualquer variável de ambiente usando flags `-e`: - -```bash -docker run -d -p 3000:3000 -p 8000:8000 -p 5133:5133 \ - -v surfsense-data:/data \ - -e EMBEDDING_MODEL=openai://text-embedding-ada-002 \ - -e OPENAI_API_KEY=your_openai_api_key \ - -e AUTH_TYPE=GOOGLE \ - -e GOOGLE_OAUTH_CLIENT_ID=your_google_client_id \ - -e GOOGLE_OAUTH_CLIENT_SECRET=your_google_client_secret \ - -e ETL_SERVICE=LLAMACLOUD \ - -e LLAMA_CLOUD_API_KEY=your_llama_cloud_key \ - --name surfsense \ - --restart unless-stopped \ - ghcr.io/modsetter/surfsense:latest -``` - -> [!NOTE] -> - Se estiver implantando atrás de um proxy reverso com HTTPS, adicione `-e BACKEND_URL=https://api.yourdomain.com` - -Após iniciar, acesse o SurfSense em: -- **Frontend**: [http://localhost:3000](http://localhost:3000) -- **API Backend**: [http://localhost:8000](http://localhost:8000) -- **Documentação da API**: [http://localhost:8000/docs](http://localhost:8000/docs) -- **Electric-SQL**: [http://localhost:5133](http://localhost:5133) - -**Comandos Úteis:** - -```bash -docker logs -f surfsense # Ver logs -docker stop surfsense # Parar -docker start surfsense # Iniciar -docker rm surfsense # Remover (dados preservados no volume) -``` - -### Opções de Instalação - -O SurfSense oferece múltiplas opções para começar: - -1. **[SurfSense Cloud](https://www.surfsense.com/login)** - A forma mais fácil de experimentar o SurfSense sem nenhuma configuração. - - Sem necessidade de instalação - - Acesso instantâneo a todas as funcionalidades - - Perfeito para começar rapidamente - -2. **Início Rápido Docker (Acima)** - Um único comando para ter o SurfSense rodando localmente. - - Imagem tudo-em-um com PostgreSQL, Redis e todos os serviços incluídos - - Perfeito para avaliação, desenvolvimento e implantações pequenas - - Dados persistidos via volume Docker - -3. **[Docker Compose (Produção)](https://www.surfsense.com/docs/docker-installation)** - Implantação de stack completo com serviços separados. - - Inclui pgAdmin para gerenciamento de banco de dados via interface web - - Suporta personalização de variáveis de ambiente via arquivo `.env` - - Opções de implantação flexíveis (stack completo ou apenas serviços principais) - - Melhor para produção com escalamento independente de serviços - -4. **[Instalação Manual](https://www.surfsense.com/docs/manual-installation)** - Para usuários que preferem mais controle sobre sua configuração ou precisam personalizar sua implantação. - -Os guias de Docker e instalação manual incluem instruções detalhadas específicas para Windows, macOS e Linux. - -Antes da instalação auto-hospedada, certifique-se de completar os [passos de configuração prévia](https://www.surfsense.com/docs/) incluindo: -- Configuração de autenticação (opcional - padrão é autenticação LOCAL) -- **Serviço ETL de Processamento de Arquivos** (opcional - padrão é Docling): - - Docling (padrão, processamento local, sem necessidade de chave API, suporta PDF, documentos Office, imagens, HTML, CSV) - - Chave API do Unstructured.io (suporta 34+ formatos) - - Chave API do LlamaIndex (análise aprimorada, suporta 50+ formatos) -- Outras chaves API conforme necessário para seu caso de uso +**Quadro Kanban:** [SurfSense Project Board](https://github.com/users/MODSetter/projects/3) ## Contribuir -Contribuições são muito bem-vindas! Uma contribuição pode ser tão pequena quanto uma ⭐ ou até mesmo encontrar e criar issues. -O ajuste fino do Backend é sempre desejado. +Todas as contribuições são bem-vindas, desde estrelas e relatórios de bugs até melhorias no backend. Consulte [CONTRIBUTING.md](CONTRIBUTING.md) para começar. -Para diretrizes detalhadas de contribuição, consulte nosso arquivo [CONTRIBUTING.md](CONTRIBUTING.md). +Obrigado a todos os nossos Surfers: + + + + ## Histórico de Stars diff --git a/README.zh-CN.md b/README.zh-CN.md index 94f8321d7..287b2fc17 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -15,6 +15,9 @@ [English](README.md) | [Español](README.es.md) | [Português](README.pt-BR.md) | [हिन्दी](README.hi.md) | [简体中文](README.zh-CN.md) + +
+MODSetter%2FSurfSense | Trendshift
# SurfSense @@ -22,101 +25,122 @@ SurfSense 是一个高度可定制的 AI 研究助手,可以连接外部数据源,如搜索引擎(SearxNG、Tavily、LinkUp)、Google Drive、Slack、Microsoft Teams、Linear、Jira、ClickUp、Confluence、BookStack、Gmail、Notion、YouTube、GitHub、Discord、Airtable、Google Calendar、Luma、Circleback、Elasticsearch、Obsidian 等,未来还会支持更多。 -
-MODSetter%2FSurfSense | Trendshift -
-# 视频演示 +# 视频 https://github.com/user-attachments/assets/cc0c84d3-1f2f-4f7a-b519-2ecce22310b1 - ## 播客示例 https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7 +## 如何使用 SurfSense +### Cloud + +1. 访问 [surfsense.com](https://www.surfsense.com) 并登录。 + +

登录

+ +2. 连接您的连接器并同步。启用定期同步以保持连接器数据更新。 + +

连接器

+ +3. 在连接器数据索引期间,上传文档。 + +

上传文档

+ +4. 一切索引完成后,尽管提问(使用场景): + + - 基本搜索和引用 + +

搜索和引用

+ + - 文档提及问答 + +

文档提及问答

+ + - 报告生成和导出(目前支持 PDF、DOCX) + +

报告生成

+ + - 播客生成 + +

播客生成

+ + - 图像生成 + +

图像生成

+ + - 更多功能即将推出。 + + +### 自托管 + +在您自己的基础设施上运行 SurfSense,实现完全的数据控制和隐私保护。 + +**快速开始(Docker 一行命令):** + +```bash +docker run -d -p 3000:3000 -p 8000:8000 -p 5133:5133 \ + -v surfsense-data:/data \ + --name surfsense \ + --restart unless-stopped \ + ghcr.io/modsetter/surfsense:latest +``` + +启动后,在浏览器中打开 [http://localhost:3000](http://localhost:3000)。 + +如需 Docker Compose、手动安装及其他部署方式,请查看[文档](https://www.surfsense.com/docs/)。 + +### 如何实时协作(Beta) + +1. 前往成员管理页面并创建邀请。 + +

邀请成员

+ +2. 队友加入后,该 SearchSpace 变为共享。 + +

邀请加入流程

+ +3. 将聊天设为共享。 + +

设为共享聊天

+ +4. 您的团队现在可以实时聊天。 + +

实时聊天

+ +5. 添加评论以标记队友。 + +

实时评论

## 核心功能 -### 💡 **理念**: -- NotebookLM、Perplexity 和 Glean 的开源替代方案。将任何 LLM 连接到您的内部知识源,并与团队实时协作。 -### 📁 **支持多种文件格式上传** -- 将您个人文件中的内容(文档、图像、视频,支持 **50+ 种文件扩展名**)保存到您自己的个人知识库。 -### 🔍 **强大的搜索功能** -- 快速研究或查找已保存内容中的任何信息。 -### 💬 **与已保存内容对话** -- 使用自然语言交互并获得引用答案。 -### 📄 **引用答案** -- 像 Perplexity 一样获得带引用的答案。 -### 🧩 **通用兼容性** -- 通过 OpenAI 规范和 LiteLLM 连接几乎任何推理提供商。 -### 🔔 **隐私保护与本地 LLM 支持** -- 完美支持 vLLM 和 Ollama 等本地大语言模型。 -### 🏠 **可自托管** -- 开源且易于本地部署。 -### 👥 **团队协作与 RBAC** -- 搜索空间的基于角色的访问控制 -- 使用可自定义的角色(所有者、管理员、编辑者、查看者)邀请团队成员 -- 对文档、聊天、连接器和设置的细粒度权限控制 -- 在组织内安全共享知识库 -- 团队聊天实时更新,支持评论线程中的"关于聊天的讨论" -### 🎙️ 播客功能 -- 超快速播客生成代理(在 20 秒内创建 3 分钟播客) -- 将聊天对话转换为引人入胜的音频内容 -- 支持本地 TTS 提供商(Kokoro TTS) -- 支持多个 TTS 提供商(OpenAI、Azure、Google Vertex AI) +| 功能 | 描述 | +|------|------| +| 开源替代方案 | 支持实时团队协作的 NotebookLM、Perplexity 和 Glean 替代品 | +| 50+ 文件格式 | 通过 LlamaCloud、Unstructured 或 Docling(本地)上传文档、图像、视频 | +| 混合搜索 | 语义搜索 + 全文搜索,结合层次化索引和倒数排名融合 | +| 引用回答 | 与知识库对话,获得 Perplexity 风格的引用回答 | +| 深度代理架构 | 基于 [LangChain Deep Agents](https://docs.langchain.com/oss/python/deepagents/overview) 构建,支持规划、子代理和文件系统访问 | +| 通用 LLM 支持 | 100+ LLM、6000+ 嵌入模型、所有主流重排序器,通过 OpenAI spec 和 LiteLLM | +| 隐私优先 | 完整本地 LLM 支持(vLLM、Ollama),您的数据由您掌控 | +| 团队协作 | RBAC 角色控制(所有者/管理员/编辑者/查看者),实时聊天和评论线程 | +| 播客生成 | 20 秒内生成 3 分钟播客;多种 TTS 提供商(OpenAI、Azure、Kokoro) | +| 浏览器扩展 | 跨浏览器扩展,保存任何网页,包括需要身份验证的页面 | +| 25+ 连接器 | 搜索引擎、Google Drive、Slack、Teams、Jira、Notion、GitHub、Discord 等[更多](#外部数据源) | +| 可自托管 | 开源,Docker 一行命令或完整 Docker Compose 用于生产环境 | -### 🤖 **深度代理架构** -- 基于 [LangChain Deep Agents](https://docs.langchain.com/oss/python/deepagents/overview) 构建 - 支持规划、子代理和文件系统的复杂任务处理代理。 +
+外部数据源完整列表 + -### 📊 **先进的 RAG 技术** -- 支持 100+ 种大语言模型 -- 支持 6000+ 种嵌入模型 -- 支持所有主流重排序器(Pinecone、Cohere、Flashrank 等) -- 使用层次化索引(2 层 RAG 设置) -- 利用混合搜索(语义搜索 + 全文搜索,结合倒数排名融合) - -### ℹ️ **外部数据源** -- 搜索引擎(Tavily、LinkUp) -- SearxNG(自托管实例) -- Google Drive -- Slack -- Microsoft Teams -- Linear -- Jira -- ClickUp -- Confluence -- BookStack -- Notion -- Gmail -- YouTube 视频 -- GitHub -- Discord -- Airtable -- Google Calendar -- Luma -- Circleback -- Elasticsearch -- Obsidian -- 更多即将推出...... - -## 📄 **支持的文件扩展名** - -| ETL 服务 | 格式 | 说明 | -|----------|------|------| -| **LlamaCloud** | 50+ 种格式 | 文档、演示文稿、电子表格、图像 | -| **Unstructured** | 34+ 种格式 | 核心格式 + 电子邮件支持 | -| **Docling** | 核心格式 | 本地处理,无需 API 密钥 | - -**音频/视频**(通过 STT 服务):`.mp3`、`.wav`、`.mp4`、`.webm` 等 - -### 🔖 跨浏览器扩展 -- SurfSense 扩展可用于保存您喜欢的任何网页。 -- 主要用途是保存需要身份验证的受保护网页。 +搜索引擎(Tavily、LinkUp)· SearxNG · Google Drive · Slack · Microsoft Teams · Linear · Jira · ClickUp · Confluence · BookStack · Notion · Gmail · YouTube 视频 · GitHub · Discord · Airtable · Google Calendar · Luma · Circleback · Elasticsearch · Obsidian,更多即将推出。 +
## 功能请求与未来规划 @@ -126,119 +150,25 @@ https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7 加入 [SurfSense Discord](https://discord.gg/ejRNvftDp9) 一起塑造 SurfSense 的未来! -## 🚀 路线图 +## 路线图 随时了解我们的开发进度和即将推出的功能! 查看我们的公开路线图并贡献您的想法或反馈: -**📋 路线图讨论:** [SurfSense 2025-2026 路线图:深度代理、实时协作与 MCP 服务器](https://github.com/MODSetter/SurfSense/discussions/565) +**路线图讨论:** [SurfSense 2026 Roadmap](https://github.com/MODSetter/SurfSense/discussions/565) -**📊 看板:** [SurfSense 项目看板](https://github.com/users/MODSetter/projects/3) - - -## 如何开始? - -### 使用 Docker 快速开始 🐳 - -> [!TIP] -> 对于生产部署,请使用完整的 [Docker Compose 设置](https://www.surfsense.com/docs/docker-installation),它提供更多控制和可扩展性。 - -**Linux/macOS:** - -```bash -docker run -d -p 3000:3000 -p 8000:8000 -p 5133:5133 \ - -v surfsense-data:/data \ - --name surfsense \ - --restart unless-stopped \ - ghcr.io/modsetter/surfsense:latest -``` - -**Windows (PowerShell):** - -```powershell -docker run -d -p 3000:3000 -p 8000:8000 -p 5133:5133 ` - -v surfsense-data:/data ` - --name surfsense ` - --restart unless-stopped ` - ghcr.io/modsetter/surfsense:latest -``` - -**使用自定义配置:** - -您可以使用 `-e` 标志传递任何环境变量: - -```bash -docker run -d -p 3000:3000 -p 8000:8000 -p 5133:5133 \ - -v surfsense-data:/data \ - -e EMBEDDING_MODEL=openai://text-embedding-ada-002 \ - -e OPENAI_API_KEY=your_openai_api_key \ - -e AUTH_TYPE=GOOGLE \ - -e GOOGLE_OAUTH_CLIENT_ID=your_google_client_id \ - -e GOOGLE_OAUTH_CLIENT_SECRET=your_google_client_secret \ - -e ETL_SERVICE=LLAMACLOUD \ - -e LLAMA_CLOUD_API_KEY=your_llama_cloud_key \ - --name surfsense \ - --restart unless-stopped \ - ghcr.io/modsetter/surfsense:latest -``` - -> [!NOTE] -> - 如果部署在带有 HTTPS 的反向代理后面,请添加 `-e BACKEND_URL=https://api.yourdomain.com` - -启动后,访问 SurfSense: -- **前端**: [http://localhost:3000](http://localhost:3000) -- **后端 API**: [http://localhost:8000](http://localhost:8000) -- **API 文档**: [http://localhost:8000/docs](http://localhost:8000/docs) -- **Electric-SQL**: [http://localhost:5133](http://localhost:5133) - -**常用命令:** - -```bash -docker logs -f surfsense # 查看日志 -docker stop surfsense # 停止 -docker start surfsense # 启动 -docker rm surfsense # 删除(数据保留在卷中) -``` - -### 安装选项 - -SurfSense 提供多种入门方式: - -1. **[SurfSense Cloud](https://www.surfsense.com/login)** - 无需任何设置即可试用 SurfSense 的最简单方法。 - - 无需安装 - - 即时访问所有功能 - - 非常适合快速上手 - -2. **快速启动 Docker(上述方法)** - 一条命令即可在本地运行 SurfSense。 - - 一体化镜像,捆绑 PostgreSQL、Redis 和所有服务 - - 非常适合评估、开发和小型部署 - - 数据通过 Docker 卷持久化 - -3. **[Docker Compose(生产环境)](https://www.surfsense.com/docs/docker-installation)** - 使用独立服务进行完整堆栈部署。 - - 包含 pgAdmin,通过 Web UI 进行数据库管理 - - 支持通过 `.env` 文件自定义环境变量 - - 灵活的部署选项(完整堆栈或仅核心服务) - - 更适合生产环境,支持独立扩展服务 - -4. **[手动安装](https://www.surfsense.com/docs/manual-installation)** - 适合希望对设置有更多控制或需要自定义部署的用户。 - -Docker 和手动安装指南都包含适用于 Windows、macOS 和 Linux 的详细操作系统特定说明。 - -在自托管安装之前,请确保完成[先决条件设置步骤](https://www.surfsense.com/docs/),包括: -- 身份验证设置(可选 - 默认为 LOCAL 身份验证) -- **文件处理 ETL 服务**(可选 - 默认为 Docling): - - Docling(默认,本地处理,无需 API 密钥,支持 PDF、Office 文档、图像、HTML、CSV) - - Unstructured.io API 密钥(支持 34+ 种格式) - - LlamaIndex API 密钥(增强解析,支持 50+ 种格式) -- 其他根据用例需要的 API 密钥 +**看板:** [SurfSense Project Board](https://github.com/users/MODSetter/projects/3) ## 贡献 -非常欢迎贡献!贡献可以小到一个 ⭐,甚至是发现和创建问题。 -后端的微调总是受欢迎的。 +欢迎所有贡献,从 Star 和 Bug 报告到后端改进。请参阅 [CONTRIBUTING.md](CONTRIBUTING.md) 开始贡献。 -有关详细的贡献指南,请参阅我们的 [CONTRIBUTING.md](CONTRIBUTING.md) 文件。 +感谢所有 Surfers: + + + + ## Star 历史 diff --git a/docker-compose.yml b/docker-compose.yml index 8efc0b506..a94cea2e5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -53,6 +53,7 @@ services: - DATABASE_URL=postgresql+asyncpg://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@db:5432/${POSTGRES_DB:-surfsense} - CELERY_BROKER_URL=redis://redis:${REDIS_PORT:-6379}/0 - CELERY_RESULT_BACKEND=redis://redis:${REDIS_PORT:-6379}/0 + - REDIS_APP_URL=redis://redis:${REDIS_PORT:-6379}/0 # Queue name isolation - prevents task collision if Redis is shared with other apps - CELERY_TASK_DEFAULT_QUEUE=surfsense - PYTHONPATH=/app @@ -62,7 +63,8 @@ services: - LANGSMITH_TRACING=false - ELECTRIC_DB_USER=${ELECTRIC_DB_USER:-electric} - ELECTRIC_DB_PASSWORD=${ELECTRIC_DB_PASSWORD:-electric_password} - - NEXT_FRONTEND_URL=http://frontend:3000 + - AUTH_TYPE=${AUTH_TYPE:-LOCAL} + - NEXT_FRONTEND_URL=${NEXT_FRONTEND_URL:-http://localhost:3000} depends_on: - db - redis diff --git a/scripts/docker/entrypoint-allinone.sh b/scripts/docker/entrypoint-allinone.sh index 4cbb34761..a6062b32c 100644 --- a/scripts/docker/entrypoint-allinone.sh +++ b/scripts/docker/entrypoint-allinone.sh @@ -168,6 +168,7 @@ NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE="${NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE:- NEXT_PUBLIC_ETL_SERVICE="${NEXT_PUBLIC_ETL_SERVICE:-DOCLING}" NEXT_PUBLIC_ELECTRIC_URL="${NEXT_PUBLIC_ELECTRIC_URL:-http://localhost:5133}" NEXT_PUBLIC_ELECTRIC_AUTH_MODE="${NEXT_PUBLIC_ELECTRIC_AUTH_MODE:-insecure}" +NEXT_PUBLIC_DEPLOYMENT_MODE="${NEXT_PUBLIC_DEPLOYMENT_MODE:-self-hosted}" # Replace placeholders in all JS files find /app/frontend -type f \( -name "*.js" -o -name "*.json" \) -exec sed -i \ @@ -176,13 +177,15 @@ find /app/frontend -type f \( -name "*.js" -o -name "*.json" \) -exec sed -i \ -e "s|__NEXT_PUBLIC_ETL_SERVICE__|${NEXT_PUBLIC_ETL_SERVICE}|g" \ -e "s|__NEXT_PUBLIC_ELECTRIC_URL__|${NEXT_PUBLIC_ELECTRIC_URL}|g" \ -e "s|__NEXT_PUBLIC_ELECTRIC_AUTH_MODE__|${NEXT_PUBLIC_ELECTRIC_AUTH_MODE}|g" \ + -e "s|__NEXT_PUBLIC_DEPLOYMENT_MODE__|${NEXT_PUBLIC_DEPLOYMENT_MODE}|g" \ {} + echo "✅ Environment configuration applied" -echo " Backend URL: ${NEXT_PUBLIC_FASTAPI_BACKEND_URL}" -echo " Auth Type: ${NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE}" -echo " ETL Service: ${NEXT_PUBLIC_ETL_SERVICE}" -echo " Electric URL: ${NEXT_PUBLIC_ELECTRIC_URL}" +echo " Backend URL: ${NEXT_PUBLIC_FASTAPI_BACKEND_URL}" +echo " Auth Type: ${NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE}" +echo " ETL Service: ${NEXT_PUBLIC_ETL_SERVICE}" +echo " Electric URL: ${NEXT_PUBLIC_ELECTRIC_URL}" +echo " Deployment Mode: ${NEXT_PUBLIC_DEPLOYMENT_MODE}" # ================================================ # Run database migrations diff --git a/scripts/docker/supervisor-allinone.conf b/scripts/docker/supervisor-allinone.conf index eb2404b3c..1a21fcc04 100644 --- a/scripts/docker/supervisor-allinone.conf +++ b/scripts/docker/supervisor-allinone.conf @@ -57,7 +57,7 @@ environment=PYTHONPATH="/app/backend",UVICORN_LOOP="asyncio",UNSTRUCTURED_HAS_PA # Celery Worker [program:celery-worker] -command=celery -A app.celery_app worker --loglevel=info --concurrency=2 --pool=solo +command=celery -A app.celery_app worker --loglevel=info --concurrency=2 --pool=solo --queues=surfsense,surfsense.connectors directory=/app/backend autostart=true autorestart=true diff --git a/surfsense_backend/Dockerfile b/surfsense_backend/Dockerfile index 446479ece..4f24d2b05 100644 --- a/surfsense_backend/Dockerfile +++ b/surfsense_backend/Dockerfile @@ -22,6 +22,14 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ git \ && rm -rf /var/lib/apt/lists/* +# Install Pandoc 3.x from GitHub as a fallback for Linux where pypandoc_binary +# may not bundle pandoc (apt ships 2.17 which has broken table rendering). +# pypandoc_binary bundles pandoc on Windows/macOS; on Linux it picks this up. +RUN ARCH=$(dpkg --print-architecture) && \ + wget -qO /tmp/pandoc.deb "https://github.com/jgm/pandoc/releases/download/3.9/pandoc-3.9-1-${ARCH}.deb" && \ + dpkg -i /tmp/pandoc.deb && \ + rm /tmp/pandoc.deb + # Update certificates and install SSL tools RUN update-ca-certificates RUN pip install --upgrade certifi pip-system-certs @@ -99,4 +107,4 @@ ENV CELERY_QUEUES="" # Run EXPOSE 8000-8001 -CMD ["/app/scripts/docker/entrypoint.sh"] \ No newline at end of file +CMD ["/app/scripts/docker/entrypoint.sh"] \ No newline at end of file diff --git a/surfsense_backend/alembic/versions/100_add_report_group_id.py b/surfsense_backend/alembic/versions/100_add_report_group_id.py new file mode 100644 index 000000000..e99ab0c83 --- /dev/null +++ b/surfsense_backend/alembic/versions/100_add_report_group_id.py @@ -0,0 +1,69 @@ +"""Add report_group_id for report versioning + +Revision ID: 100 +Revises: 99 +Create Date: 2026-02-11 + +Adds report_group_id column to reports table for grouping report versions. +Reports with the same report_group_id are versions of the same report. +For the first version (v1), report_group_id equals the report's own id. +Migration is idempotent — safe to re-run. +""" + +from collections.abc import Sequence + +from alembic import op + +revision: str = "100" +down_revision: str | None = "99" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + # Add report_group_id column (idempotent) + op.execute( + """ + DO $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_name = 'reports' AND column_name = 'report_group_id' + ) THEN + ALTER TABLE reports ADD COLUMN report_group_id INTEGER; + END IF; + END $$; + """ + ) + + # Backfill existing reports: set report_group_id = id (each is its own v1) + op.execute( + """ + UPDATE reports SET report_group_id = id WHERE report_group_id IS NULL; + """ + ) + + # Create index (idempotent) + op.execute( + """ + CREATE INDEX IF NOT EXISTS ix_reports_report_group_id + ON reports(report_group_id); + """ + ) + + +def downgrade() -> None: + op.execute("DROP INDEX IF EXISTS ix_reports_report_group_id") + op.execute( + """ + DO $$ + BEGIN + IF EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_name = 'reports' AND column_name = 'report_group_id' + ) THEN + ALTER TABLE reports DROP COLUMN report_group_id; + END IF; + END $$; + """ + ) diff --git a/surfsense_backend/alembic/versions/99_add_reports_table.py b/surfsense_backend/alembic/versions/99_add_reports_table.py new file mode 100644 index 000000000..663e6869b --- /dev/null +++ b/surfsense_backend/alembic/versions/99_add_reports_table.py @@ -0,0 +1,66 @@ +"""Add reports table + +Revision ID: 99 +Revises: 98 +Create Date: 2026-02-11 + +Adds reports table for storing generated Markdown reports. +""" + +from collections.abc import Sequence + +from alembic import op + +revision: str = "99" +down_revision: str | None = "98" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + # Create the reports table + op.execute( + """ + CREATE TABLE IF NOT EXISTS reports ( + id SERIAL PRIMARY KEY, + title VARCHAR(500) NOT NULL, + content TEXT, + report_metadata JSONB, + report_style VARCHAR(100), + search_space_id INTEGER NOT NULL + REFERENCES searchspaces(id) ON DELETE CASCADE, + thread_id INTEGER + REFERENCES new_chat_threads(id) ON DELETE SET NULL, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() + ); + """ + ) + + # Add indexes + op.execute( + """ + CREATE INDEX IF NOT EXISTS ix_reports_search_space_id + ON reports(search_space_id); + """ + ) + + op.execute( + """ + CREATE INDEX IF NOT EXISTS ix_reports_thread_id + ON reports(thread_id); + """ + ) + + op.execute( + """ + CREATE INDEX IF NOT EXISTS ix_reports_created_at + ON reports(created_at); + """ + ) + + +def downgrade() -> None: + op.execute("DROP INDEX IF EXISTS ix_reports_created_at") + op.execute("DROP INDEX IF EXISTS ix_reports_thread_id") + op.execute("DROP INDEX IF EXISTS ix_reports_search_space_id") + op.execute("DROP TABLE IF EXISTS reports") diff --git a/surfsense_backend/app/agents/new_chat/chat_deepagent.py b/surfsense_backend/app/agents/new_chat/chat_deepagent.py index d58a0fadb..3ad7aafc6 100644 --- a/surfsense_backend/app/agents/new_chat/chat_deepagent.py +++ b/surfsense_backend/app/agents/new_chat/chat_deepagent.py @@ -243,11 +243,24 @@ async def create_surfsense_deep_agent( "available_document_types": available_document_types, } + # Disable Notion action tools if no Notion connector is configured + modified_disabled_tools = list(disabled_tools) if disabled_tools else [] + has_notion_connector = ( + available_connectors is not None and "NOTION_CONNECTOR" in available_connectors + ) + if not has_notion_connector: + notion_tools = [ + "create_notion_page", + "update_notion_page", + "delete_notion_page", + ] + modified_disabled_tools.extend(notion_tools) + # Build tools using the async registry (includes MCP tools) tools = await build_tools_async( dependencies=dependencies, enabled_tools=enabled_tools, - disabled_tools=disabled_tools, + disabled_tools=modified_disabled_tools, additional_tools=list(additional_tools) if additional_tools else None, ) diff --git a/surfsense_backend/app/agents/new_chat/system_prompt.py b/surfsense_backend/app/agents/new_chat/system_prompt.py index a3520dad6..14dfe1b2e 100644 --- a/surfsense_backend/app/agents/new_chat/system_prompt.py +++ b/surfsense_backend/app/agents/new_chat/system_prompt.py @@ -21,6 +21,8 @@ You are SurfSense, a reasoning and acting AI agent designed to answer user quest Today's date (UTC): {resolved_today} +When writing mathematical formulas or equations, ALWAYS use LaTeX notation. NEVER use backtick code spans or Unicode symbols for math. + """ @@ -33,6 +35,8 @@ In this team thread, each message is prefixed with **[DisplayName of the author] Today's date (UTC): {resolved_today} +When writing mathematical formulas or equations, ALWAYS use LaTeX notation. NEVER use backtick code spans or Unicode symbols for math. + """ @@ -50,7 +54,7 @@ def _get_system_instructions( return SURFSENSE_SYSTEM_INSTRUCTIONS.format(resolved_today=resolved_today) -# Tools 0-6 (common to both private and shared prompts) +# Tools 0-7 (common to both private and shared prompts) _TOOLS_INSTRUCTIONS_COMMON = """ You have access to the following tools: @@ -92,7 +96,44 @@ You have access to the following tools: - IMPORTANT: Only one podcast can be generated at a time. If a podcast is already being generated, the tool will return status "already_generating". - After calling this tool, inform the user that podcast generation has started and they will see the player when it's ready (takes 3-5 minutes). -3. link_preview: Fetch metadata for a URL to display a rich preview card. +3. generate_report: Generate a structured Markdown report from provided content. + - Use this when the user asks to create, generate, write, produce, draft, or summarize into a report-style deliverable. + - DECISION RULE (HIGH PRIORITY): If the user asks for a report in any form, call `generate_report` instead of writing the full report directly in chat. + - Only skip `generate_report` if the user explicitly asks for chat-only output (e.g., "just answer in chat", "no report card", "don't generate a report"). + - Trigger classes include: + * Direct trigger words: report, document, memo, letter, template + * Creation-intent phrases: "write a document/report/post/article" + * File-intent words: requests containing "save", "file", or "document" when intent is to create a report-like deliverable + * Word-doc specific triggers: professional report-style deliverable, professional document, Word doc, .docx + * Other report-like output intents: one-pager, blog post, article, standalone written content, comprehensive guide + * General artifact-style intents: analysis / writing as substantial deliverables + - Trigger phrases include: + * "generate a report about", "write a report", "produce a report" + * "create a detailed report about", "make a research report on" + * "summarize this into a report", "turn this into a report" + * "write a report/document", "draft a report" + * "create an executive summary", "make a briefing note", "write a one-pager" + * "write a blog post", "write an article", "create a comprehensive guide" + * "create a small report", "write a short report", "make a quick report", "brief report for class" + - IMPORTANT FORMAT RULE: Reports are ALWAYS generated in Markdown. + - Args: + - topic: The main topic or title of the report + - source_content: The text content to base the report on. This MUST be comprehensive and include: + * If discussing the current conversation: Include a detailed summary of the FULL chat history (all user questions and your responses) + * If based on knowledge base search: Include the key findings and insights from the search results + * You can combine both: conversation context + search results for richer reports + * The more detailed the source_content, the better the report quality + - report_style: Optional style. Options: "detailed" (default), "executive_summary", "deep_research", "brief" + - user_instructions: Optional specific instructions (e.g., "focus on financial impacts", "include recommendations") + - Returns: A dictionary with status "ready" or "failed", report_id, title, and word_count. + - The report is generated immediately in Markdown and displayed inline in the chat. + - Export/download formats (e.g., PDF/DOCX) are produced from the generated Markdown report. + - SOURCE-COLLECTION RULE: + * If the user already provided enough source material (current chat content, uploaded files, pasted text, or a summarized video/article), generate the report directly from that. + * Use search_knowledge_base first when additional context is needed or the user asks for information beyond what is already available in the conversation. + - AFTER CALLING THIS TOOL: Do NOT repeat, summarize, or reproduce the report content in the chat. The report is already displayed as an interactive card that the user can open, read, copy, and export. Simply confirm that the report was generated (e.g., "I've generated your report on [topic]. You can view the Markdown report now, and export to PDF/DOCX from the card."). NEVER write out the report text in the chat. + +4. link_preview: Fetch metadata for a URL to display a rich preview card. - IMPORTANT: Use this tool WHENEVER the user shares or mentions a URL/link in their message. - This fetches the page's Open Graph metadata (title, description, thumbnail) to show a preview card. - NOTE: This tool only fetches metadata, NOT the full page content. It cannot read the article text. @@ -105,7 +146,7 @@ You have access to the following tools: - Returns: A rich preview card with title, description, thumbnail, and domain - The preview card will automatically be displayed in the chat. -4. display_image: Display an image in the chat with metadata. +5. display_image: Display an image in the chat with metadata. - Use this tool ONLY when you have a valid public HTTP/HTTPS image URL to show. - This displays the image with an optional title, description, and source attribution. - Valid use cases: @@ -130,7 +171,7 @@ You have access to the following tools: - Returns: An image card with the image, title, and description - The image will automatically be displayed in the chat. -5. generate_image: Generate images from text descriptions using AI image models. +6. generate_image: Generate images from text descriptions using AI image models. - Use this when the user asks you to create, generate, draw, design, or make an image. - Trigger phrases: "generate an image of", "create a picture of", "draw me", "make an image", "design a logo", "create artwork" - Args: @@ -144,7 +185,7 @@ You have access to the following tools: expand and improve the prompt with specific details about style, lighting, composition, and mood. - If the user's request is vague (e.g., "make me an image of a cat"), enhance the prompt with artistic details. -6. scrape_webpage: Scrape and extract the main content from a webpage. +7. scrape_webpage: Scrape and extract the main content from a webpage. - Use this when the user wants you to READ and UNDERSTAND the actual content of a webpage. - IMPORTANT: This is different from link_preview: * link_preview: Only fetches metadata (title, description, thumbnail) for display @@ -169,9 +210,9 @@ You have access to the following tools: """ -# Private (user) memory: tools 7-8 + memory-specific examples +# Private (user) memory: tools 8-9 + memory-specific examples _TOOLS_INSTRUCTIONS_MEMORY_PRIVATE = """ -7. save_memory: Save facts, preferences, or context for personalized responses. +8. save_memory: Save facts, preferences, or context for personalized responses. - Use this when the user explicitly or implicitly shares information worth remembering. - Trigger scenarios: * User says "remember this", "keep this in mind", "note that", or similar @@ -194,7 +235,7 @@ _TOOLS_INSTRUCTIONS_MEMORY_PRIVATE = """ - IMPORTANT: Only save information that would be genuinely useful for future conversations. Don't save trivial or temporary information. -8. recall_memory: Retrieve relevant memories about the user for personalized responses. +9. recall_memory: Retrieve relevant memories about the user for personalized responses. - Use this to access stored information about the user. - Trigger scenarios: * You need user context to give a better, more personalized answer @@ -232,7 +273,7 @@ _TOOLS_INSTRUCTIONS_MEMORY_PRIVATE = """ # Shared (team) memory: tools 7-8 + team memory examples _TOOLS_INSTRUCTIONS_MEMORY_SHARED = """ -7. save_memory: Save a fact, preference, or context to the team's shared memory for future reference. +8. save_memory: Save a fact, preference, or context to the team's shared memory for future reference. - Use this when the user or a team member says "remember this", "keep this in mind", or similar in this shared chat. - Use when the team agrees on something to remember (e.g., decisions, conventions). - Someone shares a preference or fact that should be visible to the whole team. @@ -247,7 +288,7 @@ _TOOLS_INSTRUCTIONS_MEMORY_SHARED = """ - Returns: Confirmation of saved memory; returned context may include who added it (added_by). - IMPORTANT: Only save information that would be genuinely useful for future team conversations in this space. -8. recall_memory: Recall relevant team memories for this space to provide contextual responses. +9. recall_memory: Recall relevant team memories for this space to provide contextual responses. - Use when you need team context to answer (e.g., "where do we store X?", "what did we decide about Y?"). - Use when someone asks about something the team agreed to remember. - Use when team preferences or conventions would improve the response. @@ -321,6 +362,17 @@ _TOOLS_INSTRUCTIONS_EXAMPLES_COMMON = """ - First search: `search_knowledge_base(query="quantum computing")` - Then: `generate_podcast(source_content="Key insights about quantum computing from the knowledge base:\\n\\n[Comprehensive summary of all relevant search results with key facts, concepts, and findings]", podcast_title="Quantum Computing Explained")` +- User: "Generate a report about AI trends" + - First search: `search_knowledge_base(query="AI trends")` + - Then: `generate_report(topic="AI Trends Report", source_content="Key insights about AI trends from the knowledge base:\\n\\n[Comprehensive summary of all relevant search results with key facts, concepts, and findings]", report_style="detailed")` + +- User: "Write a research report from this conversation" + - Call: `generate_report(topic="Research Report", source_content="Complete conversation summary:\\n\\nUser asked about [topic 1]:\\n[Your detailed response]\\n\\nUser then asked about [topic 2]:\\n[Your detailed response]\\n\\n[Continue for all exchanges in the conversation]", report_style="deep_research")` + +- User: "Create a brief executive summary about our project progress" + - First search: `search_knowledge_base(query="project progress updates")` + - Then: `generate_report(topic="Project Progress Executive Summary", source_content="[Combined search results and conversation context]", report_style="executive_summary", user_instructions="Focus on milestones achieved and upcoming deadlines")` + - User: "Check out https://dev.to/some-article" - Call: `link_preview(url="https://dev.to/some-article")` - Call: `scrape_webpage(url="https://dev.to/some-article")` diff --git a/surfsense_backend/app/agents/new_chat/tools/notion/__init__.py b/surfsense_backend/app/agents/new_chat/tools/notion/__init__.py new file mode 100644 index 000000000..6ce825dca --- /dev/null +++ b/surfsense_backend/app/agents/new_chat/tools/notion/__init__.py @@ -0,0 +1,11 @@ +"""Notion tools for creating, updating, and deleting pages.""" + +from .create_page import create_create_notion_page_tool +from .delete_page import create_delete_notion_page_tool +from .update_page import create_update_notion_page_tool + +__all__ = [ + "create_create_notion_page_tool", + "create_delete_notion_page_tool", + "create_update_notion_page_tool", +] diff --git a/surfsense_backend/app/agents/new_chat/tools/notion/create_page.py b/surfsense_backend/app/agents/new_chat/tools/notion/create_page.py new file mode 100644 index 000000000..ce00025a4 --- /dev/null +++ b/surfsense_backend/app/agents/new_chat/tools/notion/create_page.py @@ -0,0 +1,239 @@ +import logging +from typing import Any + +from langchain_core.tools import tool +from langgraph.types import interrupt +from sqlalchemy.ext.asyncio import AsyncSession + +from app.connectors.notion_history import NotionHistoryConnector +from app.services.notion import NotionToolMetadataService + +logger = logging.getLogger(__name__) + + +def create_create_notion_page_tool( + db_session: AsyncSession | None = None, + search_space_id: int | None = None, + user_id: str | None = None, + connector_id: int | None = None, +): + """ + Factory function to create the create_notion_page tool. + + Args: + db_session: Database session for accessing Notion connector + search_space_id: Search space ID to find the Notion connector + user_id: User ID for fetching user-specific context + connector_id: Optional specific connector ID (if known) + + Returns: + Configured create_notion_page tool + """ + + @tool + async def create_notion_page( + title: str, + content: str, + parent_page_id: str | None = None, + ) -> dict[str, Any]: + """Create a new page in Notion with the given title and content. + + Use this tool when the user asks you to create, save, or publish + something to Notion. The page will be created in the user's + configured Notion workspace. + + Args: + title: The title of the Notion page. + content: The markdown content for the page body (supports headings, lists, paragraphs). + parent_page_id: Optional parent page ID to create as a subpage. + If not provided, will ask for one. + + Returns: + Dictionary with: + - status: "success", "rejected", or "error" + - page_id: Created page ID (if success) + - url: URL to the created page (if success) + - title: Page title (if success) + - message: Result message + + IMPORTANT: If status is "rejected", the user explicitly declined the action. + Respond with a brief acknowledgment (e.g., "Understood, I didn't create the page.") + and move on. Do NOT ask for parent page IDs, troubleshoot, or suggest alternatives. + + Examples: + - "Create a Notion page titled 'Meeting Notes' with content 'Discussed project timeline'" + - "Save this to Notion with title 'Research Summary'" + """ + logger.info( + f"create_notion_page called: title='{title}', parent_page_id={parent_page_id}" + ) + + if db_session is None or search_space_id is None or user_id is None: + logger.error( + "Notion tool not properly configured - missing required parameters" + ) + return { + "status": "error", + "message": "Notion tool not properly configured. Please contact support.", + } + + try: + metadata_service = NotionToolMetadataService(db_session) + context = await metadata_service.get_creation_context( + search_space_id, user_id + ) + + if "error" in context: + logger.error(f"Failed to fetch creation context: {context['error']}") + return { + "status": "error", + "message": context["error"], + } + + logger.info(f"Requesting approval for creating Notion page: '{title}'") + approval = interrupt( + { + "type": "notion_page_creation", + "action": { + "tool": "create_notion_page", + "params": { + "title": title, + "content": content, + "parent_page_id": parent_page_id, + "connector_id": connector_id, + }, + }, + "context": context, + } + ) + + decisions_raw = ( + approval.get("decisions", []) if isinstance(approval, dict) else [] + ) + decisions = ( + decisions_raw if isinstance(decisions_raw, list) else [decisions_raw] + ) + decisions = [d for d in decisions if isinstance(d, dict)] + if not decisions: + logger.warning("No approval decision received") + return { + "status": "error", + "message": "No approval decision received", + } + + decision = decisions[0] + decision_type = decision.get("type") or decision.get("decision_type") + logger.info(f"User decision: {decision_type}") + + if decision_type == "reject": + logger.info("Notion page creation rejected by user") + return { + "status": "rejected", + "message": "User declined. The page was not created. Do not ask again or suggest alternatives.", + } + + edited_action = decision.get("edited_action") + final_params: dict[str, Any] = {} + if isinstance(edited_action, dict): + edited_args = edited_action.get("args") + if isinstance(edited_args, dict): + final_params = edited_args + elif isinstance(decision.get("args"), dict): + # Some interrupt payloads place args directly on the decision. + final_params = decision["args"] + + final_title = final_params.get("title", title) + final_content = final_params.get("content", content) + final_parent_page_id = final_params.get("parent_page_id", parent_page_id) + final_connector_id = final_params.get("connector_id", connector_id) + + if not final_title or not final_title.strip(): + logger.error("Title is empty or contains only whitespace") + return { + "status": "error", + "message": "Page title cannot be empty. Please provide a valid title.", + } + + logger.info( + f"Creating Notion page with final params: title='{final_title}'" + ) + + from sqlalchemy.future import select + + from app.db import SearchSourceConnector, SearchSourceConnectorType + + actual_connector_id = final_connector_id + if actual_connector_id is None: + result = await db_session.execute( + select(SearchSourceConnector).filter( + SearchSourceConnector.search_space_id == search_space_id, + SearchSourceConnector.user_id == user_id, + SearchSourceConnector.connector_type + == SearchSourceConnectorType.NOTION_CONNECTOR, + ) + ) + connector = result.scalars().first() + + if not connector: + logger.warning( + f"No Notion connector found for search_space_id={search_space_id}" + ) + return { + "status": "error", + "message": "No Notion connector found. Please connect Notion in your workspace settings.", + } + + actual_connector_id = connector.id + logger.info(f"Found Notion connector: id={actual_connector_id}") + else: + result = await db_session.execute( + select(SearchSourceConnector).filter( + SearchSourceConnector.id == actual_connector_id, + SearchSourceConnector.search_space_id == search_space_id, + SearchSourceConnector.user_id == user_id, + SearchSourceConnector.connector_type + == SearchSourceConnectorType.NOTION_CONNECTOR, + ) + ) + connector = result.scalars().first() + + if not connector: + logger.error( + f"Invalid connector_id={actual_connector_id} for search_space_id={search_space_id}" + ) + return { + "status": "error", + "message": "Selected Notion account is invalid or has been disconnected. Please select a valid account.", + } + logger.info(f"Validated Notion connector: id={actual_connector_id}") + + notion_connector = NotionHistoryConnector( + session=db_session, + connector_id=actual_connector_id, + ) + + result = await notion_connector.create_page( + title=final_title, + content=final_content, + parent_page_id=final_parent_page_id, + ) + logger.info( + f"create_page result: {result.get('status')} - {result.get('message', '')}" + ) + return result + + except Exception as e: + from langgraph.errors import GraphInterrupt + + if isinstance(e, GraphInterrupt): + raise + + logger.error(f"Error creating Notion page: {e}", exc_info=True) + return { + "status": "error", + "message": str(e) + if isinstance(e, ValueError) + else f"Unexpected error: {e!s}", + } + + return create_notion_page diff --git a/surfsense_backend/app/agents/new_chat/tools/notion/delete_page.py b/surfsense_backend/app/agents/new_chat/tools/notion/delete_page.py new file mode 100644 index 000000000..065a4f9d4 --- /dev/null +++ b/surfsense_backend/app/agents/new_chat/tools/notion/delete_page.py @@ -0,0 +1,273 @@ +import logging +from typing import Any + +from langchain_core.tools import tool +from langgraph.types import interrupt +from sqlalchemy.ext.asyncio import AsyncSession + +from app.connectors.notion_history import NotionHistoryConnector +from app.services.notion.tool_metadata_service import NotionToolMetadataService + +logger = logging.getLogger(__name__) + + +def create_delete_notion_page_tool( + db_session: AsyncSession | None = None, + search_space_id: int | None = None, + user_id: str | None = None, + connector_id: int | None = None, +): + """ + Factory function to create the delete_notion_page tool. + + Args: + db_session: Database session for accessing Notion connector + search_space_id: Search space ID to find the Notion connector + user_id: User ID for finding the correct Notion connector + connector_id: Optional specific connector ID (if known) + + Returns: + Configured delete_notion_page tool + """ + + @tool + async def delete_notion_page( + page_title: str, + delete_from_db: bool = False, + ) -> dict[str, Any]: + """Delete (archive) a Notion page. + + Use this tool when the user asks you to delete, remove, or archive + a Notion page. Note that Notion doesn't permanently delete pages, + it archives them (they can be restored from trash). + + Args: + page_title: The title of the Notion page to delete. + delete_from_db: Whether to also remove the page from the knowledge base. + Default is False (in Notion). + Set to True to permanently remove from both Notion and knowledge base. + + Returns: + Dictionary with: + - status: "success", "rejected", "not_found", or "error" + - page_id: Deleted page ID (if success) + - message: Success or error message + - deleted_from_db: Whether the page was also removed from knowledge base (if success) + + Examples: + - "Delete the 'Meeting Notes' Notion page" + - "Remove the 'Old Project Plan' Notion page" + - "Archive the 'Draft Ideas' Notion page" + """ + logger.info( + f"delete_notion_page called: page_title='{page_title}', delete_from_db={delete_from_db}" + ) + + if db_session is None or search_space_id is None or user_id is None: + logger.error( + "Notion tool not properly configured - missing required parameters" + ) + return { + "status": "error", + "message": "Notion tool not properly configured. Please contact support.", + } + + try: + # Get page context (page_id, account, title) from indexed data + metadata_service = NotionToolMetadataService(db_session) + context = await metadata_service.get_delete_context( + search_space_id, user_id, page_title + ) + + if "error" in context: + error_msg = context["error"] + # Check if it's a "not found" error (softer handling for LLM) + if "not found" in error_msg.lower(): + logger.warning(f"Page not found: {error_msg}") + return { + "status": "not_found", + "message": error_msg, + } + else: + logger.error(f"Failed to fetch delete context: {error_msg}") + return { + "status": "error", + "message": error_msg, + } + + page_id = context.get("page_id") + connector_id_from_context = context.get("account", {}).get("id") + document_id = context.get("document_id") + + logger.info( + f"Requesting approval for deleting Notion page: '{page_title}' (page_id={page_id}, delete_from_db={delete_from_db})" + ) + + # Request approval before deleting + approval = interrupt( + { + "type": "notion_page_deletion", + "action": { + "tool": "delete_notion_page", + "params": { + "page_id": page_id, + "connector_id": connector_id_from_context, + "delete_from_db": delete_from_db, + }, + }, + "context": context, + } + ) + + decisions_raw = ( + approval.get("decisions", []) if isinstance(approval, dict) else [] + ) + decisions = ( + decisions_raw if isinstance(decisions_raw, list) else [decisions_raw] + ) + decisions = [d for d in decisions if isinstance(d, dict)] + if not decisions: + logger.warning("No approval decision received") + return { + "status": "error", + "message": "No approval decision received", + } + + decision = decisions[0] + decision_type = decision.get("type") or decision.get("decision_type") + logger.info(f"User decision: {decision_type}") + + if decision_type == "reject": + logger.info("Notion page deletion rejected by user") + return { + "status": "rejected", + "message": "User declined. The page was not deleted. Do not ask again or suggest alternatives.", + } + + # Extract edited action arguments (if user modified the checkbox) + edited_action = decision.get("edited_action") + final_params: dict[str, Any] = {} + if isinstance(edited_action, dict): + edited_args = edited_action.get("args") + if isinstance(edited_args, dict): + final_params = edited_args + elif isinstance(decision.get("args"), dict): + # Some interrupt payloads place args directly on the decision. + final_params = decision["args"] + + final_page_id = final_params.get("page_id", page_id) + final_connector_id = final_params.get( + "connector_id", connector_id_from_context + ) + final_delete_from_db = final_params.get("delete_from_db", delete_from_db) + + logger.info( + f"Deleting Notion page with final params: page_id={final_page_id}, connector_id={final_connector_id}, delete_from_db={final_delete_from_db}" + ) + + from sqlalchemy.future import select + + from app.db import SearchSourceConnector, SearchSourceConnectorType + + # Validate the connector + if final_connector_id: + result = await db_session.execute( + select(SearchSourceConnector).filter( + SearchSourceConnector.id == final_connector_id, + SearchSourceConnector.search_space_id == search_space_id, + SearchSourceConnector.user_id == user_id, + SearchSourceConnector.connector_type + == SearchSourceConnectorType.NOTION_CONNECTOR, + ) + ) + connector = result.scalars().first() + + if not connector: + logger.error( + f"Invalid connector_id={final_connector_id} for search_space_id={search_space_id}" + ) + return { + "status": "error", + "message": "Selected Notion account is invalid or has been disconnected. Please select a valid account.", + } + actual_connector_id = connector.id + logger.info(f"Validated Notion connector: id={actual_connector_id}") + else: + logger.error("No connector found for this page") + return { + "status": "error", + "message": "No connector found for this page.", + } + + # Create connector instance + notion_connector = NotionHistoryConnector( + session=db_session, + connector_id=actual_connector_id, + ) + + # Delete the page from Notion + result = await notion_connector.delete_page(page_id=final_page_id) + logger.info( + f"delete_page result: {result.get('status')} - {result.get('message', '')}" + ) + + # If deletion was successful and user wants to delete from DB + deleted_from_db = False + if ( + result.get("status") == "success" + and final_delete_from_db + and document_id + ): + try: + from sqlalchemy.future import select + + from app.db import Document + + # Get the document + doc_result = await db_session.execute( + select(Document).filter(Document.id == document_id) + ) + document = doc_result.scalars().first() + + if document: + await db_session.delete(document) + await db_session.commit() + deleted_from_db = True + logger.info( + f"Deleted document {document_id} from knowledge base" + ) + else: + logger.warning(f"Document {document_id} not found in DB") + except Exception as e: + logger.error(f"Failed to delete document from DB: {e}") + # Don't fail the whole operation if DB deletion fails + # The page is already deleted from Notion, so inform the user + result["warning"] = ( + f"Page deleted from Notion, but failed to remove from knowledge base: {e!s}" + ) + + # Update result with DB deletion status + if result.get("status") == "success": + result["deleted_from_db"] = deleted_from_db + if deleted_from_db: + result["message"] = ( + f"{result.get('message', '')} (also removed from knowledge base)" + ) + + return result + + except Exception as e: + from langgraph.errors import GraphInterrupt + + if isinstance(e, GraphInterrupt): + raise + + logger.error(f"Error deleting Notion page: {e}", exc_info=True) + return { + "status": "error", + "message": str(e) + if isinstance(e, ValueError) + else f"Unexpected error: {e!s}", + } + + return delete_notion_page diff --git a/surfsense_backend/app/agents/new_chat/tools/notion/update_page.py b/surfsense_backend/app/agents/new_chat/tools/notion/update_page.py new file mode 100644 index 000000000..7a58f3aad --- /dev/null +++ b/surfsense_backend/app/agents/new_chat/tools/notion/update_page.py @@ -0,0 +1,237 @@ +import logging +from typing import Any + +from langchain_core.tools import tool +from langgraph.types import interrupt +from sqlalchemy.ext.asyncio import AsyncSession + +from app.connectors.notion_history import NotionHistoryConnector +from app.services.notion import NotionToolMetadataService + +logger = logging.getLogger(__name__) + + +def create_update_notion_page_tool( + db_session: AsyncSession | None = None, + search_space_id: int | None = None, + user_id: str | None = None, + connector_id: int | None = None, +): + """ + Factory function to create the update_notion_page tool. + + Args: + db_session: Database session for accessing Notion connector + search_space_id: Search space ID to find the Notion connector + user_id: User ID for fetching user-specific context + connector_id: Optional specific connector ID (if known) + + Returns: + Configured update_notion_page tool + """ + + @tool + async def update_notion_page( + page_title: str, + content: str, + ) -> dict[str, Any]: + """Update an existing Notion page by appending new content. + + Use this tool when the user asks you to add content to, modify, or update + a Notion page. The new content will be appended to the existing page content. + + Args: + page_title: The title of the Notion page to update. + content: The markdown content to append to the page body (supports headings, lists, paragraphs). + + Returns: + Dictionary with: + - status: "success", "rejected", "not_found", or "error" + - page_id: Updated page ID (if success) + - url: URL to the updated page (if success) + - title: Current page title (if success) + - message: Result message + + IMPORTANT: + - If status is "rejected", the user explicitly declined the action. + Respond with a brief acknowledgment (e.g., "Understood, I didn't update the page.") + and move on. Do NOT ask for alternatives or troubleshoot. + - If status is "not_found", inform the user conversationally using the exact message provided. + Example: "I couldn't find the page '[page_title]' in your indexed Notion pages. [message details]" + Do NOT treat this as an error. Do NOT invent information. Simply relay the message and + ask the user to verify the page title or check if it's been indexed. + + Examples: + - "Add 'New meeting notes from today' to the 'Meeting Notes' Notion page" + - "Append the following to the 'Project Plan' Notion page: '# Status Update\n\nCompleted phase 1'" + """ + logger.info( + f"update_notion_page called: page_title='{page_title}', content_length={len(content) if content else 0}" + ) + + if db_session is None or search_space_id is None or user_id is None: + logger.error( + "Notion tool not properly configured - missing required parameters" + ) + return { + "status": "error", + "message": "Notion tool not properly configured. Please contact support.", + } + + if not content or not content.strip(): + logger.error(f"Empty content provided for page '{page_title}'") + return { + "status": "error", + "message": "Content is required to update the page. Please provide the actual content you want to add.", + } + + try: + metadata_service = NotionToolMetadataService(db_session) + context = await metadata_service.get_update_context( + search_space_id, user_id, page_title + ) + + if "error" in context: + error_msg = context["error"] + # Check if it's a "not found" error (softer handling for LLM) + if "not found" in error_msg.lower(): + logger.warning(f"Page not found: {error_msg}") + return { + "status": "not_found", + "message": error_msg, + } + else: + logger.error(f"Failed to fetch update context: {error_msg}") + return { + "status": "error", + "message": error_msg, + } + + page_id = context.get("page_id") + connector_id_from_context = context.get("account", {}).get("id") + + logger.info( + f"Requesting approval for updating Notion page: '{page_title}' (page_id={page_id})" + ) + approval = interrupt( + { + "type": "notion_page_update", + "action": { + "tool": "update_notion_page", + "params": { + "page_id": page_id, + "content": content, + "connector_id": connector_id_from_context, + }, + }, + "context": context, + } + ) + + decisions_raw = ( + approval.get("decisions", []) if isinstance(approval, dict) else [] + ) + decisions = ( + decisions_raw if isinstance(decisions_raw, list) else [decisions_raw] + ) + decisions = [d for d in decisions if isinstance(d, dict)] + if not decisions: + logger.warning("No approval decision received") + return { + "status": "error", + "message": "No approval decision received", + } + + decision = decisions[0] + decision_type = decision.get("type") or decision.get("decision_type") + logger.info(f"User decision: {decision_type}") + + if decision_type == "reject": + logger.info("Notion page update rejected by user") + return { + "status": "rejected", + "message": "User declined. The page was not updated. Do not ask again or suggest alternatives.", + } + + edited_action = decision.get("edited_action") + final_params: dict[str, Any] = {} + if isinstance(edited_action, dict): + edited_args = edited_action.get("args") + if isinstance(edited_args, dict): + final_params = edited_args + elif isinstance(decision.get("args"), dict): + # Some interrupt payloads place args directly on the decision. + final_params = decision["args"] + + final_page_id = final_params.get("page_id", page_id) + final_content = final_params.get("content", content) + final_connector_id = final_params.get( + "connector_id", connector_id_from_context + ) + + logger.info( + f"Updating Notion page with final params: page_id={final_page_id}, has_content={final_content is not None}" + ) + + from sqlalchemy.future import select + + from app.db import SearchSourceConnector, SearchSourceConnectorType + + if final_connector_id: + result = await db_session.execute( + select(SearchSourceConnector).filter( + SearchSourceConnector.id == final_connector_id, + SearchSourceConnector.search_space_id == search_space_id, + SearchSourceConnector.user_id == user_id, + SearchSourceConnector.connector_type + == SearchSourceConnectorType.NOTION_CONNECTOR, + ) + ) + connector = result.scalars().first() + + if not connector: + logger.error( + f"Invalid connector_id={final_connector_id} for search_space_id={search_space_id}" + ) + return { + "status": "error", + "message": "Selected Notion account is invalid or has been disconnected. Please select a valid account.", + } + actual_connector_id = connector.id + logger.info(f"Validated Notion connector: id={actual_connector_id}") + else: + logger.error("No connector found for this page") + return { + "status": "error", + "message": "No connector found for this page.", + } + + notion_connector = NotionHistoryConnector( + session=db_session, + connector_id=actual_connector_id, + ) + + result = await notion_connector.update_page( + page_id=final_page_id, + content=final_content, + ) + logger.info( + f"update_page result: {result.get('status')} - {result.get('message', '')}" + ) + return result + + except Exception as e: + from langgraph.errors import GraphInterrupt + + if isinstance(e, GraphInterrupt): + raise + + logger.error(f"Error updating Notion page: {e}", exc_info=True) + return { + "status": "error", + "message": str(e) + if isinstance(e, ValueError) + else f"Unexpected error: {e!s}", + } + + return update_notion_page diff --git a/surfsense_backend/app/agents/new_chat/tools/registry.py b/surfsense_backend/app/agents/new_chat/tools/registry.py index 8092a6104..132791c4b 100644 --- a/surfsense_backend/app/agents/new_chat/tools/registry.py +++ b/surfsense_backend/app/agents/new_chat/tools/registry.py @@ -50,7 +50,13 @@ from .generate_image import create_generate_image_tool from .knowledge_base import create_search_knowledge_base_tool from .link_preview import create_link_preview_tool from .mcp_tool import load_mcp_tools +from .notion import ( + create_create_notion_page_tool, + create_delete_notion_page_tool, + create_update_notion_page_tool, +) from .podcast import create_generate_podcast_tool +from .report import create_generate_report_tool from .scrape_webpage import create_scrape_webpage_tool from .search_surfsense_docs import create_search_surfsense_docs_tool from .shared_memory import ( @@ -118,6 +124,16 @@ BUILTIN_TOOLS: list[ToolDefinition] = [ ), requires=["search_space_id", "db_session", "thread_id"], ), + # Report generation tool (inline, short-lived sessions for DB ops) + ToolDefinition( + name="generate_report", + description="Generate a structured Markdown report from provided content", + factory=lambda deps: create_generate_report_tool( + search_space_id=deps["search_space_id"], + thread_id=deps["thread_id"], + ), + requires=["search_space_id", "thread_id"], + ), # Link preview tool - fetches Open Graph metadata for URLs ToolDefinition( name="link_preview", @@ -200,15 +216,38 @@ BUILTIN_TOOLS: list[ToolDefinition] = [ requires=["user_id", "search_space_id", "db_session", "thread_visibility"], ), # ========================================================================= - # ADD YOUR CUSTOM TOOLS BELOW + # NOTION TOOLS - create, update, delete pages # ========================================================================= - # Example: - # ToolDefinition( - # name="my_custom_tool", - # description="What my tool does", - # factory=lambda deps: create_my_custom_tool(...), - # requires=["search_space_id"], - # ), + ToolDefinition( + name="create_notion_page", + description="Create a new page in the user's Notion workspace", + factory=lambda deps: create_create_notion_page_tool( + db_session=deps["db_session"], + search_space_id=deps["search_space_id"], + user_id=deps["user_id"], + ), + requires=["db_session", "search_space_id", "user_id"], + ), + ToolDefinition( + name="update_notion_page", + description="Append new content to an existing Notion page", + factory=lambda deps: create_update_notion_page_tool( + db_session=deps["db_session"], + search_space_id=deps["search_space_id"], + user_id=deps["user_id"], + ), + requires=["db_session", "search_space_id", "user_id"], + ), + ToolDefinition( + name="delete_notion_page", + description="Delete an existing Notion page", + factory=lambda deps: create_delete_notion_page_tool( + db_session=deps["db_session"], + search_space_id=deps["search_space_id"], + user_id=deps["user_id"], + ), + requires=["db_session", "search_space_id", "user_id"], + ), ] diff --git a/surfsense_backend/app/agents/new_chat/tools/report.py b/surfsense_backend/app/agents/new_chat/tools/report.py new file mode 100644 index 000000000..75f087fa2 --- /dev/null +++ b/surfsense_backend/app/agents/new_chat/tools/report.py @@ -0,0 +1,414 @@ +""" +Report generation tool for the SurfSense agent. + +This module provides a factory function for creating the generate_report tool +that generates a structured Markdown report inline (no Celery). The LLM is +called within the tool, the result is saved to the database, and the tool +returns immediately with a ready status. + +Uses short-lived database sessions to avoid holding connections during long +LLM calls (30-120+ seconds). Each DB operation (read config, save report) +opens and closes its own session, ensuring no connection is held idle during +the LLM API call. +""" + +import logging +import re +from typing import Any + +from langchain_core.messages import HumanMessage +from langchain_core.tools import tool + +from app.db import Report, async_session_maker +from app.services.llm_service import get_document_summary_llm + +logger = logging.getLogger(__name__) + +# Prompt template for report generation (new report from scratch) +_REPORT_PROMPT = """You are an expert report writer. Generate a well-structured, comprehensive Markdown report based on the provided information. + +**Topic:** {topic} + +**Report Style:** {report_style} + +{user_instructions_section} + +{previous_version_section} + +**Source Content:** +{source_content} + +--- + +**Instructions:** +1. Write the report in well-formatted Markdown. +2. Include a clear title (as a level-1 heading), an executive summary, and logically organized sections. +3. Use headings (##, ###), bullet points, numbered lists, bold/italic text, and tables where appropriate. +4. Cite specific facts, figures, and findings from the source content. +5. Be thorough and comprehensive — include all relevant information from the source content. +6. End with a conclusion or key takeaways section. +7. The report should be professional and ready to export. +8. When including code examples, ALWAYS format them as proper fenced code blocks with the correct language identifier (e.g. ```java, ```python). Code inside code blocks MUST have proper line breaks and indentation — NEVER put multiple statements on a single line. Each statement, brace, and logical block must be on its own line with correct indentation. +9. When including Mermaid diagrams, use ```mermaid fenced code blocks. Each Mermaid statement MUST be on its own line — NEVER use semicolons to join multiple statements on one line. For line breaks inside node labels, use
(NOT
). Example: + ```mermaid + graph TD + A[Source Code] --> B[Compiler] + B --> C[Bytecode] + ``` +10. When including mathematical formulas or equations, ALWAYS use LaTeX notation. NEVER use backtick code spans or Unicode symbols for math. + +Write the report now: +""" + + +def _strip_wrapping_code_fences(text: str) -> str: + """Remove wrapping code fences that LLMs often add around Markdown output. + + Handles patterns like: + ```markdown\\n...content...\\n``` + ```md\\n...content...\\n``` + ```\\n...content...\\n``` + """ + stripped = text.strip() + # Match opening fence with optional language tag (markdown, md, or bare) + m = re.match(r"^```(?:markdown|md)?\s*\n", stripped) + if m and stripped.endswith("```"): + stripped = stripped[m.end() :] # remove opening fence + stripped = stripped[:-3].rstrip() # remove closing fence + return stripped + + +def _extract_metadata(content: str) -> dict[str, Any]: + """Extract metadata from generated Markdown content.""" + # Count section headings + headings = re.findall(r"^(#{1,6})\s+(.+)$", content, re.MULTILINE) + + # Word count + word_count = len(content.split()) + + # Character count + char_count = len(content) + + return { + "status": "ready", + "word_count": word_count, + "char_count": char_count, + "section_count": len(headings), + } + + +def create_generate_report_tool( + search_space_id: int, + thread_id: int | None = None, +): + """ + Factory function to create the generate_report tool with injected dependencies. + + The tool generates a Markdown report inline using the search space's + document summary LLM, saves it to the database, and returns immediately. + + Uses short-lived database sessions for each DB operation so no connection + is held during the long LLM API call. + + Args: + search_space_id: The user's search space ID + thread_id: The chat thread ID for associating the report + + Returns: + A configured tool function for generating reports + """ + + @tool + async def generate_report( + topic: str, + source_content: str, + report_style: str = "detailed", + user_instructions: str | None = None, + parent_report_id: int | None = None, + ) -> dict[str, Any]: + """ + Generate a structured Markdown report from provided content. + + Use this tool when the user asks to create, generate, write, produce, draft, + or summarize into a report-style deliverable. + HIGH-PRIORITY DECISION RULE: + - If the user asks for a report in any form, + call this tool rather than writing the full report directly in chat. + - Only skip this tool when the user explicitly requests chat-only output and + says they do not want a generated report card. + Trigger classes include: + - Direct trigger words: report, document, memo, letter, template + - Creation-intent phrases: "write a document/report/post/article" + - File-intent words: requests containing "save", "file", or "document" when + intent is to create a report-like deliverable + - Word-doc specific triggers: professional report-style deliverable, + professional document, Word doc, .docx + - Other report-like output intents: one-pager, blog post, article, + standalone written content, comprehensive guide + - General artifact-style intents: analysis / writing as substantial deliverables + Common triggers include phrases like: + - "Generate a report about this" + - "Write a report from this conversation" + - "Create a detailed report about..." + - "Make a research report on..." + - "Summarize this into a report" + - "Turn this into a report" + - "Write a report/document" + - "Draft a report" + - "Create an executive summary" + - "Make a briefing note" + - "Write a one-pager" + - "Write a blog post" + - "Write an article" + - "Create a comprehensive guide" + - "Prepare a report" + - "Create a small report" + - "Write a short report" + - "Make a quick report" + - "Brief report for class" + + FORMAT/EXPORT RULE: + - Always generate the report content in Markdown. + - If the user requests DOCX/Word/PDF or another file format, export from + the generated Markdown report. + SOURCE-COLLECTION RULE: + - If enough source material is already present in the conversation (chat + history, pasted text, uploaded files, or a provided video/article summary), + generate directly from that source_content. + - Use knowledge-base search first only when extra context is needed beyond + what the user already provided. + + VERSIONING — parent_report_id: + - Set parent_report_id when the user wants to MODIFY, REVISE, IMPROVE, + UPDATE, EXPAND, or ADD CONTENT TO an existing report that was already + generated in this conversation. + - This includes both explicit AND implicit modification requests. If the + user references the existing report using words like "it", "this", + "here", "the report", or clearly refers to a previously generated + report, treat it as a revision request. + - The value must be the report_id from a previous generate_report + result in this same conversation. + - Do NOT set parent_report_id when: + * The user asks for a report on a completely NEW/DIFFERENT topic + * The user says "generate another report" (new report, not a revision) + * There is no prior report to reference + - When parent_report_id is set, the previous report's content will be + used as a base. Your user_instructions should describe WHAT TO CHANGE. + + Examples of when to SET parent_report_id: + User: "Make that report shorter" → parent_report_id = + User: "Add a cost analysis section to the report" → parent_report_id = + User: "Rewrite the report in a more formal tone" → parent_report_id = + User: "I want more details about pricing in here" → parent_report_id = + User: "Include more examples" → parent_report_id = + User: "Can you also cover nutrition in this?" → parent_report_id = + User: "Make it more detailed" → parent_report_id = + User: "Not bad, but expand on the budget section" → parent_report_id = + User: "Also mention the competitor landscape" → parent_report_id = + + Examples of when to LEAVE parent_report_id as None: + User: "Generate a report on climate change" → parent_report_id = None (new topic) + User: "Write me a report about the budget" → parent_report_id = None (new topic) + User: "Create another report, this time about marketing" → parent_report_id = None + User: "Now write one about travel trends in Europe" → parent_report_id = None (new topic despite "now") + User: "Do the same kind of report but for the fitness industry" → parent_report_id = None (new topic, different subject) + + Args: + topic: A short, concise title for the report (maximum 8 words). Keep it brief and descriptive — e.g. "AI in Healthcare Analysis: A Comprehensive Report" instead of "Comprehensive Analysis of Artificial Intelligence Applications in Modern Healthcare Systems". + source_content: The text content to base the report on. This MUST be comprehensive and include: + * If discussing the current conversation: a detailed summary of the FULL chat history + * If based on knowledge base search: the key findings and insights from search results + * You can combine both: conversation context + search results for richer reports + * The more detailed the source_content, the better the report quality + report_style: Style of the report. Options: "detailed", "executive_summary", "deep_research", "brief". Default: "detailed" + user_instructions: Optional specific instructions for the report (e.g., "focus on financial impacts", "include recommendations"). When revising an existing report (parent_report_id is set), this should describe the changes to make. + parent_report_id: Optional ID of a previously generated report to revise. When set, the new report is created as a new version in the same version group. The previous report's content is included as context for the LLM to refine. + + Returns: + A dictionary containing: + - status: "ready" or "failed" + - report_id: The report ID + - title: The report title + - word_count: Number of words in the report + - message: Status message (or "error" field if failed) + """ + # Initialize version tracking variables (used by _save_failed_report closure) + parent_report_content: str | None = None + report_group_id: int | None = None + + async def _save_failed_report(error_msg: str) -> int | None: + """Persist a failed report row using a short-lived session.""" + try: + async with async_session_maker() as session: + failed_report = Report( + title=topic, + content=None, + report_metadata={ + "status": "failed", + "error_message": error_msg, + }, + report_style=report_style, + search_space_id=search_space_id, + thread_id=thread_id, + report_group_id=report_group_id, + ) + session.add(failed_report) + await session.commit() + await session.refresh(failed_report) + # If this is a new group (v1 failed), set group to self + if not failed_report.report_group_id: + failed_report.report_group_id = failed_report.id + await session.commit() + logger.info( + f"[generate_report] Saved failed report {failed_report.id}: {error_msg}" + ) + return failed_report.id + except Exception: + logger.exception( + "[generate_report] Could not persist failed report row" + ) + return None + + try: + # ── Phase 1: READ (short-lived session) ────────────────────── + # Fetch parent report and LLM config, then close the session + # so no DB connection is held during the long LLM call. + async with async_session_maker() as read_session: + if parent_report_id: + parent_report = await read_session.get(Report, parent_report_id) + if parent_report: + report_group_id = parent_report.report_group_id + parent_report_content = parent_report.content + logger.info( + f"[generate_report] Creating new version from parent {parent_report_id} " + f"(group {report_group_id})" + ) + else: + logger.warning( + f"[generate_report] parent_report_id={parent_report_id} not found, " + "creating standalone report" + ) + + llm = await get_document_summary_llm(read_session, search_space_id) + # read_session closed — connection returned to pool + + if not llm: + error_msg = ( + "No LLM configured. Please configure a language model in Settings." + ) + report_id = await _save_failed_report(error_msg) + return { + "status": "failed", + "error": error_msg, + "report_id": report_id, + "title": topic, + } + + # Build the prompt + user_instructions_section = "" + if user_instructions: + user_instructions_section = ( + f"**Additional Instructions:** {user_instructions}" + ) + + # If revising, include previous version content + previous_version_section = "" + if parent_report_content: + previous_version_section = ( + "**Previous Version of This Report (refine this based on the instructions above — " + "preserve structure and quality, apply only the requested changes):**\n\n" + f"{parent_report_content}" + ) + + prompt = _REPORT_PROMPT.format( + topic=topic, + report_style=report_style, + user_instructions_section=user_instructions_section, + previous_version_section=previous_version_section, + source_content=source_content[:100000], # Cap source content + ) + + # ── Phase 2: LLM CALL (no DB connection held) ──────────────── + response = await llm.ainvoke([HumanMessage(content=prompt)]) + report_content = response.content + + if not report_content or not isinstance(report_content, str): + error_msg = "LLM returned empty or invalid content" + report_id = await _save_failed_report(error_msg) + return { + "status": "failed", + "error": error_msg, + "report_id": report_id, + "title": topic, + } + + # LLMs often wrap output in ```markdown ... ``` fences — strip them + # so the stored content is clean Markdown. + report_content = _strip_wrapping_code_fences(report_content) + + if not report_content: + error_msg = "LLM returned empty or invalid content" + report_id = await _save_failed_report(error_msg) + return { + "status": "failed", + "error": error_msg, + "report_id": report_id, + "title": topic, + } + + # Extract metadata (includes "status": "ready") + metadata = _extract_metadata(report_content) + + # ── Phase 3: WRITE (short-lived session) ───────────────────── + # Save the report to the database, then close the session. + async with async_session_maker() as write_session: + report = Report( + title=topic, + content=report_content, + report_metadata=metadata, + report_style=report_style, + search_space_id=search_space_id, + thread_id=thread_id, + report_group_id=report_group_id, + ) + write_session.add(report) + await write_session.commit() + await write_session.refresh(report) + + # If this is a brand-new report (v1), set report_group_id = own id + if not report.report_group_id: + report.report_group_id = report.id + await write_session.commit() + + saved_report_id = report.id + saved_group_id = report.report_group_id + # write_session closed — connection returned to pool + + logger.info( + f"[generate_report] Created report {saved_report_id} " + f"(group={saved_group_id}): " + f"{metadata.get('word_count', 0)} words, " + f"{metadata.get('section_count', 0)} sections" + ) + + return { + "status": "ready", + "report_id": saved_report_id, + "title": topic, + "word_count": metadata.get("word_count", 0), + "message": f"Report generated successfully: {topic}", + } + + except Exception as e: + error_message = str(e) + logger.exception(f"[generate_report] Error: {error_message}") + report_id = await _save_failed_report(error_message) + + return { + "status": "failed", + "error": error_message, + "report_id": report_id, + "title": topic, + } + + return generate_report diff --git a/surfsense_backend/app/app.py b/surfsense_backend/app/app.py index d62557b94..72475e3a7 100644 --- a/surfsense_backend/app/app.py +++ b/surfsense_backend/app/app.py @@ -1,3 +1,4 @@ +import asyncio import logging import time from collections import defaultdict @@ -8,6 +9,7 @@ import redis from fastapi import Depends, FastAPI, HTTPException, Request, status from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse +from limits.storage import MemoryStorage from slowapi import Limiter from slowapi.errors import RateLimitExceeded from slowapi.middleware import SlowAPIMiddleware @@ -36,12 +38,16 @@ rate_limit_logger = logging.getLogger("surfsense.rate_limit") # Uses the same Redis instance as Celery for zero additional infrastructure. # Protects auth endpoints from brute force and user enumeration attacks. -# SlowAPI limiter — provides default rate limits (60/min) for ALL routes +# SlowAPI limiter — provides default rate limits (1024/min) for ALL routes # via the ASGI middleware. This is the general safety net. +# in_memory_fallback ensures requests are still served (with per-worker +# in-memory limiting) when Redis is unreachable, instead of hanging. limiter = Limiter( key_func=get_remote_address, storage_uri=config.REDIS_APP_URL, - default_limits=["60/minute"], + default_limits=["1024/minute"], + in_memory_fallback_enabled=True, + in_memory_fallback=[MemoryStorage()], ) @@ -179,8 +185,15 @@ async def lifespan(app: FastAPI): initialize_llm_router() # Initialize Image Generation Router for Auto mode load balancing initialize_image_gen_router() - # Seed Surfsense documentation - await seed_surfsense_docs() + # Seed Surfsense documentation (with timeout so a slow embedding API + # doesn't block startup indefinitely and make the container unresponsive) + try: + await asyncio.wait_for(seed_surfsense_docs(), timeout=120) + except TimeoutError: + logging.getLogger(__name__).warning( + "Surfsense docs seeding timed out after 120s — skipping. " + "Docs will be indexed on the next restart." + ) yield # Cleanup: close checkpointer connection on shutdown await close_checkpointer() @@ -397,6 +410,13 @@ if config.AUTH_TYPE == "GOOGLE": app.include_router(crud_router, prefix="/api/v1", tags=["crud"]) +@app.get("/health", tags=["health"]) +@limiter.exempt +async def health_check(): + """Lightweight liveness probe exempt from rate limiting.""" + return {"status": "ok"} + + @app.get("/verify-token") async def authenticated_route( user: User = Depends(current_active_user), diff --git a/surfsense_backend/app/celery_app.py b/surfsense_backend/app/celery_app.py index 477e5369f..a6637b1bd 100644 --- a/surfsense_backend/app/celery_app.py +++ b/surfsense_backend/app/celery_app.py @@ -82,7 +82,6 @@ celery_app = Celery( "app.tasks.celery_tasks.blocknote_migration_tasks", "app.tasks.celery_tasks.document_reindex_tasks", "app.tasks.celery_tasks.stale_notification_cleanup_task", - "app.tasks.celery_tasks.connector_deletion_task", ], ) @@ -143,7 +142,6 @@ celery_app.conf.update( "index_bookstack_pages": {"queue": CONNECTORS_QUEUE}, "index_obsidian_vault": {"queue": CONNECTORS_QUEUE}, "index_composio_connector": {"queue": CONNECTORS_QUEUE}, - "delete_connector_with_documents": {"queue": CONNECTORS_QUEUE}, # Everything else (document processing, podcasts, reindexing, # schedule checker, cleanup) stays on the default fast queue. }, diff --git a/surfsense_backend/app/connectors/notion_history.py b/surfsense_backend/app/connectors/notion_history.py index 525b0b4c3..311f97ddd 100644 --- a/surfsense_backend/app/connectors/notion_history.py +++ b/surfsense_backend/app/connectors/notion_history.py @@ -1,5 +1,7 @@ import asyncio +import contextlib import logging +import re from collections.abc import Awaitable, Callable from typing import Any, TypeVar @@ -10,7 +12,6 @@ from sqlalchemy.future import select from app.config import config from app.db import SearchSourceConnector -from app.routes.notion_add_connector_route import refresh_notion_token from app.schemas.notion_auth_credentials import NotionAuthCredentialsBase from app.utils.oauth_security import TokenEncryption @@ -219,6 +220,8 @@ class NotionHistoryConnector: ) # Refresh token + from app.routes.notion_add_connector_route import refresh_notion_token + connector = await refresh_notion_token(self._session, connector) # Reload credentials after refresh @@ -439,6 +442,16 @@ class NotionHistoryConnector: if page_title not in self._pages_with_skipped_content: self._pages_with_skipped_content.append(page_title) + @staticmethod + def _api_error_message(error: APIResponseError) -> str: + """Extract a stable, human-readable message from Notion API errors.""" + body = getattr(error, "body", None) + if isinstance(body, dict): + return str(body.get("message", str(error))) + if body: + return str(body) + return str(error) + async def __aenter__(self): """Async context manager entry.""" return self @@ -777,3 +790,370 @@ class NotionHistoryConnector: # Return empty string for unsupported block types return "" + + # ========================================================================= + # WRITE OPERATIONS (create, update, delete pages) + # ========================================================================= + + async def _get_first_accessible_parent(self) -> str | None: + """ + Get the first accessible page ID that can be used as a parent. + + Returns: + Page ID string, or None if no accessible pages found + """ + try: + notion = await self._get_client() + + # Search for pages, get most recently edited first + response = await self._api_call_with_retry( + notion.search, + filter={"property": "object", "value": "page"}, + sort={"direction": "descending", "timestamp": "last_edited_time"}, + page_size=1, # We only need the first one + ) + + results = response.get("results", []) + if results: + return results[0]["id"] + + return None + + except Exception as e: + logger.error(f"Error finding accessible parent page: {e}") + return None + + def _markdown_to_blocks(self, markdown: str) -> list[dict[str, Any]]: + """ + Convert markdown content to Notion blocks. + + This is a simple converter that handles basic markdown. + For more complex markdown, consider using a proper markdown parser. + + Args: + markdown: Markdown content + + Returns: + List of Notion block objects + """ + blocks = [] + lines = markdown.split("\n") + + for line in lines: + line = line.strip() + + if not line: + continue + + # Heading 1 + if line.startswith("# "): + blocks.append( + { + "object": "block", + "type": "heading_1", + "heading_1": { + "rich_text": [ + {"type": "text", "text": {"content": line[2:]}} + ] + }, + } + ) + # Heading 2 + elif line.startswith("## "): + blocks.append( + { + "object": "block", + "type": "heading_2", + "heading_2": { + "rich_text": [ + {"type": "text", "text": {"content": line[3:]}} + ] + }, + } + ) + # Heading 3 + elif line.startswith("### "): + blocks.append( + { + "object": "block", + "type": "heading_3", + "heading_3": { + "rich_text": [ + {"type": "text", "text": {"content": line[4:]}} + ] + }, + } + ) + # Bullet list + elif line.startswith("- ") or line.startswith("* "): + blocks.append( + { + "object": "block", + "type": "bulleted_list_item", + "bulleted_list_item": { + "rich_text": [ + {"type": "text", "text": {"content": line[2:]}} + ] + }, + } + ) + # Numbered list + elif match := re.match(r"^(\d+)\.\s+(.*)$", line): + content = match.group(2) # Extract text after "number. " + blocks.append( + { + "object": "block", + "type": "numbered_list_item", + "numbered_list_item": { + "rich_text": [ + {"type": "text", "text": {"content": content}} + ] + }, + } + ) + # Regular paragraph + else: + blocks.append( + { + "object": "block", + "type": "paragraph", + "paragraph": { + "rich_text": [{"type": "text", "text": {"content": line}}] + }, + } + ) + + return blocks + + async def create_page( + self, title: str, content: str, parent_page_id: str | None = None + ) -> dict[str, Any]: + """ + Create a new Notion page. + + Args: + title: Page title + content: Page content (markdown format) + parent_page_id: Optional parent page ID (creates as subpage if provided) + + Returns: + Dictionary with page details: + - page_id: Created page ID + - url: Page URL + - title: Page title + - status: "success" or "error" + - message: Success/error message + + Raises: + APIResponseError: If Notion API returns an error + """ + try: + logger.info( + f"Creating Notion page: title='{title}', parent_page_id={parent_page_id}" + ) + + # Get Notion client + notion = await self._get_client() + + # Convert markdown content to Notion blocks + children = self._markdown_to_blocks(content) + + # Prepare parent - find first available page if not provided + if not parent_page_id: + logger.info( + "No parent_page_id provided, searching for first accessible page..." + ) + parent_page_id = await self._get_first_accessible_parent() + if not parent_page_id: + logger.warning("No accessible parent pages found") + return { + "status": "error", + "message": "Could not find any accessible Notion pages to use as parent. " + "Please make sure your Notion integration has access to at least one page.", + } + logger.info(f"Using parent_page_id: {parent_page_id}") + + parent = {"type": "page_id", "page_id": parent_page_id} + + # Create the page with standard title property + properties = { + "title": {"title": [{"type": "text", "text": {"content": title}}]} + } + + response = await self._api_call_with_retry( + notion.pages.create, + parent=parent, + properties=properties, + children=children[:100], # Notion API limit: 100 blocks per request + ) + + page_id = response["id"] + page_url = response["url"] + + # If content has more than 100 blocks, append them + if len(children) > 100: + for i in range(100, len(children), 100): + batch = children[i : i + 100] + await self._api_call_with_retry( + notion.blocks.children.append, block_id=page_id, children=batch + ) + + return { + "status": "success", + "page_id": page_id, + "url": page_url, + "title": title, + "message": f"Created Notion page '{title}'", + } + + except APIResponseError as e: + logger.error(f"Notion API error creating page: {e}") + error_msg = self._api_error_message(e) + return { + "status": "error", + "message": f"Failed to create Notion page: {error_msg}", + } + except Exception as e: + logger.error(f"Unexpected error creating Notion page: {e}") + return { + "status": "error", + "message": f"Failed to create Notion page: {e!s}", + } + + async def update_page( + self, page_id: str, content: str | None = None + ) -> dict[str, Any]: + """ + Update an existing Notion page by appending new content. + + Note: Content is appended to the page, not replaced. + + Args: + page_id: Page ID to update + content: New markdown content to append to the page (optional) + + Returns: + Dictionary with update result + + Raises: + APIResponseError: If Notion API returns an error + """ + try: + notion = await self._get_client() + + # Append content if provided + if content: + # Convert new content to blocks + try: + children = self._markdown_to_blocks(content) + if not children: + logger.warning( + "No blocks generated from content, skipping append" + ) + return { + "status": "error", + "message": "Content conversion failed: no valid blocks generated", + } + except Exception as e: + logger.error(f"Failed to convert markdown to blocks: {e}") + return { + "status": "error", + "message": f"Failed to parse content: {e!s}", + } + + # Append new content blocks + try: + for i in range(0, len(children), 100): + batch = children[i : i + 100] + await self._api_call_with_retry( + notion.blocks.children.append, + block_id=page_id, + children=batch, + ) + logger.info( + f"Successfully appended {len(children)} new blocks to page {page_id}" + ) + except Exception as e: + logger.error(f"Failed to append content blocks: {e}") + return { + "status": "error", + "message": f"Failed to append content: {e!s}", + } + + # Get updated page info + response = await self._api_call_with_retry( + notion.pages.retrieve, page_id=page_id + ) + page_url = response["url"] + page_title = response["properties"]["title"]["title"][0]["text"]["content"] + + return { + "status": "success", + "page_id": page_id, + "url": page_url, + "title": page_title, + "message": f"Updated Notion page '{page_title}' (content appended)", + } + + except APIResponseError as e: + logger.error(f"Notion API error updating page: {e}") + error_msg = self._api_error_message(e) + return { + "status": "error", + "message": f"Failed to update Notion page: {error_msg}", + } + except Exception as e: + logger.error(f"Unexpected error updating Notion page: {e}") + return { + "status": "error", + "message": f"Failed to update Notion page: {e!s}", + } + + async def delete_page(self, page_id: str) -> dict[str, Any]: + """ + Delete (archive) a Notion page. + + Note: Notion doesn't truly delete pages, it archives them. + + Args: + page_id: Page ID to delete + + Returns: + Dictionary with deletion result + + Raises: + APIResponseError: If Notion API returns an error + """ + try: + notion = await self._get_client() + + # Archive the page (Notion's way of "deleting") + response = await self._api_call_with_retry( + notion.pages.update, page_id=page_id, archived=True + ) + + page_title = "Unknown" + with contextlib.suppress(KeyError, IndexError): + page_title = response["properties"]["title"]["title"][0]["text"][ + "content" + ] + + return { + "status": "success", + "page_id": page_id, + "message": f"Deleted Notion page '{page_title}'", + } + + except APIResponseError as e: + logger.error(f"Notion API error deleting page: {e}") + error_msg = self._api_error_message(e) + return { + "status": "error", + "message": f"Failed to delete Notion page: {error_msg}", + } + except Exception as e: + logger.error(f"Unexpected error deleting Notion page: {e}") + return { + "status": "error", + "message": f"Failed to delete Notion page: {e!s}", + } diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py index 90c839980..e232f0e14 100644 --- a/surfsense_backend/app/db.py +++ b/surfsense_backend/app/db.py @@ -1031,6 +1031,36 @@ class Podcast(BaseModel, TimestampMixin): thread = relationship("NewChatThread") +class Report(BaseModel, TimestampMixin): + """Report model for storing generated Markdown reports.""" + + __tablename__ = "reports" + + title = Column(String(500), nullable=False) + content = Column(Text, nullable=True) # Markdown body + report_metadata = Column(JSONB, nullable=True) # section headings, word count, etc. + report_style = Column( + String(100), nullable=True + ) # e.g. "executive_summary", "deep_research" + + search_space_id = Column( + Integer, ForeignKey("searchspaces.id", ondelete="CASCADE"), nullable=False + ) + search_space = relationship("SearchSpace", back_populates="reports") + + # Versioning: reports sharing the same report_group_id are versions of the same report. + # For v1, report_group_id = the report's own id (set after insert). + report_group_id = Column(Integer, nullable=True, index=True) + + thread_id = Column( + Integer, + ForeignKey("new_chat_threads.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + thread = relationship("NewChatThread") + + class ImageGenerationConfig(BaseModel, TimestampMixin): """ Dedicated configuration table for image generation models. @@ -1185,6 +1215,12 @@ class SearchSpace(BaseModel, TimestampMixin): order_by="Podcast.id.desc()", cascade="all, delete-orphan", ) + reports = relationship( + "Report", + back_populates="search_space", + order_by="Report.id.desc()", + cascade="all, delete-orphan", + ) image_generations = relationship( "ImageGeneration", back_populates="search_space", diff --git a/surfsense_backend/app/routes/__init__.py b/surfsense_backend/app/routes/__init__.py index d9353284c..db4e81026 100644 --- a/surfsense_backend/app/routes/__init__.py +++ b/surfsense_backend/app/routes/__init__.py @@ -34,6 +34,7 @@ from .notion_add_connector_route import router as notion_add_connector_router from .podcasts_routes import router as podcasts_router from .public_chat_routes import router as public_chat_router from .rbac_routes import router as rbac_router +from .reports_routes import router as reports_router from .search_source_connectors_routes import router as search_source_connectors_router from .search_spaces_routes import router as search_spaces_router from .slack_add_connector_route import router as slack_add_connector_router @@ -50,6 +51,7 @@ router.include_router(notes_router) router.include_router(new_chat_router) # Chat with assistant-ui persistence router.include_router(chat_comments_router) router.include_router(podcasts_router) # Podcast task status and audio +router.include_router(reports_router) # Report CRUD and export (PDF/DOCX) router.include_router(image_generation_router) # Image generation via litellm router.include_router(search_source_connectors_router) router.include_router(google_calendar_add_connector_router) diff --git a/surfsense_backend/app/routes/documents_routes.py b/surfsense_backend/app/routes/documents_routes.py index c8aeac0c9..ed4d6dea3 100644 --- a/surfsense_backend/app/routes/documents_routes.py +++ b/surfsense_backend/app/routes/documents_routes.py @@ -97,7 +97,10 @@ async def create_documents( raise HTTPException(status_code=400, detail="Invalid document type") await session.commit() - return {"message": "Documents processed successfully"} + return { + "message": "Documents queued for background processing", + "status": "queued", + } except HTTPException: raise except Exception as e: diff --git a/surfsense_backend/app/routes/new_chat_routes.py b/surfsense_backend/app/routes/new_chat_routes.py index ecb1c2a6f..11a8abeae 100644 --- a/surfsense_backend/app/routes/new_chat_routes.py +++ b/surfsense_backend/app/routes/new_chat_routes.py @@ -43,11 +43,12 @@ from app.schemas.new_chat import ( PublicChatSnapshotCreateResponse, PublicChatSnapshotListResponse, RegenerateRequest, + ResumeRequest, ThreadHistoryLoadResponse, ThreadListItem, ThreadListResponse, ) -from app.tasks.chat.stream_new_chat import stream_new_chat +from app.tasks.chat.stream_new_chat import stream_new_chat, stream_resume_chat from app.users import current_active_user from app.utils.rbac import check_permission @@ -634,8 +635,16 @@ async def delete_thread( # For PRIVATE threads, only the creator can delete # For SEARCH_SPACE threads, any member with permission can delete + # Legacy threads (created_by_id is NULL) have no recorded creator, + # so we skip strict ownership and fall through to legacy handling + # which allows the search space owner to delete them if db_thread.visibility == ChatVisibility.PRIVATE: - await check_thread_access(session, db_thread, user, require_ownership=True) + await check_thread_access( + session, + db_thread, + user, + require_ownership=(db_thread.created_by_id is not None), + ) await session.delete(db_thread) await session.commit() @@ -1326,3 +1335,78 @@ async def regenerate_response( status_code=500, detail=f"An unexpected error occurred during regeneration: {e!s}", ) from None + + +# ============================================================================= +# Resume Interrupted Chat Endpoint +# ============================================================================= + + +@router.post("/threads/{thread_id}/resume") +async def resume_chat( + thread_id: int, + request: ResumeRequest, + session: AsyncSession = Depends(get_async_session), + user: User = Depends(current_active_user), +): + try: + result = await session.execute( + select(NewChatThread).filter(NewChatThread.id == thread_id) + ) + thread = result.scalars().first() + + if not thread: + raise HTTPException(status_code=404, detail="Thread not found") + + await check_permission( + session, + user, + thread.search_space_id, + Permission.CHATS_CREATE.value, + "You don't have permission to chat in this search space", + ) + + await check_thread_access(session, thread, user) + + search_space_result = await session.execute( + select(SearchSpace).filter(SearchSpace.id == request.search_space_id) + ) + search_space = search_space_result.scalars().first() + + if not search_space: + raise HTTPException(status_code=404, detail="Search space not found") + + llm_config_id = ( + search_space.agent_llm_id if search_space.agent_llm_id is not None else -1 + ) + + decisions = [d.model_dump() for d in request.decisions] + + return StreamingResponse( + stream_resume_chat( + chat_id=thread_id, + search_space_id=request.search_space_id, + decisions=decisions, + session=session, + user_id=str(user.id), + llm_config_id=llm_config_id, + thread_visibility=thread.visibility, + ), + media_type="text/event-stream", + headers={ + "Cache-Control": "no-cache", + "Connection": "keep-alive", + "X-Accel-Buffering": "no", + }, + ) + + except HTTPException: + raise + except Exception as e: + import traceback + + traceback.print_exc() + raise HTTPException( + status_code=500, + detail=f"An unexpected error occurred during resume: {e!s}", + ) from None diff --git a/surfsense_backend/app/routes/public_chat_routes.py b/surfsense_backend/app/routes/public_chat_routes.py index b5f523429..9afcbc188 100644 --- a/surfsense_backend/app/routes/public_chat_routes.py +++ b/surfsense_backend/app/routes/public_chat_routes.py @@ -20,6 +20,7 @@ from app.services.public_chat_service import ( clone_from_snapshot, get_public_chat, get_snapshot_podcast, + get_snapshot_report, ) from app.users import current_active_user @@ -114,3 +115,37 @@ async def stream_public_podcast( "Content-Disposition": f"inline; filename={os.path.basename(file_path)}", }, ) + + +@router.get("/{share_token}/reports/{report_id}/content") +async def get_public_report_content( + share_token: str, + report_id: int, + session: AsyncSession = Depends(get_async_session), +): + """ + Get report content from a public chat snapshot. + + No authentication required - the share_token provides access. + Returns report content including title, markdown body, metadata, and versions. + """ + from app.services.public_chat_service import get_snapshot_report_versions + + report_info = await get_snapshot_report(session, share_token, report_id) + + if not report_info: + raise HTTPException(status_code=404, detail="Report not found") + + # Get version siblings from the same snapshot + versions = await get_snapshot_report_versions( + session, share_token, report_info.get("report_group_id") + ) + + return { + "id": report_info.get("original_id"), + "title": report_info.get("title"), + "content": report_info.get("content"), + "report_metadata": report_info.get("report_metadata"), + "report_group_id": report_info.get("report_group_id"), + "versions": versions, + } diff --git a/surfsense_backend/app/routes/reports_routes.py b/surfsense_backend/app/routes/reports_routes.py new file mode 100644 index 000000000..c9974ddb0 --- /dev/null +++ b/surfsense_backend/app/routes/reports_routes.py @@ -0,0 +1,398 @@ +""" +Report routes for read, export (PDF/DOCX), and delete operations. + +No create or update endpoints here — reports are generated inline by the +agent tool during chat and stored as Markdown in the database. +Export to PDF/DOCX is on-demand — PDF uses pypandoc (Markdown→Typst) + typst-py +(Typst→PDF); DOCX uses pypandoc directly. + +Authorization: lightweight search-space membership checks (no granular RBAC) +since reports are chat-generated artifacts, not standalone managed resources. +""" + +import asyncio +import io +import logging +import os +import re +import tempfile +from enum import Enum + +import pypandoc +import typst +from fastapi import APIRouter, Depends, HTTPException, Query +from fastapi.responses import StreamingResponse +from sqlalchemy import select +from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db import ( + Report, + SearchSpace, + SearchSpaceMembership, + User, + get_async_session, +) +from app.schemas import ReportContentRead, ReportRead +from app.schemas.reports import ReportVersionInfo +from app.users import current_active_user +from app.utils.rbac import check_search_space_access + +logger = logging.getLogger(__name__) + +router = APIRouter() + +MAX_REPORT_LIST_LIMIT = 500 + + +class ExportFormat(str, Enum): + PDF = "pdf" + DOCX = "docx" + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +_CODE_FENCE_RE = re.compile(r"^```(?:markdown|md)?\s*\n", re.MULTILINE) + + +def _strip_wrapping_code_fences(text: str) -> str: + """Remove wrapping code fences (```markdown...```) that LLMs often add.""" + stripped = text.strip() + m = _CODE_FENCE_RE.match(stripped) + if m and stripped.endswith("```"): + stripped = stripped[m.end() : -3].rstrip() + return stripped + + +def _normalize_latex_delimiters(text: str) -> str: + """Convert all LaTeX math delimiters to dollar-sign form. + + Pandoc's ``tex_math_dollars`` extension (on the ``gfm`` reader) handles + ``$…$`` and ``$$…$$`` natively. This function converts every other + delimiter style that LLMs produce into dollar-sign form so pandoc can + parse them as math. + + Supported conversions: + \\[…\\] → $$…$$ (display math) + \\(…\\) → $…$ (inline math) + \\begin{equation}…\\end{equation} → $$…$$ (display math) + \\begin{displaymath}…\\end{displaymath}→ $$…$$ (display math) + \\begin{math}…\\end{math} → $…$ (inline math) + `$$…$$` / `$…$` → strip wrapping backticks + """ + # 1. Block math: \[...\] → $$...$$ + text = re.sub(r"\\\[([\s\S]*?)\\\]", lambda m: f"$${m.group(1)}$$", text) + # 2. Inline math: \(...\) → $...$ + text = re.sub(r"\\\(([\s\S]*?)\\\)", lambda m: f"${m.group(1)}$", text) + # 3. \begin{equation}...\end{equation} → $$...$$ + text = re.sub( + r"\\begin\{equation\}([\s\S]*?)\\end\{equation\}", + lambda m: f"$${m.group(1)}$$", + text, + ) + # 4. \begin{displaymath}...\end{displaymath} → $$...$$ + text = re.sub( + r"\\begin\{displaymath\}([\s\S]*?)\\end\{displaymath\}", + lambda m: f"$${m.group(1)}$$", + text, + ) + # 5. \begin{math}...\end{math} → $...$ + text = re.sub( + r"\\begin\{math\}([\s\S]*?)\\end\{math\}", + lambda m: f"${m.group(1)}$", + text, + ) + # 6. Strip backtick wrapping around math: `$$...$$` → $$...$$ and `$...$` → $...$ + text = re.sub(r"`(\${1,2})((?:(?!\1).)+)\1`", r"\1\2\1", text) + + # 7. Trim whitespace inside inline math $...$. + # Pandoc's tex_math_dollars requires NO space after the opening $ and + # NO space before the closing $. LLMs frequently produce "$ e^x $" + # or "\( e^x \)" (which step 2 converts to "$ e^x $"). Without + # trimming, pandoc treats these as literal dollar-sign text. + # We require spaces on BOTH sides to avoid false-positives on + # currency like "$50" or "$50 and $100". + def _trim_inline_math(m: re.Match) -> str: + inner = m.group(1).strip() + return f"${inner}$" if inner else m.group(0) + + text = re.sub(r"(? Report: + """Fetch a report and verify the user belongs to its search space. + + Raises HTTPException(404) if not found, HTTPException(403) if no access. + """ + result = await session.execute(select(Report).filter(Report.id == report_id)) + report = result.scalars().first() + + if not report: + raise HTTPException(status_code=404, detail="Report not found") + + # Lightweight membership check - no granular RBAC, just "is the user a + # member of the search space this report belongs to?" + await check_search_space_access(session, user, report.search_space_id) + + return report + + +async def _get_version_siblings( + session: AsyncSession, + report: Report, +) -> list[ReportVersionInfo]: + """Get all versions in the same report group, ordered by created_at.""" + if not report.report_group_id: + # Legacy report without group — it's the only version + return [ReportVersionInfo(id=report.id, created_at=report.created_at)] + + result = await session.execute( + select(Report.id, Report.created_at) + .filter(Report.report_group_id == report.report_group_id) + .order_by(Report.created_at.asc()) + ) + rows = result.all() + return [ReportVersionInfo(id=row[0], created_at=row[1]) for row in rows] + + +# --------------------------------------------------------------------------- +# Routes +# --------------------------------------------------------------------------- + + +@router.get("/reports", response_model=list[ReportRead]) +async def read_reports( + skip: int = Query(default=0, ge=0), + limit: int = Query(default=100, ge=1, le=MAX_REPORT_LIST_LIMIT), + search_space_id: int | None = None, + session: AsyncSession = Depends(get_async_session), + user: User = Depends(current_active_user), +): + """ + List reports the user has access to. + Filters by search space membership. + """ + try: + if search_space_id is not None: + # Verify the caller is a member of the requested search space + await check_search_space_access(session, user, search_space_id) + + result = await session.execute( + select(Report) + .filter(Report.search_space_id == search_space_id) + .order_by(Report.id.desc()) + .offset(skip) + .limit(limit) + ) + else: + result = await session.execute( + select(Report) + .join(SearchSpace) + .join(SearchSpaceMembership) + .filter(SearchSpaceMembership.user_id == user.id) + .order_by(Report.id.desc()) + .offset(skip) + .limit(limit) + ) + return result.scalars().all() + except HTTPException: + raise + except SQLAlchemyError: + raise HTTPException( + status_code=500, detail="Database error occurred while fetching reports" + ) from None + + +@router.get("/reports/{report_id}", response_model=ReportRead) +async def read_report( + report_id: int, + session: AsyncSession = Depends(get_async_session), + user: User = Depends(current_active_user), +): + """ + Get a specific report by ID (metadata only, no content). + """ + try: + return await _get_report_with_access(report_id, session, user) + except HTTPException: + raise + except SQLAlchemyError: + raise HTTPException( + status_code=500, detail="Database error occurred while fetching report" + ) from None + + +@router.get("/reports/{report_id}/content", response_model=ReportContentRead) +async def read_report_content( + report_id: int, + session: AsyncSession = Depends(get_async_session), + user: User = Depends(current_active_user), +): + """ + Get full Markdown content of a report, including version siblings. + """ + try: + report = await _get_report_with_access(report_id, session, user) + versions = await _get_version_siblings(session, report) + + return ReportContentRead( + id=report.id, + title=report.title, + content=report.content, + report_metadata=report.report_metadata, + report_group_id=report.report_group_id, + versions=versions, + ) + except HTTPException: + raise + except SQLAlchemyError: + raise HTTPException( + status_code=500, + detail="Database error occurred while fetching report content", + ) from None + + +@router.get("/reports/{report_id}/export") +async def export_report( + report_id: int, + format: ExportFormat = Query( + ExportFormat.PDF, description="Export format: pdf or docx" + ), + session: AsyncSession = Depends(get_async_session), + user: User = Depends(current_active_user), +): + """ + Export a report as PDF or DOCX. + """ + try: + report = await _get_report_with_access(report_id, session, user) + + if not report.content: + raise HTTPException( + status_code=400, detail="Report has no content to export" + ) + + # Strip wrapping code fences that LLMs sometimes add around Markdown. + # Without this, pandoc treats the entire content as a code block. + markdown_content = _strip_wrapping_code_fences(report.content) + + # Normalise all LaTeX math delimiters (\(\), \[\], \begin{equation}, + # etc.) into $/$$ form that pandoc's tex_math_dollars extension can parse. + markdown_content = _normalize_latex_delimiters(markdown_content) + + # Convert Markdown to the requested format. + # + # DOCX: pypandoc (pandoc) handles the full conversion directly. + # + # PDF: two-step pipeline — pypandoc converts Markdown → Typst markup, + # then the `typst` Python library compiles Typst → PDF. This avoids + # requiring the Typst CLI on the system PATH; the typst pip package + # bundles the compiler as a native extension. Typst produces + # professional styling for tables, headings, code blocks, etc. + # + # Use "gfm" as the base input format because LLM output uses GFM-style + # pipe tables that pandoc's stricter default "markdown" may mangle. + # The +tex_math_dollars extension enables $/$$ math recognition. + + def _convert_and_read() -> bytes: + """Run all blocking I/O (tempfile, pandoc/typst, file read, cleanup) in a thread.""" + if format == ExportFormat.PDF: + # Step 1: Markdown → Typst markup via pandoc. + # We must set mainfont / monofont so the generated template's + # `font` parameter is non-empty; without it pandoc emits + # `font: ()` which makes Typst error with + # "font fallback list must not be empty". + # We use fonts that ship embedded inside typst-py so this + # works even on systems with no fonts installed. + typst_markup: str = pypandoc.convert_text( + markdown_content, + "typst", + format="gfm+tex_math_dollars", + extra_args=[ + "--standalone", + "-V", + "mainfont:Libertinus Serif", + "-V", + "monofont:DejaVu Sans Mono", + ], + ) + # Step 2: Typst markup → PDF via typst Python library + pdf_bytes: bytes = typst.compile(typst_markup.encode("utf-8")) + return pdf_bytes + else: + # DOCX: let pandoc handle the full conversion + fd, tmp_path = tempfile.mkstemp(suffix=f".{format.value}") + os.close(fd) + try: + pypandoc.convert_text( + markdown_content, + format.value, + format="gfm+tex_math_dollars", + extra_args=["--standalone"], + outputfile=tmp_path, + ) + with open(tmp_path, "rb") as f: + return f.read() + finally: + os.unlink(tmp_path) + + loop = asyncio.get_running_loop() + output = await loop.run_in_executor(None, _convert_and_read) + + # Sanitize filename + safe_title = ( + "".join( + c if c.isalnum() or c in " -_" else "_" for c in report.title + ).strip()[:80] + or "report" + ) + + media_types = { + ExportFormat.PDF: "application/pdf", + ExportFormat.DOCX: "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + } + + return StreamingResponse( + io.BytesIO(output), + media_type=media_types[format], + headers={ + "Content-Disposition": f'attachment; filename="{safe_title}.{format.value}"', + }, + ) + + except HTTPException: + raise + except Exception as e: + logger.exception("Report export failed") + raise HTTPException(status_code=500, detail=f"Export failed: {e!s}") from e + + +@router.delete("/reports/{report_id}", response_model=dict) +async def delete_report( + report_id: int, + session: AsyncSession = Depends(get_async_session), + user: User = Depends(current_active_user), +): + """ + Delete a report. + """ + try: + db_report = await _get_report_with_access(report_id, session, user) + + await session.delete(db_report) + await session.commit() + return {"message": "Report deleted successfully"} + except HTTPException: + raise + except SQLAlchemyError: + await session.rollback() + raise HTTPException( + status_code=500, detail="Database error occurred while deleting report" + ) from None diff --git a/surfsense_backend/app/routes/search_source_connectors_routes.py b/surfsense_backend/app/routes/search_source_connectors_routes.py index ba6877376..b69238837 100644 --- a/surfsense_backend/app/routes/search_source_connectors_routes.py +++ b/surfsense_backend/app/routes/search_source_connectors_routes.py @@ -532,14 +532,16 @@ async def delete_search_source_connector( """ Delete a search source connector and all its associated documents. - The deletion runs in background via Celery task. User is notified - via the notification system when complete (no polling required). + The deletion happens inline (documents are deleted in batches, + then the connector record is removed). Requires CONNECTORS_DELETE permission. """ - from app.tasks.celery_tasks.connector_deletion_task import ( - delete_connector_with_documents_task, - ) + from sqlalchemy import delete as sa_delete, func + + from app.db import Document + + deletion_batch_size = 500 try: # Get the connector first @@ -562,12 +564,10 @@ async def delete_search_source_connector( "You don't have permission to delete this connector", ) - # Store connector info before we queue the deletion task + # Store connector info before deletion connector_name = db_connector.name - connector_type = db_connector.connector_type.value - search_space_id = db_connector.search_space_id - # Delete any periodic schedule associated with this connector (lightweight, sync) + # Delete any periodic schedule associated with this connector if db_connector.periodic_indexing_enabled: success = delete_periodic_schedule(connector_id) if not success: @@ -575,7 +575,7 @@ async def delete_search_source_connector( f"Failed to delete periodic schedule for connector {connector_id}" ) - # For Composio connectors, delete the connected account in Composio (lightweight API call, sync) + # For Composio connectors, delete the connected account in Composio composio_connector_types = [ SearchSourceConnectorType.COMPOSIO_GOOGLE_DRIVE_CONNECTOR, SearchSourceConnectorType.COMPOSIO_GMAIL_CONNECTOR, @@ -602,30 +602,58 @@ async def delete_search_source_connector( f"for connector {connector_id}" ) except Exception as composio_error: - # Log but don't fail the deletion - Composio account may already be deleted logger.warning( f"Error deleting Composio connected account {composio_connected_account_id}: {composio_error!s}" ) - # Queue background task to delete documents and connector - # This handles potentially large document counts without blocking the API - delete_connector_with_documents_task.delay( - connector_id=connector_id, - user_id=str(user.id), - search_space_id=search_space_id, - connector_name=connector_name, - connector_type=connector_type, + # Delete documents in batches (chunks are deleted via CASCADE) + total_deleted = 0 + count_result = await session.execute( + select(func.count(Document.id)).where(Document.connector_id == connector_id) ) + total_docs = count_result.scalar() or 0 logger.info( - f"Queued deletion task for connector {connector_id} ({connector_name})" + f"Starting deletion of connector {connector_id} ({connector_name}). " + f"Documents to delete: {total_docs}" ) + while True: + result = await session.execute( + select(Document.id) + .where(Document.connector_id == connector_id) + .limit(deletion_batch_size) + ) + doc_ids = [row[0] for row in result.fetchall()] + + if not doc_ids: + break + + await session.execute(sa_delete(Document).where(Document.id.in_(doc_ids))) + await session.commit() + + total_deleted += len(doc_ids) + logger.info( + f"Deleted batch of {len(doc_ids)} documents. " + f"Progress: {total_deleted}/{total_docs}" + ) + + # Delete the connector record + await session.delete(db_connector) + await session.commit() + + logger.info( + f"Connector {connector_id} ({connector_name}) deleted successfully. " + f"Total documents deleted: {total_deleted}" + ) + + doc_text = "document" if total_deleted == 1 else "documents" return { - "message": "Connector deletion started. You will be notified when complete.", - "status": "queued", + "message": f"Connector '{connector_name}' deleted. {total_deleted} {doc_text} removed.", + "status": "completed", "connector_id": connector_id, "connector_name": connector_name, + "documents_deleted": total_deleted, } except HTTPException: raise diff --git a/surfsense_backend/app/schemas/__init__.py b/surfsense_backend/app/schemas/__init__.py index b015cf715..e3f03b29c 100644 --- a/surfsense_backend/app/schemas/__init__.py +++ b/surfsense_backend/app/schemas/__init__.py @@ -76,6 +76,7 @@ from .rbac_schemas import ( RoleUpdate, UserSearchSpaceAccess, ) +from .reports import ReportBase, ReportContentRead, ReportRead, ReportVersionInfo from .search_source_connector import ( MCPConnectorCreate, MCPConnectorRead, @@ -185,6 +186,11 @@ __all__ = [ "PodcastUpdate", "RefreshTokenRequest", "RefreshTokenResponse", + # Report schemas + "ReportBase", + "ReportContentRead", + "ReportRead", + "ReportVersionInfo", "RoleCreate", "RoleRead", "RoleUpdate", diff --git a/surfsense_backend/app/schemas/new_chat.py b/surfsense_backend/app/schemas/new_chat.py index efa314979..916f42024 100644 --- a/surfsense_backend/app/schemas/new_chat.py +++ b/surfsense_backend/app/schemas/new_chat.py @@ -7,7 +7,7 @@ These schemas follow the assistant-ui ThreadHistoryAdapter pattern: """ from datetime import datetime -from typing import Any +from typing import Any, Literal from uuid import UUID from pydantic import BaseModel, ConfigDict, Field @@ -193,6 +193,16 @@ class RegenerateRequest(BaseModel): mentioned_surfsense_doc_ids: list[int] | None = None +class ResumeDecision(BaseModel): + type: Literal["approve", "edit", "reject"] + edited_action: dict[str, Any] | None = None + + +class ResumeRequest(BaseModel): + search_space_id: int + decisions: list[ResumeDecision] + + # ============================================================================= # Public Chat Snapshot Schemas # ============================================================================= diff --git a/surfsense_backend/app/schemas/reports.py b/surfsense_backend/app/schemas/reports.py new file mode 100644 index 000000000..9909d8601 --- /dev/null +++ b/surfsense_backend/app/schemas/reports.py @@ -0,0 +1,53 @@ +"""Report schemas for API responses.""" + +from datetime import datetime +from typing import Any + +from pydantic import BaseModel + + +class ReportBase(BaseModel): + """Base report schema.""" + + title: str + content: str | None = None + report_style: str | None = None + search_space_id: int + + +class ReportRead(BaseModel): + """Schema for reading a report (list view, no content).""" + + id: int + title: str + report_style: str | None = None + report_metadata: dict[str, Any] | None = None + report_group_id: int | None = None + created_at: datetime + + class Config: + from_attributes = True + + +class ReportVersionInfo(BaseModel): + """Lightweight version entry for the version switcher UI.""" + + id: int + created_at: datetime + + class Config: + from_attributes = True + + +class ReportContentRead(BaseModel): + """Schema for reading a report with full Markdown content.""" + + id: int + title: str + content: str | None = None + report_metadata: dict[str, Any] | None = None + report_group_id: int | None = None + versions: list[ReportVersionInfo] = [] + + class Config: + from_attributes = True diff --git a/surfsense_backend/app/services/new_streaming_service.py b/surfsense_backend/app/services/new_streaming_service.py index 57fbc9663..52a215997 100644 --- a/surfsense_backend/app/services/new_streaming_service.py +++ b/surfsense_backend/app/services/new_streaming_service.py @@ -504,6 +504,63 @@ class VercelStreamingService: }, ) + def format_interrupt_request(self, interrupt_value: dict[str, Any]) -> str: + """Format an interrupt request for human-in-the-loop approval. + + Args: + interrupt_value: The interrupt payload from either: + - interrupt_on config: {action_requests: [...], review_configs: [...]} + - interrupt() primitive: {type: "...", message: "...", action: {...}, context: {...}} + + Returns: + str: SSE formatted interrupt request data part + """ + normalized_payload = self._normalize_interrupt_payload(interrupt_value) + return self.format_data("interrupt-request", normalized_payload) + + def _normalize_interrupt_payload( + self, interrupt_value: dict[str, Any] + ) -> dict[str, Any]: + """Normalize interrupt payloads from different sources into a consistent format. + + Handles two interrupt sources: + 1. interrupt_on config (Deep Agent built-in): Already has action_requests/review_configs + 2. interrupt() primitive (custom tool code): Has type/action/context (message is optional) + + Args: + interrupt_value: Raw interrupt payload from Deep Agent + + Returns: + dict: Normalized payload with action_requests, review_configs, and optional context/message + """ + if "action_requests" in interrupt_value and "review_configs" in interrupt_value: + return interrupt_value + + interrupt_type = interrupt_value.get("type", "unknown") + message = interrupt_value.get("message") + action = interrupt_value.get("action", {}) + context = interrupt_value.get("context", {}) + + normalized = { + "action_requests": [ + { + "name": action.get("tool", "unknown_tool"), + "args": action.get("params", {}), + } + ], + "review_configs": [ + { + "action_name": action.get("tool", "unknown_tool"), + "allowed_decisions": ["approve", "edit", "reject"], + } + ], + "interrupt_type": interrupt_type, + "context": context, + } + if message: + normalized["message"] = message + return normalized + # ========================================================================= # Error Part # ========================================================================= diff --git a/surfsense_backend/app/services/notion/__init__.py b/surfsense_backend/app/services/notion/__init__.py new file mode 100644 index 000000000..3818d1af3 --- /dev/null +++ b/surfsense_backend/app/services/notion/__init__.py @@ -0,0 +1,11 @@ +from app.services.notion.tool_metadata_service import ( + NotionAccount, + NotionPage, + NotionToolMetadataService, +) + +__all__ = [ + "NotionAccount", + "NotionPage", + "NotionToolMetadataService", +] diff --git a/surfsense_backend/app/services/notion/tool_metadata_service.py b/surfsense_backend/app/services/notion/tool_metadata_service.py new file mode 100644 index 000000000..c3caed358 --- /dev/null +++ b/surfsense_backend/app/services/notion/tool_metadata_service.py @@ -0,0 +1,200 @@ +from dataclasses import dataclass + +from sqlalchemy import and_, func +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.future import select + +from app.db import ( + Document, + DocumentType, + SearchSourceConnector, + SearchSourceConnectorType, +) + + +@dataclass +class NotionAccount: + id: int + name: str + workspace_id: str | None + workspace_name: str + workspace_icon: str + + @classmethod + def from_connector(cls, connector: SearchSourceConnector) -> "NotionAccount": + return cls( + id=connector.id, + name=connector.name, + workspace_id=connector.config.get("workspace_id"), + workspace_name=connector.config.get("workspace_name", "Unnamed Workspace"), + workspace_icon=connector.config.get("workspace_icon", "📄"), + ) + + def to_dict(self) -> dict: + return { + "id": self.id, + "name": self.name, + "workspace_id": self.workspace_id, + "workspace_name": self.workspace_name, + "workspace_icon": self.workspace_icon, + } + + +@dataclass +class NotionPage: + page_id: str + title: str + connector_id: int + document_id: int + + @classmethod + def from_document(cls, document: Document) -> "NotionPage": + return cls( + page_id=document.document_metadata.get("page_id"), + title=document.title, + connector_id=document.connector_id, + document_id=document.id, + ) + + def to_dict(self) -> dict: + return { + "page_id": self.page_id, + "title": self.title, + "connector_id": self.connector_id, + "document_id": self.document_id, + } + + +class NotionToolMetadataService: + def __init__(self, db_session: AsyncSession): + self._db_session = db_session + + async def get_creation_context(self, search_space_id: int, user_id: str) -> dict: + accounts = await self._get_notion_accounts(search_space_id, user_id) + + if not accounts: + return { + "accounts": [], + "parent_pages": {}, + "error": "No Notion accounts connected", + } + + parent_pages = await self._get_parent_pages_by_account( + search_space_id, accounts + ) + + return { + "accounts": [acc.to_dict() for acc in accounts], + "parent_pages": parent_pages, + } + + async def get_update_context( + self, search_space_id: int, user_id: str, page_title: str + ) -> dict: + result = await self._db_session.execute( + select(Document) + .join( + SearchSourceConnector, Document.connector_id == SearchSourceConnector.id + ) + .filter( + and_( + Document.search_space_id == search_space_id, + Document.document_type == DocumentType.NOTION_CONNECTOR, + func.lower(Document.title) == func.lower(page_title), + SearchSourceConnector.user_id == user_id, + ) + ) + ) + document = result.scalars().first() + + if not document: + return { + "error": f"Page '{page_title}' not found in your indexed Notion pages. " + "This could mean: (1) the page doesn't exist, (2) it hasn't been indexed yet, " + "or (3) the page title is different. Please check the exact page title in Notion." + } + + if not document.connector_id: + return {"error": "Document has no associated connector"} + + result = await self._db_session.execute( + select(SearchSourceConnector).filter( + and_( + SearchSourceConnector.id == document.connector_id, + SearchSourceConnector.user_id == user_id, + ) + ) + ) + connector = result.scalars().first() + + if not connector: + return {"error": "Connector not found or access denied"} + + account = NotionAccount.from_connector(connector) + + page_id = document.document_metadata.get("page_id") + if not page_id: + return {"error": "Page ID not found in document metadata"} + + return { + "account": account.to_dict(), + "page_id": page_id, + "current_title": document.title, + "document_id": document.id, + "indexed_at": document.document_metadata.get("indexed_at"), + } + + async def get_delete_context( + self, search_space_id: int, user_id: str, page_title: str + ) -> dict: + return await self.get_update_context(search_space_id, user_id, page_title) + + async def _get_notion_accounts( + self, search_space_id: int, user_id: str + ) -> list[NotionAccount]: + result = await self._db_session.execute( + select(SearchSourceConnector) + .filter( + and_( + SearchSourceConnector.search_space_id == search_space_id, + SearchSourceConnector.user_id == user_id, + SearchSourceConnector.connector_type + == SearchSourceConnectorType.NOTION_CONNECTOR, + ) + ) + .order_by(SearchSourceConnector.last_indexed_at.desc()) + ) + connectors = result.scalars().all() + return [NotionAccount.from_connector(conn) for conn in connectors] + + async def _get_parent_pages_by_account( + self, search_space_id: int, accounts: list[NotionAccount] + ) -> dict: + parent_pages = {} + + for account in accounts: + result = await self._db_session.execute( + select(Document) + .filter( + and_( + Document.search_space_id == search_space_id, + Document.connector_id == account.id, + Document.document_type == DocumentType.NOTION_CONNECTOR, + ) + ) + .order_by(Document.updated_at.desc()) + .limit(50) + ) + documents = result.scalars().all() + + parent_pages[account.id] = [ + { + "page_id": doc.document_metadata.get("page_id"), + "title": doc.title, + "document_id": doc.id, + } + for doc in documents + if doc.document_metadata.get("page_id") + ] + + return parent_pages diff --git a/surfsense_backend/app/services/public_chat_service.py b/surfsense_backend/app/services/public_chat_service.py index ba2dd0079..a74d45991 100644 --- a/surfsense_backend/app/services/public_chat_service.py +++ b/surfsense_backend/app/services/public_chat_service.py @@ -29,6 +29,7 @@ from app.db import ( Podcast, PodcastStatus, PublicChatSnapshot, + Report, SearchSpaceMembership, User, ) @@ -38,6 +39,7 @@ UI_TOOLS = { "display_image", "link_preview", "generate_podcast", + "generate_report", "scrape_webpage", "multi_link_preview", } @@ -195,19 +197,22 @@ async def create_snapshot( message_ids = [] podcasts_data = [] podcast_ids_seen: set[int] = set() + reports_data = [] + report_ids_seen: set[int] = set() for msg in sorted(thread.messages, key=lambda m: m.created_at): author = await get_author_display(session, msg.author_id, user_cache) sanitized_content = sanitize_content_for_public(msg.content) - # Extract podcast references and update status to "ready" for completed podcasts + # Extract podcast/report references and update status to "ready" for completed ones if isinstance(sanitized_content, list): for part in sanitized_content: - if ( - isinstance(part, dict) - and part.get("type") == "tool-call" - and part.get("toolName") == "generate_podcast" - ): + if not isinstance(part, dict) or part.get("type") != "tool-call": + continue + + tool_name = part.get("toolName") + + if tool_name == "generate_podcast": result_data = part.get("result", {}) podcast_id = result_data.get("podcast_id") if podcast_id and podcast_id not in podcast_ids_seen: @@ -220,6 +225,17 @@ async def create_snapshot( # Update status to "ready" so frontend renders PodcastPlayer part["result"] = {**result_data, "status": "ready"} + elif tool_name == "generate_report": + result_data = part.get("result", {}) + report_id = result_data.get("report_id") + if report_id and report_id not in report_ids_seen: + report_info = await _get_report_for_snapshot(session, report_id) + if report_info: + reports_data.append(report_info) + report_ids_seen.add(report_id) + # Update status to "ready" so frontend renders ReportCard + part["result"] = {**result_data, "status": "ready"} + messages_data.append( { "id": msg.id, @@ -266,6 +282,7 @@ async def create_snapshot( "author": thread_author, "messages": messages_data, "podcasts": podcasts_data, + "reports": reports_data, } # Create new snapshot @@ -309,6 +326,27 @@ async def _get_podcast_for_snapshot( } +async def _get_report_for_snapshot( + session: AsyncSession, + report_id: int, +) -> dict | None: + """Get report info for embedding in snapshot_data.""" + result = await session.execute(select(Report).filter(Report.id == report_id)) + report = result.scalars().first() + + if not report: + return None + + return { + "original_id": report.id, + "title": report.title, + "content": report.content, + "report_metadata": report.report_metadata, + "report_group_id": report.report_group_id, + "created_at": report.created_at.isoformat() if report.created_at else None, + } + + # ============================================================================= # Snapshot Retrieval # ============================================================================= @@ -578,6 +616,7 @@ async def clone_from_snapshot( data = snapshot.snapshot_data messages_data = data.get("messages", []) podcasts_lookup = {p.get("original_id"): p for p in data.get("podcasts", [])} + reports_lookup = {r.get("original_id"): r for r in data.get("reports", [])} new_thread = NewChatThread( title=data.get("title", "Cloned Chat"), @@ -594,6 +633,7 @@ async def clone_from_snapshot( await session.flush() podcast_id_mapping: dict[int, int] = {} + report_id_mapping: dict[int, int] = {} # Check which authors from snapshot still exist in DB author_ids_from_snapshot: set[UUID] = set() @@ -655,6 +695,37 @@ async def clone_from_snapshot( "podcast_id": podcast_id_mapping[old_podcast_id], } + if ( + isinstance(part, dict) + and part.get("type") == "tool-call" + and part.get("toolName") == "generate_report" + ): + result = part.get("result", {}) + old_report_id = result.get("report_id") + + if old_report_id and old_report_id not in report_id_mapping: + report_info = reports_lookup.get(old_report_id) + if report_info: + new_report = Report( + title=report_info.get("title", "Cloned Report"), + content=report_info.get("content"), + report_metadata=report_info.get("report_metadata"), + search_space_id=target_search_space_id, + thread_id=new_thread.id, + ) + session.add(new_report) + await session.flush() + # For cloned reports, set report_group_id = own id + # (each cloned report starts as its own v1) + new_report.report_group_id = new_report.id + report_id_mapping[old_report_id] = new_report.id + + if old_report_id and old_report_id in report_id_mapping: + part["result"] = { + **result, + "report_id": report_id_mapping[old_report_id], + } + new_message = NewChatMessage( thread_id=new_thread.id, role=role, @@ -696,3 +767,59 @@ async def get_snapshot_podcast( return podcast return None + + +async def get_snapshot_report( + session: AsyncSession, + share_token: str, + report_id: int, +) -> dict | None: + """ + Get report info from a snapshot by original report ID. + + Used for displaying report content in public view. + Looks up the report by its original_id in the snapshot's reports array. + """ + snapshot = await get_snapshot_by_token(session, share_token) + + if not snapshot: + return None + + reports = snapshot.snapshot_data.get("reports", []) + + # Find report by original_id + for report in reports: + if report.get("original_id") == report_id: + return report + + return None + + +async def get_snapshot_report_versions( + session: AsyncSession, + share_token: str, + report_group_id: int | None, +) -> list[dict]: + """ + Get all report versions in the same group from a snapshot. + + Returns a list of lightweight version entries (id + created_at) + for the version switcher UI, sorted by original_id (insertion order). + """ + if not report_group_id: + return [] + + snapshot = await get_snapshot_by_token(session, share_token) + if not snapshot: + return [] + + reports = snapshot.snapshot_data.get("reports", []) + siblings = [r for r in reports if r.get("report_group_id") == report_group_id] + + # Sort by original_id (ascending = insertion order ≈ created_at order) + siblings.sort(key=lambda r: r.get("original_id", 0)) + + return [ + {"id": r.get("original_id"), "created_at": r.get("created_at")} + for r in siblings + ] diff --git a/surfsense_backend/app/tasks/celery_tasks/connector_deletion_task.py b/surfsense_backend/app/tasks/celery_tasks/connector_deletion_task.py deleted file mode 100644 index e024aca29..000000000 --- a/surfsense_backend/app/tasks/celery_tasks/connector_deletion_task.py +++ /dev/null @@ -1,269 +0,0 @@ -"""Celery task for background connector deletion. - -This task handles the deletion of all documents associated with a connector -in the background, then deletes the connector itself. User is notified via -the notification system when complete (no polling required). - -Features: -- Batch deletion to handle large document counts -- Automatic retry on failure -- Progress tracking via notifications -- Handles both success and failure notifications -""" - -import asyncio -import logging -from uuid import UUID - -from sqlalchemy import delete, func, select -from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine -from sqlalchemy.pool import NullPool - -from app.celery_app import celery_app -from app.config import config -from app.db import Document, Notification, SearchSourceConnector - -logger = logging.getLogger(__name__) - -# Batch size for document deletion -DELETION_BATCH_SIZE = 500 - - -def _get_celery_session_maker(): - """Create async session maker for Celery tasks.""" - engine = create_async_engine( - config.DATABASE_URL, - poolclass=NullPool, - echo=False, - ) - return async_sessionmaker(engine, expire_on_commit=False), engine - - -@celery_app.task( - bind=True, - name="delete_connector_with_documents", - max_retries=3, - default_retry_delay=60, - autoretry_for=(Exception,), - retry_backoff=True, -) -def delete_connector_with_documents_task( - self, - connector_id: int, - user_id: str, - search_space_id: int, - connector_name: str, - connector_type: str, -): - """ - Background task to delete a connector and all its associated documents. - - Creates a notification when complete (success or failure). - No polling required - user sees notification in UI. - - Args: - connector_id: ID of the connector to delete - user_id: ID of the user who initiated the deletion - search_space_id: ID of the search space - connector_name: Name of the connector (for notification message) - connector_type: Type of the connector (for logging) - """ - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - - try: - return loop.run_until_complete( - _delete_connector_async( - connector_id=connector_id, - user_id=user_id, - search_space_id=search_space_id, - connector_name=connector_name, - connector_type=connector_type, - ) - ) - finally: - loop.close() - - -async def _delete_connector_async( - connector_id: int, - user_id: str, - search_space_id: int, - connector_name: str, - connector_type: str, -) -> dict: - """ - Async implementation of connector deletion. - - Steps: - 1. Count total documents to delete - 2. Delete documents in batches (chunks cascade automatically) - 3. Delete the connector record - 4. Create success notification - - On failure, creates failure notification and re-raises exception. - """ - session_maker, engine = _get_celery_session_maker() - total_deleted = 0 - - try: - async with session_maker() as session: - # Step 1: Count total documents for this connector - count_result = await session.execute( - select(func.count(Document.id)).where( - Document.connector_id == connector_id - ) - ) - total_docs = count_result.scalar() or 0 - - logger.info( - f"Starting deletion of connector {connector_id} ({connector_name}). " - f"Documents to delete: {total_docs}" - ) - - # Step 2: Delete documents in batches - while True: - # Get batch of document IDs - result = await session.execute( - select(Document.id) - .where(Document.connector_id == connector_id) - .limit(DELETION_BATCH_SIZE) - ) - doc_ids = [row[0] for row in result.fetchall()] - - if not doc_ids: - break - - # Delete this batch (chunks are deleted via CASCADE) - await session.execute(delete(Document).where(Document.id.in_(doc_ids))) - await session.commit() - - total_deleted += len(doc_ids) - logger.info( - f"Deleted batch of {len(doc_ids)} documents. " - f"Progress: {total_deleted}/{total_docs}" - ) - - # Step 3: Delete the connector record - result = await session.execute( - select(SearchSourceConnector).where( - SearchSourceConnector.id == connector_id - ) - ) - connector = result.scalar_one_or_none() - - if connector: - await session.delete(connector) - logger.info(f"Deleted connector record: {connector_id}") - else: - logger.warning( - f"Connector {connector_id} not found - may have been already deleted" - ) - - # Step 4: Create success notification - doc_text = "document" if total_deleted == 1 else "documents" - notification = Notification( - user_id=UUID(user_id), - search_space_id=search_space_id, - type="connector_deletion", - title=f"{connector_name} removed", - message=f"Cleanup complete. {total_deleted} {doc_text} removed.", - notification_metadata={ - "connector_id": connector_id, - "connector_name": connector_name, - "connector_type": connector_type, - "documents_deleted": total_deleted, - "status": "completed", - }, - ) - session.add(notification) - await session.commit() - - logger.info( - f"Connector {connector_id} ({connector_name}) deleted successfully. " - f"Total documents deleted: {total_deleted}" - ) - - return { - "status": "success", - "connector_id": connector_id, - "connector_name": connector_name, - "documents_deleted": total_deleted, - } - - except Exception as e: - logger.error( - f"Failed to delete connector {connector_id} ({connector_name}): {e!s}", - exc_info=True, - ) - - # Create failure notification - try: - async with session_maker() as session: - notification = Notification( - user_id=UUID(user_id), - search_space_id=search_space_id, - type="connector_deletion", - title=f"Failed to Remove {connector_name}", - message="Something went wrong while removing this connector. Please try again.", - notification_metadata={ - "connector_id": connector_id, - "connector_name": connector_name, - "connector_type": connector_type, - "documents_deleted": total_deleted, - "status": "failed", - "error": str(e), - }, - ) - session.add(notification) - await session.commit() - except Exception as notify_error: - logger.error( - f"Failed to create failure notification: {notify_error!s}", - exc_info=True, - ) - - # Re-raise to trigger Celery retry - raise - - finally: - await engine.dispose() - - -async def delete_documents_by_connector_id( - session, - connector_id: int, - batch_size: int = DELETION_BATCH_SIZE, -) -> int: - """ - Delete all documents associated with a connector in batches. - - This is a utility function that can be used independently of the Celery task - for synchronous deletion scenarios (e.g., small document counts). - - Args: - session: AsyncSession instance - connector_id: ID of the connector - batch_size: Number of documents to delete per batch - - Returns: - Total number of documents deleted - """ - total_deleted = 0 - - while True: - result = await session.execute( - select(Document.id) - .where(Document.connector_id == connector_id) - .limit(batch_size) - ) - doc_ids = [row[0] for row in result.fetchall()] - - if not doc_ids: - break - - await session.execute(delete(Document).where(Document.id.in_(doc_ids))) - await session.commit() - total_deleted += len(doc_ids) - - return total_deleted diff --git a/surfsense_backend/app/tasks/chat/stream_new_chat.py b/surfsense_backend/app/tasks/chat/stream_new_chat.py index 31e67c7ff..a319554a5 100644 --- a/surfsense_backend/app/tasks/chat/stream_new_chat.py +++ b/surfsense_backend/app/tasks/chat/stream_new_chat.py @@ -11,6 +11,8 @@ Supports loading LLM configurations from: import json from collections.abc import AsyncGenerator +from dataclasses import dataclass +from typing import Any from uuid import UUID from langchain_core.messages import HumanMessage @@ -178,6 +180,660 @@ def extract_todos_from_deepagents(command_output) -> dict: return {"todos": todos_data} +@dataclass +class StreamResult: + accumulated_text: str = "" + is_interrupted: bool = False + interrupt_value: dict[str, Any] | None = None + + +async def _stream_agent_events( + agent: Any, + config: dict[str, Any], + input_data: Any, + streaming_service: VercelStreamingService, + result: StreamResult, + step_prefix: str = "thinking", + initial_step_id: str | None = None, + initial_step_title: str = "", + initial_step_items: list[str] | None = None, +) -> AsyncGenerator[str, None]: + """Shared async generator that streams and formats astream_events from the agent. + + Yields SSE-formatted strings. After exhausting, inspect the ``result`` + object for accumulated_text and interrupt state. + + Args: + agent: The compiled LangGraph agent. + config: LangGraph config dict (must include configurable.thread_id). + input_data: The input to pass to agent.astream_events (dict or Command). + streaming_service: VercelStreamingService instance for formatting events. + result: Mutable StreamResult populated with accumulated_text / interrupt info. + step_prefix: Prefix for thinking step IDs (e.g. "thinking" or "thinking-resume"). + initial_step_id: If set, the helper inherits an already-active thinking step. + initial_step_title: Title of the inherited thinking step. + initial_step_items: Items of the inherited thinking step. + + Yields: + SSE-formatted strings for each event. + """ + accumulated_text = "" + current_text_id: str | None = None + thinking_step_counter = 1 if initial_step_id else 0 + tool_step_ids: dict[str, str] = {} + completed_step_ids: set[str] = set() + last_active_step_id: str | None = initial_step_id + last_active_step_title: str = initial_step_title + last_active_step_items: list[str] = initial_step_items or [] + just_finished_tool: bool = False + + def next_thinking_step_id() -> str: + nonlocal thinking_step_counter + thinking_step_counter += 1 + return f"{step_prefix}-{thinking_step_counter}" + + def complete_current_step() -> str | None: + nonlocal last_active_step_id + if last_active_step_id and last_active_step_id not in completed_step_ids: + completed_step_ids.add(last_active_step_id) + event = streaming_service.format_thinking_step( + step_id=last_active_step_id, + title=last_active_step_title, + status="completed", + items=last_active_step_items if last_active_step_items else None, + ) + last_active_step_id = None + return event + return None + + async for event in agent.astream_events(input_data, config=config, version="v2"): + event_type = event.get("event", "") + + if event_type == "on_chat_model_stream": + chunk = event.get("data", {}).get("chunk") + if chunk and hasattr(chunk, "content"): + content = chunk.content + if content and isinstance(content, str): + if current_text_id is None: + completion_event = complete_current_step() + if completion_event: + yield completion_event + if just_finished_tool: + last_active_step_id = None + last_active_step_title = "" + last_active_step_items = [] + just_finished_tool = False + current_text_id = streaming_service.generate_text_id() + yield streaming_service.format_text_start(current_text_id) + yield streaming_service.format_text_delta(current_text_id, content) + accumulated_text += content + + elif event_type == "on_tool_start": + tool_name = event.get("name", "unknown_tool") + run_id = event.get("run_id", "") + tool_input = event.get("data", {}).get("input", {}) + + if current_text_id is not None: + yield streaming_service.format_text_end(current_text_id) + current_text_id = None + + if last_active_step_title != "Synthesizing response": + completion_event = complete_current_step() + if completion_event: + yield completion_event + + just_finished_tool = False + tool_step_id = next_thinking_step_id() + tool_step_ids[run_id] = tool_step_id + last_active_step_id = tool_step_id + + if tool_name == "search_knowledge_base": + query = ( + tool_input.get("query", "") + if isinstance(tool_input, dict) + else str(tool_input) + ) + last_active_step_title = "Searching knowledge base" + last_active_step_items = [ + f"Query: {query[:100]}{'...' if len(query) > 100 else ''}" + ] + yield streaming_service.format_thinking_step( + step_id=tool_step_id, + title="Searching knowledge base", + status="in_progress", + items=last_active_step_items, + ) + elif tool_name == "link_preview": + url = ( + tool_input.get("url", "") + if isinstance(tool_input, dict) + else str(tool_input) + ) + last_active_step_title = "Fetching link preview" + last_active_step_items = [ + f"URL: {url[:80]}{'...' if len(url) > 80 else ''}" + ] + yield streaming_service.format_thinking_step( + step_id=tool_step_id, + title="Fetching link preview", + status="in_progress", + items=last_active_step_items, + ) + elif tool_name == "display_image": + src = ( + tool_input.get("src", "") + if isinstance(tool_input, dict) + else str(tool_input) + ) + title = ( + tool_input.get("title", "") if isinstance(tool_input, dict) else "" + ) + last_active_step_title = "Analyzing the image" + last_active_step_items = [ + f"Analyzing: {title[:50] if title else src[:50]}{'...' if len(title or src) > 50 else ''}" + ] + yield streaming_service.format_thinking_step( + step_id=tool_step_id, + title="Analyzing the image", + status="in_progress", + items=last_active_step_items, + ) + elif tool_name == "scrape_webpage": + url = ( + tool_input.get("url", "") + if isinstance(tool_input, dict) + else str(tool_input) + ) + last_active_step_title = "Scraping webpage" + last_active_step_items = [ + f"URL: {url[:80]}{'...' if len(url) > 80 else ''}" + ] + yield streaming_service.format_thinking_step( + step_id=tool_step_id, + title="Scraping webpage", + status="in_progress", + items=last_active_step_items, + ) + elif tool_name == "generate_podcast": + podcast_title = ( + tool_input.get("podcast_title", "SurfSense Podcast") + if isinstance(tool_input, dict) + else "SurfSense Podcast" + ) + content_len = len( + tool_input.get("source_content", "") + if isinstance(tool_input, dict) + else "" + ) + last_active_step_title = "Generating podcast" + last_active_step_items = [ + f"Title: {podcast_title}", + f"Content: {content_len:,} characters", + "Preparing audio generation...", + ] + yield streaming_service.format_thinking_step( + step_id=tool_step_id, + title="Generating podcast", + status="in_progress", + items=last_active_step_items, + ) + elif tool_name == "generate_report": + report_topic = ( + tool_input.get("topic", "Report") + if isinstance(tool_input, dict) + else "Report" + ) + report_style = ( + tool_input.get("report_style", "detailed") + if isinstance(tool_input, dict) + else "detailed" + ) + content_len = len( + tool_input.get("source_content", "") + if isinstance(tool_input, dict) + else "" + ) + last_active_step_title = "Generating report" + last_active_step_items = [ + f"Topic: {report_topic}", + f"Style: {report_style}", + f"Source content: {content_len:,} characters", + "Generating report with LLM...", + ] + yield streaming_service.format_thinking_step( + step_id=tool_step_id, + title="Generating report", + status="in_progress", + items=last_active_step_items, + ) + else: + last_active_step_title = f"Using {tool_name.replace('_', ' ')}" + last_active_step_items = [] + yield streaming_service.format_thinking_step( + step_id=tool_step_id, + title=last_active_step_title, + status="in_progress", + ) + + tool_call_id = ( + f"call_{run_id[:32]}" + if run_id + else streaming_service.generate_tool_call_id() + ) + yield streaming_service.format_tool_input_start(tool_call_id, tool_name) + yield streaming_service.format_tool_input_available( + tool_call_id, + tool_name, + tool_input if isinstance(tool_input, dict) else {"input": tool_input}, + ) + + elif event_type == "on_tool_end": + run_id = event.get("run_id", "") + tool_name = event.get("name", "unknown_tool") + raw_output = event.get("data", {}).get("output", "") + + if hasattr(raw_output, "content"): + content = raw_output.content + if isinstance(content, str): + try: + tool_output = json.loads(content) + except (json.JSONDecodeError, TypeError): + tool_output = {"result": content} + elif isinstance(content, dict): + tool_output = content + else: + tool_output = {"result": str(content)} + elif isinstance(raw_output, dict): + tool_output = raw_output + else: + tool_output = {"result": str(raw_output) if raw_output else "completed"} + + tool_call_id = f"call_{run_id[:32]}" if run_id else "call_unknown" + original_step_id = tool_step_ids.get( + run_id, f"{step_prefix}-unknown-{run_id[:8]}" + ) + completed_step_ids.add(original_step_id) + + if tool_name == "search_knowledge_base": + result_info = "Search completed" + if isinstance(tool_output, dict): + result_len = tool_output.get("result_length", 0) + if result_len > 0: + result_info = f"Found relevant information ({result_len} chars)" + completed_items = [*last_active_step_items, result_info] + yield streaming_service.format_thinking_step( + step_id=original_step_id, + title="Searching knowledge base", + status="completed", + items=completed_items, + ) + elif tool_name == "link_preview": + if isinstance(tool_output, dict): + title = tool_output.get("title", "Link") + domain = tool_output.get("domain", "") + has_error = "error" in tool_output + if has_error: + completed_items = [ + *last_active_step_items, + f"Error: {tool_output.get('error', 'Failed to fetch')}", + ] + else: + completed_items = [ + *last_active_step_items, + f"Title: {title[:60]}{'...' if len(title) > 60 else ''}", + f"Domain: {domain}" if domain else "Preview loaded", + ] + else: + completed_items = [*last_active_step_items, "Preview loaded"] + yield streaming_service.format_thinking_step( + step_id=original_step_id, + title="Fetching link preview", + status="completed", + items=completed_items, + ) + elif tool_name == "display_image": + if isinstance(tool_output, dict): + title = tool_output.get("title", "") + alt = tool_output.get("alt", "Image") + display_name = title or alt + completed_items = [ + *last_active_step_items, + f"Analyzed: {display_name[:50]}{'...' if len(display_name) > 50 else ''}", + ] + else: + completed_items = [*last_active_step_items, "Image analyzed"] + yield streaming_service.format_thinking_step( + step_id=original_step_id, + title="Analyzing the image", + status="completed", + items=completed_items, + ) + elif tool_name == "scrape_webpage": + if isinstance(tool_output, dict): + title = tool_output.get("title", "Webpage") + word_count = tool_output.get("word_count", 0) + has_error = "error" in tool_output + if has_error: + completed_items = [ + *last_active_step_items, + f"Error: {tool_output.get('error', 'Failed to scrape')[:50]}", + ] + else: + completed_items = [ + *last_active_step_items, + f"Title: {title[:50]}{'...' if len(title) > 50 else ''}", + f"Extracted: {word_count:,} words", + ] + else: + completed_items = [*last_active_step_items, "Content extracted"] + yield streaming_service.format_thinking_step( + step_id=original_step_id, + title="Scraping webpage", + status="completed", + items=completed_items, + ) + elif tool_name == "generate_podcast": + podcast_status = ( + tool_output.get("status", "unknown") + if isinstance(tool_output, dict) + else "unknown" + ) + podcast_title = ( + tool_output.get("title", "Podcast") + if isinstance(tool_output, dict) + else "Podcast" + ) + if podcast_status == "processing": + completed_items = [ + f"Title: {podcast_title}", + "Audio generation started", + "Processing in background...", + ] + elif podcast_status == "already_generating": + completed_items = [ + f"Title: {podcast_title}", + "Podcast already in progress", + "Please wait for it to complete", + ] + elif podcast_status == "error": + error_msg = ( + tool_output.get("error", "Unknown error") + if isinstance(tool_output, dict) + else "Unknown error" + ) + completed_items = [ + f"Title: {podcast_title}", + f"Error: {error_msg[:50]}", + ] + else: + completed_items = last_active_step_items + yield streaming_service.format_thinking_step( + step_id=original_step_id, + title="Generating podcast", + status="completed", + items=completed_items, + ) + elif tool_name == "generate_report": + report_status = ( + tool_output.get("status", "unknown") + if isinstance(tool_output, dict) + else "unknown" + ) + report_title = ( + tool_output.get("title", "Report") + if isinstance(tool_output, dict) + else "Report" + ) + word_count = ( + tool_output.get("word_count", 0) + if isinstance(tool_output, dict) + else 0 + ) + + if report_status == "ready": + completed_items = [ + f"Title: {report_title}", + f"Words: {word_count:,}", + "Report generated successfully", + ] + elif report_status == "failed": + error_msg = ( + tool_output.get("error", "Unknown error") + if isinstance(tool_output, dict) + else "Unknown error" + ) + completed_items = [ + f"Title: {report_title}", + f"Error: {error_msg[:50]}", + ] + else: + completed_items = last_active_step_items + + yield streaming_service.format_thinking_step( + step_id=original_step_id, + title="Generating report", + status="completed", + items=completed_items, + ) + elif tool_name == "ls": + if isinstance(tool_output, dict): + ls_output = tool_output.get("result", "") + elif isinstance(tool_output, str): + ls_output = tool_output + else: + ls_output = str(tool_output) if tool_output else "" + file_names: list[str] = [] + if ls_output: + for line in ls_output.strip().split("\n"): + line = line.strip() + if line: + name = line.rstrip("/").split("/")[-1] + if name and len(name) <= 40: + file_names.append(name) + elif name: + file_names.append(name[:37] + "...") + if file_names: + if len(file_names) <= 5: + completed_items = [f"[{name}]" for name in file_names] + else: + completed_items = [f"[{name}]" for name in file_names[:4]] + completed_items.append(f"(+{len(file_names) - 4} more)") + else: + completed_items = ["No files found"] + yield streaming_service.format_thinking_step( + step_id=original_step_id, + title="Exploring files", + status="completed", + items=completed_items, + ) + else: + yield streaming_service.format_thinking_step( + step_id=original_step_id, + title=f"Using {tool_name.replace('_', ' ')}", + status="completed", + items=last_active_step_items, + ) + + just_finished_tool = True + last_active_step_id = None + last_active_step_title = "" + last_active_step_items = [] + + if tool_name == "generate_podcast": + yield streaming_service.format_tool_output_available( + tool_call_id, + tool_output + if isinstance(tool_output, dict) + else {"result": tool_output}, + ) + if ( + isinstance(tool_output, dict) + and tool_output.get("status") == "success" + ): + yield streaming_service.format_terminal_info( + f"Podcast generated successfully: {tool_output.get('title', 'Podcast')}", + "success", + ) + else: + error_msg = ( + tool_output.get("error", "Unknown error") + if isinstance(tool_output, dict) + else "Unknown error" + ) + yield streaming_service.format_terminal_info( + f"Podcast generation failed: {error_msg}", + "error", + ) + elif tool_name == "link_preview": + yield streaming_service.format_tool_output_available( + tool_call_id, + tool_output + if isinstance(tool_output, dict) + else {"result": tool_output}, + ) + if isinstance(tool_output, dict) and "error" not in tool_output: + title = tool_output.get("title", "Link") + yield streaming_service.format_terminal_info( + f"Link preview loaded: {title[:50]}{'...' if len(title) > 50 else ''}", + "success", + ) + else: + error_msg = ( + tool_output.get("error", "Failed to fetch") + if isinstance(tool_output, dict) + else "Failed to fetch" + ) + yield streaming_service.format_terminal_info( + f"Link preview failed: {error_msg}", + "error", + ) + elif tool_name == "display_image": + yield streaming_service.format_tool_output_available( + tool_call_id, + tool_output + if isinstance(tool_output, dict) + else {"result": tool_output}, + ) + if isinstance(tool_output, dict): + title = tool_output.get("title") or tool_output.get("alt", "Image") + yield streaming_service.format_terminal_info( + f"Image analyzed: {title[:40]}{'...' if len(title) > 40 else ''}", + "success", + ) + elif tool_name == "scrape_webpage": + if isinstance(tool_output, dict): + display_output = { + k: v for k, v in tool_output.items() if k != "content" + } + if "content" in tool_output: + content = tool_output.get("content", "") + display_output["content_preview"] = ( + content[:500] + "..." if len(content) > 500 else content + ) + yield streaming_service.format_tool_output_available( + tool_call_id, + display_output, + ) + else: + yield streaming_service.format_tool_output_available( + tool_call_id, + {"result": tool_output}, + ) + if isinstance(tool_output, dict) and "error" not in tool_output: + title = tool_output.get("title", "Webpage") + word_count = tool_output.get("word_count", 0) + yield streaming_service.format_terminal_info( + f"Scraped: {title[:40]}{'...' if len(title) > 40 else ''} ({word_count:,} words)", + "success", + ) + else: + error_msg = ( + tool_output.get("error", "Failed to scrape") + if isinstance(tool_output, dict) + else "Failed to scrape" + ) + yield streaming_service.format_terminal_info( + f"Scrape failed: {error_msg}", + "error", + ) + elif tool_name == "search_knowledge_base": + yield streaming_service.format_tool_output_available( + tool_call_id, + {"status": "completed", "result_length": len(str(tool_output))}, + ) + yield streaming_service.format_terminal_info( + "Knowledge base search completed", "success" + ) + elif tool_name == "generate_report": + # Stream the full report result so frontend can render the ReportCard + yield streaming_service.format_tool_output_available( + tool_call_id, + tool_output + if isinstance(tool_output, dict) + else {"result": tool_output}, + ) + # Send appropriate terminal message based on status + if ( + isinstance(tool_output, dict) + and tool_output.get("status") == "ready" + ): + word_count = tool_output.get("word_count", 0) + yield streaming_service.format_terminal_info( + f"Report generated: {tool_output.get('title', 'Report')} ({word_count:,} words)", + "success", + ) + else: + error_msg = ( + tool_output.get("error", "Unknown error") + if isinstance(tool_output, dict) + else "Unknown error" + ) + yield streaming_service.format_terminal_info( + f"Report generation failed: {error_msg}", + "error", + ) + elif tool_name in ( + "create_notion_page", + "update_notion_page", + "delete_notion_page", + ): + yield streaming_service.format_tool_output_available( + tool_call_id, + tool_output + if isinstance(tool_output, dict) + else {"result": tool_output}, + ) + else: + yield streaming_service.format_tool_output_available( + tool_call_id, + {"status": "completed", "result_length": len(str(tool_output))}, + ) + yield streaming_service.format_terminal_info( + f"Tool {tool_name} completed", "success" + ) + + elif event_type in ("on_chain_end", "on_agent_end"): + if current_text_id is not None: + yield streaming_service.format_text_end(current_text_id) + current_text_id = None + + if current_text_id is not None: + yield streaming_service.format_text_end(current_text_id) + + completion_event = complete_current_step() + if completion_event: + yield completion_event + + result.accumulated_text = accumulated_text + + state = await agent.aget_state(config) + is_interrupted = state.tasks and any(task.interrupts for task in state.tasks) + if is_interrupted: + result.is_interrupted = True + result.interrupt_value = state.tasks[0].interrupts[0].value + yield streaming_service.format_interrupt_request(result.interrupt_value) + + async def stream_new_chat( user_query: str, search_space_id: int, @@ -215,9 +871,6 @@ async def stream_new_chat( """ streaming_service = VercelStreamingService() - # Track the current text block for streaming (defined early for exception handling) - current_text_id: str | None = None - try: # Mark AI as responding to this user for live collaboration if user_id: @@ -394,63 +1047,18 @@ async def stream_new_chat( yield streaming_service.format_message_start() yield streaming_service.format_start_step() - # Reset text tracking for this stream - accumulated_text = "" - - # Track thinking steps for chain-of-thought display - thinking_step_counter = 0 - # Map run_id -> step_id for tool calls so we can update them on completion - tool_step_ids: dict[str, str] = {} - # Track the last active step so we can mark it complete at the end - last_active_step_id: str | None = None - last_active_step_title: str = "" - last_active_step_items: list[str] = [] - # Track which steps have been completed to avoid duplicate completions - completed_step_ids: set[str] = set() - # Track if we just finished a tool (text flows silently after tools) - just_finished_tool: bool = False - # Track write_todos calls to show "Creating plan" vs "Updating plan" - # Disabled for now - # write_todos_call_count: int = 0 - - def next_thinking_step_id() -> str: - nonlocal thinking_step_counter - thinking_step_counter += 1 - return f"thinking-{thinking_step_counter}" - - def complete_current_step() -> str | None: - """Complete the current active step and return the completion event, if any.""" - nonlocal last_active_step_id, last_active_step_title, last_active_step_items - if last_active_step_id and last_active_step_id not in completed_step_ids: - completed_step_ids.add(last_active_step_id) - return streaming_service.format_thinking_step( - step_id=last_active_step_id, - title=last_active_step_title, - status="completed", - items=last_active_step_items if last_active_step_items else None, - ) - return None - # Initial thinking step - analyzing the request - analyze_step_id = next_thinking_step_id() - last_active_step_id = analyze_step_id - - # Determine step title and action verb based on context if mentioned_documents or mentioned_surfsense_docs: - last_active_step_title = "Analyzing referenced content" + initial_title = "Analyzing referenced content" action_verb = "Analyzing" else: - last_active_step_title = "Understanding your request" + initial_title = "Understanding your request" action_verb = "Processing" - # Build the message with inline context about referenced documents processing_parts = [] - - # Add the user query query_text = user_query[:80] + ("..." if len(user_query) > 80 else "") processing_parts.append(query_text) - # Add mentioned document names inline if mentioned_documents: doc_names = [] for doc in mentioned_documents: @@ -463,7 +1071,6 @@ async def stream_new_chat( else: processing_parts.append(f"[{len(doc_names)} documents]") - # Add mentioned SurfSense docs inline if mentioned_surfsense_docs: doc_names = [] for doc in mentioned_surfsense_docs: @@ -476,704 +1083,37 @@ async def stream_new_chat( else: processing_parts.append(f"[{len(doc_names)} docs]") - last_active_step_items = [f"{action_verb}: {' '.join(processing_parts)}"] + initial_items = [f"{action_verb}: {' '.join(processing_parts)}"] + initial_step_id = "thinking-1" yield streaming_service.format_thinking_step( - step_id=analyze_step_id, - title=last_active_step_title, + step_id=initial_step_id, + title=initial_title, status="in_progress", - items=last_active_step_items, + items=initial_items, ) - # Stream the agent response with thread config for memory - async for event in agent.astream_events( - input_state, config=config, version="v2" + stream_result = StreamResult() + async for sse in _stream_agent_events( + agent=agent, + config=config, + input_data=input_state, + streaming_service=streaming_service, + result=stream_result, + step_prefix="thinking", + initial_step_id=initial_step_id, + initial_step_title=initial_title, + initial_step_items=initial_items, ): - event_type = event.get("event", "") + yield sse - # Handle chat model stream events (text streaming) - if event_type == "on_chat_model_stream": - chunk = event.get("data", {}).get("chunk") - if chunk and hasattr(chunk, "content"): - content = chunk.content - if content and isinstance(content, str): - # Start a new text block if needed - if current_text_id is None: - # Complete any previous step - completion_event = complete_current_step() - if completion_event: - yield completion_event + if stream_result.is_interrupted: + yield streaming_service.format_finish_step() + yield streaming_service.format_finish() + yield streaming_service.format_done() + return - if just_finished_tool: - # Clear the active step tracking - text flows without a dedicated step - last_active_step_id = None - last_active_step_title = "" - last_active_step_items = [] - just_finished_tool = False - - current_text_id = streaming_service.generate_text_id() - yield streaming_service.format_text_start(current_text_id) - - # Stream the text delta - yield streaming_service.format_text_delta( - current_text_id, content - ) - accumulated_text += content - - # Handle tool calls - elif event_type == "on_tool_start": - tool_name = event.get("name", "unknown_tool") - run_id = event.get("run_id", "") - tool_input = event.get("data", {}).get("input", {}) - - # End current text block if any - if current_text_id is not None: - yield streaming_service.format_text_end(current_text_id) - current_text_id = None - - # Complete any previous step EXCEPT "Synthesizing response" - # (we want to reuse the Synthesizing step after tools complete) - if last_active_step_title != "Synthesizing response": - completion_event = complete_current_step() - if completion_event: - yield completion_event - - # Reset the just_finished_tool flag since we're starting a new tool - just_finished_tool = False - - # Create thinking step for the tool call and store it for later update - tool_step_id = next_thinking_step_id() - tool_step_ids[run_id] = tool_step_id - last_active_step_id = tool_step_id - if tool_name == "search_knowledge_base": - query = ( - tool_input.get("query", "") - if isinstance(tool_input, dict) - else str(tool_input) - ) - last_active_step_title = "Searching knowledge base" - last_active_step_items = [ - f"Query: {query[:100]}{'...' if len(query) > 100 else ''}" - ] - yield streaming_service.format_thinking_step( - step_id=tool_step_id, - title="Searching knowledge base", - status="in_progress", - items=last_active_step_items, - ) - elif tool_name == "link_preview": - url = ( - tool_input.get("url", "") - if isinstance(tool_input, dict) - else str(tool_input) - ) - last_active_step_title = "Fetching link preview" - last_active_step_items = [ - f"URL: {url[:80]}{'...' if len(url) > 80 else ''}" - ] - yield streaming_service.format_thinking_step( - step_id=tool_step_id, - title="Fetching link preview", - status="in_progress", - items=last_active_step_items, - ) - elif tool_name == "display_image": - src = ( - tool_input.get("src", "") - if isinstance(tool_input, dict) - else str(tool_input) - ) - title = ( - tool_input.get("title", "") - if isinstance(tool_input, dict) - else "" - ) - last_active_step_title = "Analyzing the image" - last_active_step_items = [ - f"Analyzing: {title[:50] if title else src[:50]}{'...' if len(title or src) > 50 else ''}" - ] - yield streaming_service.format_thinking_step( - step_id=tool_step_id, - title="Analyzing the image", - status="in_progress", - items=last_active_step_items, - ) - elif tool_name == "scrape_webpage": - url = ( - tool_input.get("url", "") - if isinstance(tool_input, dict) - else str(tool_input) - ) - last_active_step_title = "Scraping webpage" - last_active_step_items = [ - f"URL: {url[:80]}{'...' if len(url) > 80 else ''}" - ] - yield streaming_service.format_thinking_step( - step_id=tool_step_id, - title="Scraping webpage", - status="in_progress", - items=last_active_step_items, - ) - # elif tool_name == "write_todos": # Disabled for now - # # Track write_todos calls for better messaging - # write_todos_call_count += 1 - # todos = ( - # tool_input.get("todos", []) - # if isinstance(tool_input, dict) - # else [] - # ) - # todo_count = len(todos) if isinstance(todos, list) else 0 - - # if write_todos_call_count == 1: - # # First call - creating the plan - # last_active_step_title = "Creating plan" - # last_active_step_items = [f"Defining {todo_count} tasks..."] - # else: - # # Subsequent calls - updating the plan - # # Try to provide context about what's being updated - # in_progress_count = ( - # sum( - # 1 - # for t in todos - # if isinstance(t, dict) - # and t.get("status") == "in_progress" - # ) - # if isinstance(todos, list) - # else 0 - # ) - # completed_count = ( - # sum( - # 1 - # for t in todos - # if isinstance(t, dict) - # and t.get("status") == "completed" - # ) - # if isinstance(todos, list) - # else 0 - # ) - - # last_active_step_title = "Updating progress" - # last_active_step_items = ( - # [ - # f"Progress: {completed_count}/{todo_count} completed", - # f"In progress: {in_progress_count} tasks", - # ] - # if completed_count > 0 - # else [f"Working on {todo_count} tasks"] - # ) - - # yield streaming_service.format_thinking_step( - # step_id=tool_step_id, - # title=last_active_step_title, - # status="in_progress", - # items=last_active_step_items, - # ) - elif tool_name == "generate_podcast": - podcast_title = ( - tool_input.get("podcast_title", "SurfSense Podcast") - if isinstance(tool_input, dict) - else "SurfSense Podcast" - ) - # Get content length for context - content_len = len( - tool_input.get("source_content", "") - if isinstance(tool_input, dict) - else "" - ) - last_active_step_title = "Generating podcast" - last_active_step_items = [ - f"Title: {podcast_title}", - f"Content: {content_len:,} characters", - "Preparing audio generation...", - ] - yield streaming_service.format_thinking_step( - step_id=tool_step_id, - title="Generating podcast", - status="in_progress", - items=last_active_step_items, - ) - # elif tool_name == "ls": - # last_active_step_title = "Exploring files" - # last_active_step_items = [] - # yield streaming_service.format_thinking_step( - # step_id=tool_step_id, - # title="Exploring files", - # status="in_progress", - # items=None, - # ) - else: - last_active_step_title = f"Using {tool_name.replace('_', ' ')}" - last_active_step_items = [] - yield streaming_service.format_thinking_step( - step_id=tool_step_id, - title=last_active_step_title, - status="in_progress", - ) - - # Stream tool info - tool_call_id = ( - f"call_{run_id[:32]}" - if run_id - else streaming_service.generate_tool_call_id() - ) - yield streaming_service.format_tool_input_start(tool_call_id, tool_name) - yield streaming_service.format_tool_input_available( - tool_call_id, - tool_name, - tool_input - if isinstance(tool_input, dict) - else {"input": tool_input}, - ) - - elif event_type == "on_tool_end": - run_id = event.get("run_id", "") - tool_name = event.get("name", "unknown_tool") - raw_output = event.get("data", {}).get("output", "") - - # Handle deepagents' write_todos Command object specially - # Disabled for now - # if tool_name == "write_todos" and hasattr(raw_output, "update"): - # # deepagents returns a Command object - extract todos directly - # tool_output = extract_todos_from_deepagents(raw_output) - # elif hasattr(raw_output, "content"): - if hasattr(raw_output, "content"): - # It's a ToolMessage object - extract the content - content = raw_output.content - # If content is a string that looks like JSON, try to parse it - if isinstance(content, str): - try: - tool_output = json.loads(content) - except (json.JSONDecodeError, TypeError): - tool_output = {"result": content} - elif isinstance(content, dict): - tool_output = content - else: - tool_output = {"result": str(content)} - elif isinstance(raw_output, dict): - tool_output = raw_output - else: - tool_output = { - "result": str(raw_output) if raw_output else "completed" - } - - tool_call_id = f"call_{run_id[:32]}" if run_id else "call_unknown" - - # Get the original tool step ID to update it (not create a new one) - original_step_id = tool_step_ids.get( - run_id, f"thinking-unknown-{run_id[:8]}" - ) - - # Mark the tool thinking step as completed using the SAME step ID - # Also add to completed set so we don't try to complete it again - completed_step_ids.add(original_step_id) - if tool_name == "search_knowledge_base": - # Get result count if available - result_info = "Search completed" - if isinstance(tool_output, dict): - result_len = tool_output.get("result_length", 0) - if result_len > 0: - result_info = ( - f"Found relevant information ({result_len} chars)" - ) - # Include original query in completed items - completed_items = [*last_active_step_items, result_info] - yield streaming_service.format_thinking_step( - step_id=original_step_id, - title="Searching knowledge base", - status="completed", - items=completed_items, - ) - elif tool_name == "link_preview": - # Build completion items based on link preview result - if isinstance(tool_output, dict): - title = tool_output.get("title", "Link") - domain = tool_output.get("domain", "") - has_error = "error" in tool_output - if has_error: - completed_items = [ - *last_active_step_items, - f"Error: {tool_output.get('error', 'Failed to fetch')}", - ] - else: - completed_items = [ - *last_active_step_items, - f"Title: {title[:60]}{'...' if len(title) > 60 else ''}", - f"Domain: {domain}" if domain else "Preview loaded", - ] - else: - completed_items = [*last_active_step_items, "Preview loaded"] - yield streaming_service.format_thinking_step( - step_id=original_step_id, - title="Fetching link preview", - status="completed", - items=completed_items, - ) - elif tool_name == "display_image": - # Build completion items for image analysis - if isinstance(tool_output, dict): - title = tool_output.get("title", "") - alt = tool_output.get("alt", "Image") - display_name = title or alt - completed_items = [ - *last_active_step_items, - f"Analyzed: {display_name[:50]}{'...' if len(display_name) > 50 else ''}", - ] - else: - completed_items = [*last_active_step_items, "Image analyzed"] - yield streaming_service.format_thinking_step( - step_id=original_step_id, - title="Analyzing the image", - status="completed", - items=completed_items, - ) - elif tool_name == "scrape_webpage": - # Build completion items for webpage scraping - if isinstance(tool_output, dict): - title = tool_output.get("title", "Webpage") - word_count = tool_output.get("word_count", 0) - has_error = "error" in tool_output - if has_error: - completed_items = [ - *last_active_step_items, - f"Error: {tool_output.get('error', 'Failed to scrape')[:50]}", - ] - else: - completed_items = [ - *last_active_step_items, - f"Title: {title[:50]}{'...' if len(title) > 50 else ''}", - f"Extracted: {word_count:,} words", - ] - else: - completed_items = [*last_active_step_items, "Content extracted"] - yield streaming_service.format_thinking_step( - step_id=original_step_id, - title="Scraping webpage", - status="completed", - items=completed_items, - ) - elif tool_name == "generate_podcast": - # Build detailed completion items based on podcast status - podcast_status = ( - tool_output.get("status", "unknown") - if isinstance(tool_output, dict) - else "unknown" - ) - podcast_title = ( - tool_output.get("title", "Podcast") - if isinstance(tool_output, dict) - else "Podcast" - ) - - if podcast_status == "processing": - completed_items = [ - f"Title: {podcast_title}", - "Audio generation started", - "Processing in background...", - ] - elif podcast_status == "already_generating": - completed_items = [ - f"Title: {podcast_title}", - "Podcast already in progress", - "Please wait for it to complete", - ] - elif podcast_status == "error": - error_msg = ( - tool_output.get("error", "Unknown error") - if isinstance(tool_output, dict) - else "Unknown error" - ) - completed_items = [ - f"Title: {podcast_title}", - f"Error: {error_msg[:50]}", - ] - else: - completed_items = last_active_step_items - - yield streaming_service.format_thinking_step( - step_id=original_step_id, - title="Generating podcast", - status="completed", - items=completed_items, - ) - # elif tool_name == "write_todos": # Disabled for now - # # Build completion items for planning/updating - # if isinstance(tool_output, dict): - # todos = tool_output.get("todos", []) - # todo_count = len(todos) if isinstance(todos, list) else 0 - # completed_count = ( - # sum( - # 1 - # for t in todos - # if isinstance(t, dict) - # and t.get("status") == "completed" - # ) - # if isinstance(todos, list) - # else 0 - # ) - # in_progress_count = ( - # sum( - # 1 - # for t in todos - # if isinstance(t, dict) - # and t.get("status") == "in_progress" - # ) - # if isinstance(todos, list) - # else 0 - # ) - - # # Use context-aware completion message - # if last_active_step_title == "Creating plan": - # completed_items = [f"Created {todo_count} tasks"] - # else: - # # Updating progress - show stats - # completed_items = [ - # f"Progress: {completed_count}/{todo_count} completed", - # ] - # if in_progress_count > 0: - # # Find the currently in-progress task name - # in_progress_task = next( - # ( - # t.get("content", "")[:40] - # for t in todos - # if isinstance(t, dict) - # and t.get("status") == "in_progress" - # ), - # None, - # ) - # if in_progress_task: - # completed_items.append( - # f"Current: {in_progress_task}..." - # ) - # else: - # completed_items = ["Plan updated"] - # yield streaming_service.format_thinking_step( - # step_id=original_step_id, - # title=last_active_step_title, - # status="completed", - # items=completed_items, - # ) - elif tool_name == "ls": - # Build completion items showing file names found - if isinstance(tool_output, dict): - result = tool_output.get("result", "") - elif isinstance(tool_output, str): - result = tool_output - else: - result = str(tool_output) if tool_output else "" - - # Parse file paths and extract just the file names - file_names = [] - if result: - # The ls tool returns paths, extract just the file/folder names - for line in result.strip().split("\n"): - line = line.strip() - if line: - # Get just the filename from the path - name = line.rstrip("/").split("/")[-1] - if name and len(name) <= 40: - file_names.append(name) - elif name: - file_names.append(name[:37] + "...") - - # Build display items - wrap file names in brackets for icon rendering - if file_names: - if len(file_names) <= 5: - # Wrap each file name in brackets for styled tile rendering - completed_items = [f"[{name}]" for name in file_names] - else: - # Show first few with brackets and count - completed_items = [f"[{name}]" for name in file_names[:4]] - completed_items.append(f"(+{len(file_names) - 4} more)") - else: - completed_items = ["No files found"] - - yield streaming_service.format_thinking_step( - step_id=original_step_id, - title="Exploring files", - status="completed", - items=completed_items, - ) - else: - yield streaming_service.format_thinking_step( - step_id=original_step_id, - title=f"Using {tool_name.replace('_', ' ')}", - status="completed", - items=last_active_step_items, - ) - - # Mark that we just finished a tool - "Synthesizing response" will be created - # when text actually starts flowing (not immediately) - just_finished_tool = True - # Clear the active step since the tool is done - last_active_step_id = None - last_active_step_title = "" - last_active_step_items = [] - - # Handle different tool outputs - if tool_name == "generate_podcast": - # Stream the full podcast result so frontend can render the audio player - yield streaming_service.format_tool_output_available( - tool_call_id, - tool_output - if isinstance(tool_output, dict) - else {"result": tool_output}, - ) - # Send appropriate terminal message based on status - if ( - isinstance(tool_output, dict) - and tool_output.get("status") == "success" - ): - yield streaming_service.format_terminal_info( - f"Podcast generated successfully: {tool_output.get('title', 'Podcast')}", - "success", - ) - else: - error_msg = ( - tool_output.get("error", "Unknown error") - if isinstance(tool_output, dict) - else "Unknown error" - ) - yield streaming_service.format_terminal_info( - f"Podcast generation failed: {error_msg}", - "error", - ) - elif tool_name == "link_preview": - # Stream the full link preview result so frontend can render the MediaCard - yield streaming_service.format_tool_output_available( - tool_call_id, - tool_output - if isinstance(tool_output, dict) - else {"result": tool_output}, - ) - # Send appropriate terminal message - if isinstance(tool_output, dict) and "error" not in tool_output: - title = tool_output.get("title", "Link") - yield streaming_service.format_terminal_info( - f"Link preview loaded: {title[:50]}{'...' if len(title) > 50 else ''}", - "success", - ) - else: - error_msg = ( - tool_output.get("error", "Failed to fetch") - if isinstance(tool_output, dict) - else "Failed to fetch" - ) - yield streaming_service.format_terminal_info( - f"Link preview failed: {error_msg}", - "error", - ) - elif tool_name == "display_image": - # Stream the full image result so frontend can render the Image component - yield streaming_service.format_tool_output_available( - tool_call_id, - tool_output - if isinstance(tool_output, dict) - else {"result": tool_output}, - ) - # Send terminal message - if isinstance(tool_output, dict): - title = tool_output.get("title") or tool_output.get( - "alt", "Image" - ) - yield streaming_service.format_terminal_info( - f"Image analyzed: {title[:40]}{'...' if len(title) > 40 else ''}", - "success", - ) - elif tool_name == "scrape_webpage": - # Stream the scrape result so frontend can render the Article component - # Note: We send metadata for display, but content goes to LLM for processing - if isinstance(tool_output, dict): - # Create a display-friendly output (without full content for the card) - display_output = { - k: v for k, v in tool_output.items() if k != "content" - } - # But keep a truncated content preview - if "content" in tool_output: - content = tool_output.get("content", "") - display_output["content_preview"] = ( - content[:500] + "..." if len(content) > 500 else content - ) - yield streaming_service.format_tool_output_available( - tool_call_id, - display_output, - ) - else: - yield streaming_service.format_tool_output_available( - tool_call_id, - {"result": tool_output}, - ) - # Send terminal message - if isinstance(tool_output, dict) and "error" not in tool_output: - title = tool_output.get("title", "Webpage") - word_count = tool_output.get("word_count", 0) - yield streaming_service.format_terminal_info( - f"Scraped: {title[:40]}{'...' if len(title) > 40 else ''} ({word_count:,} words)", - "success", - ) - else: - error_msg = ( - tool_output.get("error", "Failed to scrape") - if isinstance(tool_output, dict) - else "Failed to scrape" - ) - yield streaming_service.format_terminal_info( - f"Scrape failed: {error_msg}", - "error", - ) - elif tool_name == "search_knowledge_base": - # Don't stream the full output for search (can be very large), just acknowledge - yield streaming_service.format_tool_output_available( - tool_call_id, - {"status": "completed", "result_length": len(str(tool_output))}, - ) - yield streaming_service.format_terminal_info( - "Knowledge base search completed", "success" - ) - # elif tool_name == "write_todos": # Disabled for now - # # Stream the full write_todos result so frontend can render the Plan component - # yield streaming_service.format_tool_output_available( - # tool_call_id, - # tool_output - # if isinstance(tool_output, dict) - # else {"result": tool_output}, - # ) - # # Send terminal message with plan info - # if isinstance(tool_output, dict): - # todos = tool_output.get("todos", []) - # todo_count = len(todos) if isinstance(todos, list) else 0 - # yield streaming_service.format_terminal_info( - # f"Plan created ({todo_count} tasks)", - # "success", - # ) - # else: - # yield streaming_service.format_terminal_info( - # "Plan created", - # "success", - # ) - else: - # Default handling for other tools - yield streaming_service.format_tool_output_available( - tool_call_id, - {"status": "completed", "result_length": len(str(tool_output))}, - ) - yield streaming_service.format_terminal_info( - f"Tool {tool_name} completed", "success" - ) - - # Handle chain/agent end to close any open text blocks - elif event_type in ("on_chain_end", "on_agent_end"): - if current_text_id is not None: - yield streaming_service.format_text_end(current_text_id) - current_text_id = None - - # Ensure text block is closed - if current_text_id is not None: - yield streaming_service.format_text_end(current_text_id) - - # Mark the last active thinking step as completed using the same title - completion_event = complete_current_step() - if completion_event: - yield completion_event + accumulated_text = stream_result.accumulated_text # Generate LLM title for new chats after first response # Check if this is the first assistant response by counting existing assistant messages @@ -1245,10 +1185,6 @@ async def stream_new_chat( print(f"[stream_new_chat] Exception type: {type(e).__name__}") print(f"[stream_new_chat] Traceback:\n{traceback.format_exc()}") - # Close any open text block - if current_text_id is not None: - yield streaming_service.format_text_end(current_text_id) - yield streaming_service.format_error(error_message) yield streaming_service.format_finish_step() yield streaming_service.format_finish() @@ -1257,3 +1193,120 @@ async def stream_new_chat( finally: # Clear AI responding state for live collaboration await clear_ai_responding(session, chat_id) + + +async def stream_resume_chat( + chat_id: int, + search_space_id: int, + decisions: list[dict], + session: AsyncSession, + user_id: str | None = None, + llm_config_id: int = -1, + thread_visibility: ChatVisibility | None = None, +) -> AsyncGenerator[str, None]: + streaming_service = VercelStreamingService() + + try: + if user_id: + await set_ai_responding(session, chat_id, UUID(user_id)) + + agent_config: AgentConfig | None = None + if llm_config_id >= 0: + agent_config = await load_agent_config( + session=session, + config_id=llm_config_id, + search_space_id=search_space_id, + ) + if not agent_config: + yield streaming_service.format_error( + f"Failed to load NewLLMConfig with id {llm_config_id}" + ) + yield streaming_service.format_done() + return + llm = create_chat_litellm_from_agent_config(agent_config) + else: + llm_config = load_llm_config_from_yaml(llm_config_id=llm_config_id) + if not llm_config: + yield streaming_service.format_error( + f"Failed to load LLM config with id {llm_config_id}" + ) + yield streaming_service.format_done() + return + llm = create_chat_litellm_from_config(llm_config) + agent_config = AgentConfig.from_yaml_config(llm_config) + + if not llm: + yield streaming_service.format_error("Failed to create LLM instance") + yield streaming_service.format_done() + return + + connector_service = ConnectorService(session, search_space_id=search_space_id) + + from app.db import SearchSourceConnectorType + + firecrawl_api_key = None + webcrawler_connector = await connector_service.get_connector_by_type( + SearchSourceConnectorType.WEBCRAWLER_CONNECTOR, search_space_id + ) + if webcrawler_connector and webcrawler_connector.config: + firecrawl_api_key = webcrawler_connector.config.get("FIRECRAWL_API_KEY") + + checkpointer = await get_checkpointer() + visibility = thread_visibility or ChatVisibility.PRIVATE + + agent = await create_surfsense_deep_agent( + llm=llm, + search_space_id=search_space_id, + db_session=session, + connector_service=connector_service, + checkpointer=checkpointer, + user_id=user_id, + thread_id=chat_id, + agent_config=agent_config, + firecrawl_api_key=firecrawl_api_key, + thread_visibility=visibility, + ) + + from langgraph.types import Command + + config = { + "configurable": {"thread_id": str(chat_id)}, + "recursion_limit": 80, + } + + yield streaming_service.format_message_start() + yield streaming_service.format_start_step() + + stream_result = StreamResult() + async for sse in _stream_agent_events( + agent=agent, + config=config, + input_data=Command(resume={"decisions": decisions}), + streaming_service=streaming_service, + result=stream_result, + step_prefix="thinking-resume", + ): + yield sse + if stream_result.is_interrupted: + yield streaming_service.format_finish_step() + yield streaming_service.format_finish() + yield streaming_service.format_done() + return + + yield streaming_service.format_finish_step() + yield streaming_service.format_finish() + yield streaming_service.format_done() + + except Exception as e: + import traceback + + error_message = f"Error during resume: {e!s}" + print(f"[stream_resume_chat] {error_message}") + print(f"[stream_resume_chat] Traceback:\n{traceback.format_exc()}") + yield streaming_service.format_error(error_message) + yield streaming_service.format_finish_step() + yield streaming_service.format_finish() + yield streaming_service.format_done() + + finally: + await clear_ai_responding(session, chat_id) diff --git a/surfsense_backend/app/utils/periodic_scheduler.py b/surfsense_backend/app/utils/periodic_scheduler.py index aa8c07ce4..9ea45df63 100644 --- a/surfsense_backend/app/utils/periodic_scheduler.py +++ b/surfsense_backend/app/utils/periodic_scheduler.py @@ -34,6 +34,7 @@ CONNECTOR_TASK_MAP = { SearchSourceConnectorType.ELASTICSEARCH_CONNECTOR: "index_elasticsearch_documents", SearchSourceConnectorType.WEBCRAWLER_CONNECTOR: "index_crawled_urls", SearchSourceConnectorType.BOOKSTACK_CONNECTOR: "index_bookstack_pages", + SearchSourceConnectorType.OBSIDIAN_CONNECTOR: "index_obsidian_vault", } @@ -99,6 +100,7 @@ def create_periodic_schedule( index_linear_issues_task, index_luma_events_task, index_notion_pages_task, + index_obsidian_vault_task, index_slack_messages_task, ) @@ -119,6 +121,7 @@ def create_periodic_schedule( SearchSourceConnectorType.ELASTICSEARCH_CONNECTOR: index_elasticsearch_documents_task, SearchSourceConnectorType.WEBCRAWLER_CONNECTOR: index_crawled_urls_task, SearchSourceConnectorType.BOOKSTACK_CONNECTOR: index_bookstack_pages_task, + SearchSourceConnectorType.OBSIDIAN_CONNECTOR: index_obsidian_vault_task, } # Trigger the first run immediately diff --git a/surfsense_backend/pyproject.toml b/surfsense_backend/pyproject.toml index 99c888b14..fe997ee16 100644 --- a/surfsense_backend/pyproject.toml +++ b/surfsense_backend/pyproject.toml @@ -63,6 +63,8 @@ dependencies = [ "unstructured-client>=0.42.3", "langchain-unstructured>=1.0.1", "slowapi>=0.1.9", + "pypandoc_binary>=1.16.2", + "typst>=0.14.0", ] [dependency-groups] diff --git a/surfsense_backend/scripts/docker/entrypoint.sh b/surfsense_backend/scripts/docker/entrypoint.sh index ce0f1ce13..e1a2778fc 100644 --- a/surfsense_backend/scripts/docker/entrypoint.sh +++ b/surfsense_backend/scripts/docker/entrypoint.sh @@ -73,6 +73,12 @@ start_worker() { QUEUE_ARGS="" if [ -n "${CELERY_QUEUES}" ]; then QUEUE_ARGS="--queues=${CELERY_QUEUES}" + else + # When no queues specified, consume from BOTH the default queue and + # the connectors queue. Without --queues, Celery only consumes from + # the default queue, leaving connector indexing tasks stuck. + DEFAULT_Q="${CELERY_TASK_DEFAULT_QUEUE:-surfsense}" + QUEUE_ARGS="--queues=${DEFAULT_Q},${DEFAULT_Q}.connectors" fi echo "Starting Celery Worker (autoscale=${CELERY_MAX_WORKERS},${CELERY_MIN_WORKERS}, max-tasks-per-child=${CELERY_MAX_TASKS_PER_CHILD}, queues=${CELERY_QUEUES:-all})..." diff --git a/surfsense_backend/uv.lock b/surfsense_backend/uv.lock index 0697cd9fe..ac2576d74 100644 --- a/surfsense_backend/uv.lock +++ b/surfsense_backend/uv.lock @@ -5719,6 +5719,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bb/e9/b145683854189bba84437ea569bfa786f408c8dc5bc16d8eb0753f5583bf/pypandoc-1.16.2-py3-none-any.whl", hash = "sha256:c200c1139c8e3247baf38d1e9279e85d9f162499d1999c6aa8418596558fe79b", size = 19451, upload-time = "2025-11-13T16:30:07.66Z" }, ] +[[package]] +name = "pypandoc-binary" +version = "1.16.2" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/e0/dcccafc2d8b343408abe1e4f84fa5f854eb29f2b59323a6d9fb95db333d1/pypandoc_binary-1.16.2-py3-none-macosx_10_9_x86_64.whl", hash = "sha256:dc71533dcce1cb112994e44f95a21c042c02e4fd4b4815442d8cbf7fe0159a88", size = 25526445, upload-time = "2025-11-13T16:30:09.786Z" }, + { url = "https://files.pythonhosted.org/packages/49/8f/5f8e7d7febad6972b826537fdb4d78db2dbed8f40f141f58cf1bd95cca7e/pypandoc_binary-1.16.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:a240d645963266be5d68c2b5a0fb9703dfa5bb5e6a8367e36060e2584236b457", size = 25526446, upload-time = "2025-11-13T16:30:12.627Z" }, + { url = "https://files.pythonhosted.org/packages/31/ae/904a96cd84ef568fa7a8a6a2a783265dda1542fd6ee489ef6fbaf32dfc04/pypandoc_binary-1.16.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d9d4e14bbf973578ba635d18730131f76d11cf6926e4be977b39fad0bf330a8", size = 36942728, upload-time = "2025-11-13T16:30:15.302Z" }, + { url = "https://files.pythonhosted.org/packages/55/97/4c7e2543013ed6b8c1c7d7344ae5e3f0fa8df67c6712b90a7d2dc87669c0/pypandoc_binary-1.16.2-py3-none-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6108d0661298f97def07695527b73f8499e7c425c8386ae0de2e8bf95d5799c", size = 34049538, upload-time = "2025-11-13T16:30:18.467Z" }, + { url = "https://files.pythonhosted.org/packages/63/72/7ddda95c6ea454296c450191860ecb6f833534d605288fa120bfbfa97c1f/pypandoc_binary-1.16.2-py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:a7e03052c7e526188e970aa782245e10be815a292c74b14d350d81c19fdeeafb", size = 36942720, upload-time = "2025-11-13T16:30:21.116Z" }, + { url = "https://files.pythonhosted.org/packages/6f/fd/0dadb4c5ed01a98bfef1a3d63386686a43e19cf779181f5aea3e31bba800/pypandoc_binary-1.16.2-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:49a2bea5a79b20eca546b67d466f41d97cecf10e2efb366a7bef7cc3d4edef15", size = 34049518, upload-time = "2025-11-13T16:30:24.461Z" }, + { url = "https://files.pythonhosted.org/packages/73/a5/8065640aa3acefafe784ed9dc8338ea4516d40c60e151d727369f609eaa1/pypandoc_binary-1.16.2-py3-none-win_amd64.whl", hash = "sha256:9005c32c9d62152cbdd00e5e38579e17c0e708789ef554852367e24fb0cc6cf3", size = 40229130, upload-time = "2025-11-13T16:30:27.624Z" }, +] + [[package]] name = "pyparsing" version = "3.2.3" @@ -6868,6 +6882,7 @@ dependencies = [ { name = "playwright" }, { name = "psycopg", extra = ["binary", "pool"] }, { name = "pyarrow" }, + { name = "pypandoc-binary" }, { name = "pypdf" }, { name = "python-ffmpeg" }, { name = "redis" }, @@ -6882,6 +6897,7 @@ dependencies = [ { name = "static-ffmpeg" }, { name = "tavily-python" }, { name = "trafilatura" }, + { name = "typst" }, { name = "unstructured", extra = ["all-docs"] }, { name = "unstructured-client" }, { name = "uvicorn", extra = ["standard"] }, @@ -6936,6 +6952,7 @@ requires-dist = [ { name = "playwright", specifier = ">=1.50.0" }, { name = "psycopg", extras = ["binary", "pool"], specifier = ">=3.3.2" }, { name = "pyarrow", specifier = ">=15.0.0,<19.0.0" }, + { name = "pypandoc-binary", specifier = ">=1.16.2" }, { name = "pypdf", specifier = ">=5.1.0" }, { name = "python-ffmpeg", specifier = ">=2.0.12" }, { name = "redis", specifier = ">=5.2.1" }, @@ -6950,6 +6967,7 @@ requires-dist = [ { name = "static-ffmpeg", specifier = ">=2.13" }, { name = "tavily-python", specifier = ">=0.3.2" }, { name = "trafilatura", specifier = ">=2.0.0" }, + { name = "typst", specifier = ">=0.14.0" }, { name = "unstructured", extras = ["all-docs"], specifier = ">=0.18.31" }, { name = "unstructured-client", specifier = ">=0.42.3" }, { name = "uvicorn", extras = ["standard"], specifier = ">=0.34.0" }, @@ -7481,6 +7499,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/17/69/cd203477f944c353c31bade965f880aa1061fd6bf05ded0726ca845b6ff7/typing_inspection-0.4.1-py3-none-any.whl", hash = "sha256:389055682238f53b04f7badcb49b989835495a96700ced5dab2d8feae4b26f51", size = 14552, upload-time = "2025-05-21T18:55:22.152Z" }, ] +[[package]] +name = "typst" +version = "0.14.8" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0e/17/011059074fe6c51ed775991d5066c73443f17d49b3d4ab9c1a969dcdb5cb/typst-0.14.8.tar.gz", hash = "sha256:8ffb8d5896aa6a20a7b88ae3fa1dfcf062fdd09b5b6a0a164f92f78ad1a2d8cd", size = 62369, upload-time = "2026-02-08T02:31:21.753Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/67/af5551e95261fc425f6dbf241ec08bf1172fd10ef239787ff6e009bb2f08/typst-0.14.8-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:4697b9de12d7b1bc85209960e1ef7e2c4947cffd7d6ef68201aea03597cf38bd", size = 22935370, upload-time = "2026-02-08T02:30:30.418Z" }, + { url = "https://files.pythonhosted.org/packages/6f/93/cbb32c7e830a806105ee0f6d9b6c780f2736a9c75d8121602e7842a316d2/typst-0.14.8-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:e2ecb523ff7e3eb68667ad693ff4c460ac58aedfbeb6514054efce2718e7563f", size = 22624078, upload-time = "2026-02-08T02:30:33.762Z" }, + { url = "https://files.pythonhosted.org/packages/77/38/070c068442a8be93125366b27e5cf1a6b1dd62c85dab62bd6d4355643d29/typst-0.14.8-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f9db137ca037bd12c0ebbbbfa1190fffaa75a2043d04adacb273cc98f0265a32", size = 26894087, upload-time = "2026-02-08T02:30:36.876Z" }, + { url = "https://files.pythonhosted.org/packages/ee/32/8754413c4cdf631c51e16690775dcfd28e783c1ccc0efc71d92ef73e0db3/typst-0.14.8-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d8c4ac751c3480b0fcfc7fce273025bb7392654db5a3aa65904f8678192c54f8", size = 26489748, upload-time = "2026-02-08T02:30:40.169Z" }, + { url = "https://files.pythonhosted.org/packages/27/2b/3b1256033c7b971d0c79af41fadff552c1df7a9f9774a540f1a2ede97937/typst-0.14.8-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:37da60ec4afcd82b55664612aab10cac11a8ebc075686057705261de9e901523", size = 28023293, upload-time = "2026-02-08T02:30:43.222Z" }, + { url = "https://files.pythonhosted.org/packages/61/1b/8769c89998299525e4b04fddce1b15977d18051695c65760203b55f7ed47/typst-0.14.8-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e1cfbc313ba3b883da8c45233506766a503da307057a5d8d39e360023733c463", size = 27109055, upload-time = "2026-02-08T02:30:46.214Z" }, + { url = "https://files.pythonhosted.org/packages/51/97/b1f43e29051401289b6ef37398eb83d78584f52e0b213f8675b9b10b0c0b/typst-0.14.8-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b509e7a599dd07e36e18495f0258511de527f5e0dc145622025d204c84db5246", size = 26017464, upload-time = "2026-02-08T02:30:49.271Z" }, + { url = "https://files.pythonhosted.org/packages/5b/ca/44732fc1e486be822ba65ee9a02f0bc5f28d1cb08284c9dd1286d975f9ce/typst-0.14.8-cp314-cp314t-win_amd64.whl", hash = "sha256:10710c58dbc8820a954970ba5d0af5611c7c57f8ddacfebb1a85ddb6449f01eb", size = 21471708, upload-time = "2026-02-08T02:30:54.185Z" }, + { url = "https://files.pythonhosted.org/packages/5b/cb/e49219a75d39ce866ae5d64e0a1d8d712b394ed3a1e7de3a8f4a35cde78e/typst-0.14.8-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f47fe029f6ebe907f981ce0cb5208eab27eaf7342e319e6c798ac1dbae976f58", size = 22936285, upload-time = "2026-02-08T02:30:57.338Z" }, + { url = "https://files.pythonhosted.org/packages/f8/6b/d36f312c32b70303abd88d0abe6ffb50f8f7fcc0b457c914c78d791ed934/typst-0.14.8-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:aba11243463f6994ca1140b8515e70be1a98fd3025ae3211b84103499b0c5a5a", size = 22632767, upload-time = "2026-02-08T02:31:00.454Z" }, + { url = "https://files.pythonhosted.org/packages/6f/b4/87d2d24078b94645ba8788c8b4a5bbab6a3c779370141c31a02e2003ee0f/typst-0.14.8-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:544fcd9ce55b140115d7442b3661c45897778650c307e2eb0749efed29bbfcea", size = 26907232, upload-time = "2026-02-08T02:31:03.528Z" }, + { url = "https://files.pythonhosted.org/packages/bc/e8/3efdebcf37639daa4799e7a4c833a280f14685f7e6058fed576c6fb2e722/typst-0.14.8-cp38-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a296f85bf0d27043b031d1d2d74a34802e4876a8936f70784fbe99021b0dad4d", size = 26501791, upload-time = "2026-02-08T02:31:06.898Z" }, + { url = "https://files.pythonhosted.org/packages/7e/28/094d4b9f0ff4ee81f88eee2df00dbcfbd961070df981973bc385a1544ff8/typst-0.14.8-cp38-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6e3891a2e5551017c9030dd6de31587a29b97c18464df6bcff05f30f7cdab677", size = 28028881, upload-time = "2026-02-08T02:31:10.437Z" }, + { url = "https://files.pythonhosted.org/packages/11/a1/15cd399dfc5ce0ea9e05d5bbc274c95f8ecababc04b4210bae8d583fe454/typst-0.14.8-cp38-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7a19cf938607c73fd8c5245a7cb32c94af413080a3d747fcf7e16df88713c686", size = 27128399, upload-time = "2026-02-08T02:31:13.597Z" }, + { url = "https://files.pythonhosted.org/packages/4c/6f/ff1c58dac9245d4c355bfced006090b14a2f17497e9cf79a84d9d720663a/typst-0.14.8-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a12766a83e390377008722a8c80afdd9195a297261fd3c9d1f3720f9aecd2b19", size = 26026753, upload-time = "2026-02-08T02:31:16.591Z" }, + { url = "https://files.pythonhosted.org/packages/a9/42/db15d775c09f0da92191ea1b50cee056e46b599bb5524e2d8ff51f973765/typst-0.14.8-cp38-abi3-win_amd64.whl", hash = "sha256:66eb2ebfe13275cf2a63ed7ff261eb5af3da5293077a5d6ca16e27a96d0d2f5e", size = 21475900, upload-time = "2026-02-08T02:31:19.484Z" }, +] + [[package]] name = "tzdata" version = "2025.2" diff --git a/surfsense_web/app/(home)/announcements/page.tsx b/surfsense_web/app/(home)/announcements/page.tsx new file mode 100644 index 000000000..5707a6182 --- /dev/null +++ b/surfsense_web/app/(home)/announcements/page.tsx @@ -0,0 +1,350 @@ +"use client"; + +import { + Bell, + BellOff, + CheckCheck, + ExternalLink, + Filter, + Info, + type Megaphone, + Rocket, + Wrench, + X, + Zap, +} from "lucide-react"; +import Link from "next/link"; +import { useState } from "react"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { + Card, + CardContent, + CardDescription, + CardFooter, + CardHeader, + CardTitle, +} from "@/components/ui/card"; +import { + DropdownMenu, + DropdownMenuCheckboxItem, + DropdownMenuContent, + DropdownMenuLabel, + DropdownMenuSeparator, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; +import { Separator } from "@/components/ui/separator"; +import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; +import type { AnnouncementCategory } from "@/contracts/types/announcement.types"; +import { type AnnouncementWithState, useAnnouncements } from "@/hooks/use-announcements"; +import { formatRelativeDate } from "@/lib/format-date"; + +// --------------------------------------------------------------------------- +// Category configuration +// --------------------------------------------------------------------------- + +const categoryConfig: Record< + AnnouncementCategory, + { + label: string; + icon: typeof Megaphone; + color: string; + badgeVariant: "default" | "secondary" | "destructive" | "outline"; + } +> = { + feature: { + label: "Feature", + icon: Rocket, + color: "text-emerald-500", + badgeVariant: "default", + }, + update: { + label: "Update", + icon: Zap, + color: "text-blue-500", + badgeVariant: "secondary", + }, + maintenance: { + label: "Maintenance", + icon: Wrench, + color: "text-amber-500", + badgeVariant: "outline", + }, + info: { + label: "Info", + icon: Info, + color: "text-muted-foreground", + badgeVariant: "secondary", + }, +}; + +// --------------------------------------------------------------------------- +// Announcement card +// --------------------------------------------------------------------------- + +function AnnouncementCard({ + announcement, + onMarkRead, + onDismiss, +}: { + announcement: AnnouncementWithState; + onMarkRead: (id: string) => void; + onDismiss: (id: string) => void; +}) { + const config = categoryConfig[announcement.category]; + const Icon = config.icon; + + return ( + + +
+
+
+ +
+
+
+ {announcement.title} + + {config.label} + + {announcement.isImportant && ( + + + Important + + )} + {!announcement.isRead && ( + + )} +
+ + {formatRelativeDate(announcement.date)} + +
+
+ + {/* Actions */} +
+ {!announcement.isRead && ( + + + + + Mark as read + + )} + + + + + Dismiss + +
+
+
+ + +

{announcement.description}

+
+ + {announcement.link && ( + + + + )} +
+ ); +} + +// --------------------------------------------------------------------------- +// Empty state +// --------------------------------------------------------------------------- + +function EmptyState({ hasFilters }: { hasFilters: boolean }) { + return ( +
+
+ {hasFilters ? ( + + ) : ( + + )} +
+

+ {hasFilters ? "No matching announcements" : "No announcements"} +

+

+ {hasFilters + ? "Try adjusting your filters to see more announcements." + : "You're all caught up! New announcements will appear here."} +

+
+ ); +} + +// --------------------------------------------------------------------------- +// Page +// --------------------------------------------------------------------------- + +export default function AnnouncementsPage() { + const [activeCategories, setActiveCategories] = useState([]); + const [showOnlyUnread, setShowOnlyUnread] = useState(false); + + const { announcements, unreadCount, markRead, markAllRead, dismiss } = useAnnouncements({ + includeDismissed: false, + }); + + // Apply local filters + const filteredAnnouncements = announcements.filter((a) => { + if (activeCategories.length > 0 && !activeCategories.includes(a.category)) return false; + if (showOnlyUnread && a.isRead) return false; + return true; + }); + + const hasActiveFilters = activeCategories.length > 0 || showOnlyUnread; + + const toggleCategory = (cat: AnnouncementCategory) => { + setActiveCategories((prev) => + prev.includes(cat) ? prev.filter((c) => c !== cat) : [...prev, cat] + ); + }; + + return ( + +
+ {/* Header */} +
+
+
+

+ Announcements +

+
+
+
+ + {/* Content */} +
+ {/* Toolbar */} +
+
+ {/* Category filter dropdown */} + + + + + + Categories + + {(Object.keys(categoryConfig) as AnnouncementCategory[]).map((cat) => { + const cfg = categoryConfig[cat]; + const CatIcon = cfg.icon; + return ( + toggleCategory(cat)} + > + + {cfg.label} + + ); + })} + + setShowOnlyUnread((v) => !v)} + > + + Unread only + + + + + {hasActiveFilters && ( + + )} +
+ + {/* Mark all read */} + {unreadCount > 0 && ( + + )} +
+ + + + {/* Announcement list */} + {filteredAnnouncements.length === 0 ? ( + + ) : ( +
+ {filteredAnnouncements.map((announcement) => ( + + ))} +
+ )} +
+
+
+ ); +} diff --git a/surfsense_web/app/(home)/page.tsx b/surfsense_web/app/(home)/page.tsx index e0478fce3..b5a726423 100644 --- a/surfsense_web/app/(home)/page.tsx +++ b/surfsense_web/app/(home)/page.tsx @@ -5,11 +5,13 @@ import { FeaturesBentoGrid } from "@/components/homepage/features-bento-grid"; import { FeaturesCards } from "@/components/homepage/features-card"; import { HeroSection } from "@/components/homepage/hero-section"; import ExternalIntegrations from "@/components/homepage/integrations"; +import { UseCasesGrid } from "@/components/homepage/use-cases-grid"; export default function HomePage() { return (
+ diff --git a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx index bc5aca91e..fde462a44 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx @@ -28,15 +28,21 @@ import { // extractWriteTodosFromContent, hydratePlanStateAtom, } from "@/atoms/chat/plan-state.atom"; +import { closeReportPanelAtom } from "@/atoms/chat/report-panel.atom"; import { membersAtom } from "@/atoms/members/members-query.atoms"; import { currentUserAtom } from "@/atoms/user/user-query.atoms"; import { Thread } from "@/components/assistant-ui/thread"; import { ChatHeader } from "@/components/new-chat/chat-header"; +import { ReportPanel } from "@/components/report-panel/report-panel"; +import { CreateNotionPageToolUI } from "@/components/tool-ui/create-notion-page"; import type { ThinkingStep } from "@/components/tool-ui/deepagent-thinking"; +import { DeleteNotionPageToolUI } from "@/components/tool-ui/delete-notion-page"; import { DisplayImageToolUI } from "@/components/tool-ui/display-image"; import { GeneratePodcastToolUI } from "@/components/tool-ui/generate-podcast"; +import { GenerateReportToolUI } from "@/components/tool-ui/generate-report"; import { LinkPreviewToolUI } from "@/components/tool-ui/link-preview"; import { ScrapeWebpageToolUI } from "@/components/tool-ui/scrape-webpage"; +import { UpdateNotionPageToolUI } from "@/components/tool-ui/update-notion-page"; import { RecallMemoryToolUI, SaveMemoryToolUI } from "@/components/tool-ui/user-memory"; import { Skeleton } from "@/components/ui/skeleton"; import { useChatSessionStateSync } from "@/hooks/use-chat-session-state"; @@ -50,6 +56,17 @@ import { looksLikePodcastRequest, setActivePodcastTaskId, } from "@/lib/chat/podcast-state"; +import { + addToolCall, + appendText, + buildContentForPersistence, + buildContentForUI, + type ContentPart, + type ContentPartsState, + readSSEStream, + type ThinkingStepData, + updateToolCall, +} from "@/lib/chat/streaming-state"; import { appendMessage, createThread, @@ -117,22 +134,16 @@ function extractMentionedDocuments(content: unknown): MentionedDocumentInfo[] { */ const TOOLS_WITH_UI = new Set([ "generate_podcast", + "generate_report", "link_preview", "display_image", + "delete_notion_page", "scrape_webpage", + "create_notion_page", + "update_notion_page", // "write_todos", // Disabled for now ]); -/** - * Type for thinking step data from the backend - */ -interface ThinkingStepData { - id: string; - title: string; - status: "pending" | "in_progress" | "completed"; - items: string[]; -} - export default function NewChatPage() { const params = useParams(); const queryClient = useQueryClient(); @@ -147,6 +158,11 @@ export default function NewChatPage() { new Map() ); const abortControllerRef = useRef(null); + const [pendingInterrupt, setPendingInterrupt] = useState<{ + threadId: number; + assistantMsgId: string; + interruptData: Record; + } | null>(null); // Get mentioned document IDs from the composer const mentionedDocumentIds = useAtomValue(mentionedDocumentIdsAtom); @@ -158,6 +174,7 @@ export default function NewChatPage() { const setCurrentThreadState = useSetAtom(currentThreadAtom); const setTargetCommentId = useSetAtom(setTargetCommentIdAtom); const clearTargetCommentId = useSetAtom(clearTargetCommentIdAtom); + const closeReportPanel = useSetAtom(closeReportPanelAtom); // Get current user for author info in shared chats const { data: currentUser } = useAtomValue(currentUserAtom); @@ -251,6 +268,7 @@ export default function NewChatPage() { setMentionedDocuments([]); setMessageDocumentsMap({}); clearPlanOwnerRegistry(); // Reset plan ownership for new chat + closeReportPanel(); // Close report panel when switching chats try { if (urlChatId > 0) { @@ -315,6 +333,7 @@ export default function NewChatPage() { setMentionedDocumentIds, setMentionedDocuments, hydratePlanState, + closeReportPanel, ]); // Initialize on mount @@ -378,6 +397,16 @@ export default function NewChatPage() { })); }, [currentThread, setCurrentThreadState]); + // Cleanup on unmount - abort any in-flight requests + useEffect(() => { + return () => { + if (abortControllerRef.current) { + abortControllerRef.current.abort(); + abortControllerRef.current = null; + } + }; + }, []); + // Cancel ongoing request const cancelRun = useCallback(async () => { if (abortControllerRef.current) { @@ -533,101 +562,13 @@ export default function NewChatPage() { const assistantMsgId = `msg-assistant-${Date.now()}`; const currentThinkingSteps = new Map(); - // Ordered content parts to preserve inline tool call positions - // Each part is either a text segment or a tool call - type ContentPart = - | { type: "text"; text: string } - | { - type: "tool-call"; - toolCallId: string; - toolName: string; - args: Record; - result?: unknown; - }; - const contentParts: ContentPart[] = []; - - // Track the current text segment index (for appending text deltas) - let currentTextPartIndex = -1; - - // Map to track tool call indices for updating results - const toolCallIndices = new Map(); - - // Helper to get or create the current text part for appending text - const appendText = (delta: string) => { - if (currentTextPartIndex >= 0 && contentParts[currentTextPartIndex]?.type === "text") { - // Append to existing text part - (contentParts[currentTextPartIndex] as { type: "text"; text: string }).text += delta; - } else { - // Create new text part - contentParts.push({ type: "text", text: delta }); - currentTextPartIndex = contentParts.length - 1; - } - }; - - // Helper to add a tool call (this "breaks" the current text segment) - const addToolCall = (toolCallId: string, toolName: string, args: Record) => { - if (TOOLS_WITH_UI.has(toolName)) { - contentParts.push({ - type: "tool-call", - toolCallId, - toolName, - args, - }); - toolCallIndices.set(toolCallId, contentParts.length - 1); - // Reset text part index so next text creates a new segment - currentTextPartIndex = -1; - } - }; - - // Helper to update a tool call's args or result - const updateToolCall = ( - toolCallId: string, - update: { args?: Record; result?: unknown } - ) => { - const index = toolCallIndices.get(toolCallId); - if (index !== undefined && contentParts[index]?.type === "tool-call") { - const tc = contentParts[index] as ContentPart & { type: "tool-call" }; - if (update.args) tc.args = update.args; - if (update.result !== undefined) tc.result = update.result; - } - }; - - // Helper to build content for UI (without thinking-steps to avoid assistant-ui errors) - const buildContentForUI = (): ThreadMessageLike["content"] => { - // Filter to only include text parts with content and tool-calls with UI - const filtered = contentParts.filter((part) => { - if (part.type === "text") return part.text.length > 0; - if (part.type === "tool-call") return TOOLS_WITH_UI.has(part.toolName); - return false; - }); - return filtered.length > 0 - ? (filtered as ThreadMessageLike["content"]) - : [{ type: "text", text: "" }]; - }; - - // Helper to build content for persistence (includes thinking-steps for restoration) - const buildContentForPersistence = (): unknown[] => { - const parts: unknown[] = []; - - // Include thinking steps for persistence - if (currentThinkingSteps.size > 0) { - parts.push({ - type: "thinking-steps", - steps: Array.from(currentThinkingSteps.values()), - }); - } - - // Add content parts (filtered) - for (const part of contentParts) { - if (part.type === "text" && part.text.length > 0) { - parts.push(part); - } else if (part.type === "tool-call" && TOOLS_WITH_UI.has(part.toolName)) { - parts.push(part); - } - } - - return parts.length > 0 ? parts : [{ type: "text", text: "" }]; + const contentPartsState: ContentPartsState = { + contentParts: [], + currentTextPartIndex: -1, + toolCallIndices: new Map(), }; + const { contentParts, toolCallIndices } = contentPartsState; + let wasInterrupted = false; // Add placeholder assistant message setMessages((prev) => [ @@ -693,150 +634,172 @@ export default function NewChatPage() { throw new Error(`Backend error: ${response.status}`); } - if (!response.body) { - throw new Error("No response body"); - } + for await (const parsed of readSSEStream(response)) { + switch (parsed.type) { + case "text-delta": + appendText(contentPartsState, parsed.delta); + setMessages((prev) => + prev.map((m) => + m.id === assistantMsgId + ? { ...m, content: buildContentForUI(contentPartsState, TOOLS_WITH_UI) } + : m + ) + ); + break; - // Parse SSE stream - const reader = response.body.getReader(); - const decoder = new TextDecoder(); - let buffer = ""; + case "tool-input-start": + // Add tool call inline - this breaks the current text segment + addToolCall(contentPartsState, TOOLS_WITH_UI, parsed.toolCallId, parsed.toolName, {}); + setMessages((prev) => + prev.map((m) => + m.id === assistantMsgId + ? { ...m, content: buildContentForUI(contentPartsState, TOOLS_WITH_UI) } + : m + ) + ); + break; - try { - while (true) { - const { done, value } = await reader.read(); - if (done) break; + case "tool-input-available": { + // Update existing tool call's args, or add if not exists + if (toolCallIndices.has(parsed.toolCallId)) { + updateToolCall(contentPartsState, parsed.toolCallId, { args: parsed.input || {} }); + } else { + addToolCall( + contentPartsState, + TOOLS_WITH_UI, + parsed.toolCallId, + parsed.toolName, + parsed.input || {} + ); + } + setMessages((prev) => + prev.map((m) => + m.id === assistantMsgId + ? { ...m, content: buildContentForUI(contentPartsState, TOOLS_WITH_UI) } + : m + ) + ); + break; + } - buffer += decoder.decode(value, { stream: true }); - const events = buffer.split(/\r?\n\r?\n/); - buffer = events.pop() || ""; - - for (const event of events) { - const lines = event.split(/\r?\n/); - for (const line of lines) { - if (!line.startsWith("data: ")) continue; - const data = line.slice(6).trim(); - if (!data || data === "[DONE]") continue; - - try { - const parsed = JSON.parse(data); - - switch (parsed.type) { - case "text-delta": - appendText(parsed.delta); - setMessages((prev) => - prev.map((m) => - m.id === assistantMsgId ? { ...m, content: buildContentForUI() } : m - ) - ); - break; - - case "tool-input-start": - // Add tool call inline - this breaks the current text segment - addToolCall(parsed.toolCallId, parsed.toolName, {}); - setMessages((prev) => - prev.map((m) => - m.id === assistantMsgId ? { ...m, content: buildContentForUI() } : m - ) - ); - break; - - case "tool-input-available": { - // Update existing tool call's args, or add if not exists - if (toolCallIndices.has(parsed.toolCallId)) { - updateToolCall(parsed.toolCallId, { args: parsed.input || {} }); - } else { - addToolCall(parsed.toolCallId, parsed.toolName, parsed.input || {}); - } - setMessages((prev) => - prev.map((m) => - m.id === assistantMsgId ? { ...m, content: buildContentForUI() } : m - ) - ); - break; - } - - case "tool-output-available": { - // Update the tool call with its result - updateToolCall(parsed.toolCallId, { result: parsed.output }); - // Handle podcast-specific logic - if (parsed.output?.status === "pending" && parsed.output?.podcast_id) { - // Check if this is a podcast tool by looking at the content part - const idx = toolCallIndices.get(parsed.toolCallId); - if (idx !== undefined) { - const part = contentParts[idx]; - if (part?.type === "tool-call" && part.toolName === "generate_podcast") { - setActivePodcastTaskId(String(parsed.output.podcast_id)); - } - } - } - setMessages((prev) => - prev.map((m) => - m.id === assistantMsgId ? { ...m, content: buildContentForUI() } : m - ) - ); - break; - } - - case "data-thinking-step": { - // Handle thinking step events for chain-of-thought display - const stepData = parsed.data as ThinkingStepData; - if (stepData?.id) { - currentThinkingSteps.set(stepData.id, stepData); - // Update thinking steps state for rendering - // The ThinkingStepsScrollHandler in Thread component - // will handle auto-scrolling when this state changes - setMessageThinkingSteps((prev) => { - const newMap = new Map(prev); - newMap.set(assistantMsgId, Array.from(currentThinkingSteps.values())); - return newMap; - }); - } - break; - } - - case "data-thread-title-update": { - // Handle thread title update from LLM-generated title - const titleData = parsed.data as { threadId: number; title: string }; - if (titleData?.title && titleData?.threadId === currentThreadId) { - // Update current thread state with new title - setCurrentThread((prev) => - prev ? { ...prev, title: titleData.title } : prev - ); - // Invalidate thread list to refresh sidebar - queryClient.invalidateQueries({ - queryKey: ["threads", String(searchSpaceId)], - }); - // Invalidate thread detail for breadcrumb update - queryClient.invalidateQueries({ - queryKey: [ - "threads", - String(searchSpaceId), - "detail", - String(titleData.threadId), - ], - }); - } - break; - } - - case "error": - throw new Error(parsed.errorText || "Server error"); + case "tool-output-available": { + // Update the tool call with its result + updateToolCall(contentPartsState, parsed.toolCallId, { result: parsed.output }); + // Handle podcast-specific logic + if (parsed.output?.status === "pending" && parsed.output?.podcast_id) { + // Check if this is a podcast tool by looking at the content part + const idx = toolCallIndices.get(parsed.toolCallId); + if (idx !== undefined) { + const part = contentParts[idx]; + if (part?.type === "tool-call" && part.toolName === "generate_podcast") { + setActivePodcastTaskId(String(parsed.output.podcast_id)); } - } catch (e) { - if (e instanceof SyntaxError) continue; - throw e; } } + setMessages((prev) => + prev.map((m) => + m.id === assistantMsgId + ? { ...m, content: buildContentForUI(contentPartsState, TOOLS_WITH_UI) } + : m + ) + ); + break; } + + case "data-thinking-step": { + // Handle thinking step events for chain-of-thought display + const stepData = parsed.data as ThinkingStepData; + if (stepData?.id) { + currentThinkingSteps.set(stepData.id, stepData); + // Update thinking steps state for rendering + // The ThinkingStepsScrollHandler in Thread component + // will handle auto-scrolling when this state changes + setMessageThinkingSteps((prev) => { + const newMap = new Map(prev); + newMap.set(assistantMsgId, Array.from(currentThinkingSteps.values())); + return newMap; + }); + } + break; + } + + case "data-thread-title-update": { + // Handle thread title update from LLM-generated title + const titleData = parsed.data as { threadId: number; title: string }; + if (titleData?.title && titleData?.threadId === currentThreadId) { + // Update current thread state with new title + setCurrentThread((prev) => (prev ? { ...prev, title: titleData.title } : prev)); + // Invalidate thread list to refresh sidebar + queryClient.invalidateQueries({ + queryKey: ["threads", String(searchSpaceId)], + }); + // Invalidate thread detail for breadcrumb update + queryClient.invalidateQueries({ + queryKey: [ + "threads", + String(searchSpaceId), + "detail", + String(titleData.threadId), + ], + }); + } + break; + } + + case "data-interrupt-request": { + wasInterrupted = true; + const interruptData = parsed.data as Record; + const actionRequests = (interruptData.action_requests ?? []) as Array<{ + name: string; + args: Record; + }>; + for (const action of actionRequests) { + const existingIdx = Array.from(toolCallIndices.entries()).find(([, idx]) => { + const part = contentParts[idx]; + return part?.type === "tool-call" && part.toolName === action.name; + }); + if (existingIdx) { + updateToolCall(contentPartsState, existingIdx[0], { + result: { __interrupt__: true, ...interruptData }, + }); + } else { + const tcId = `interrupt-${action.name}`; + addToolCall(contentPartsState, TOOLS_WITH_UI, tcId, action.name, action.args); + updateToolCall(contentPartsState, tcId, { + result: { __interrupt__: true, ...interruptData }, + }); + } + } + setMessages((prev) => + prev.map((m) => + m.id === assistantMsgId + ? { ...m, content: buildContentForUI(contentPartsState, TOOLS_WITH_UI) } + : m + ) + ); + if (currentThreadId) { + setPendingInterrupt({ + threadId: currentThreadId, + assistantMsgId, + interruptData, + }); + } + break; + } + + case "error": + throw new Error(parsed.errorText || "Server error"); } - } finally { - reader.releaseLock(); } // Persist assistant message (with thinking steps for restoration on refresh) - const finalContent = buildContentForPersistence(); - if (contentParts.length > 0) { + // Skip persistence for interrupted messages -- handleResume will persist the final version + const finalContent = buildContentForPersistence( + contentPartsState, + TOOLS_WITH_UI, + currentThinkingSteps + ); + if (contentParts.length > 0 && !wasInterrupted) { try { const savedMessage = await appendMessage(currentThreadId, { role: "assistant", @@ -849,6 +812,13 @@ export default function NewChatPage() { prev.map((m) => (m.id === assistantMsgId ? { ...m, id: newMsgId } : m)) ); + // Update pending interrupt with the new persisted message ID + setPendingInterrupt((prev) => + prev && prev.assistantMsgId === assistantMsgId + ? { ...prev, assistantMsgId: newMsgId } + : prev + ); + // Also update thinking steps map with new ID setMessageThinkingSteps((prev) => { const steps = prev.get(assistantMsgId); @@ -876,7 +846,11 @@ export default function NewChatPage() { (part.type === "tool-call" && TOOLS_WITH_UI.has(part.toolName)) ); if (hasContent && currentThreadId) { - const partialContent = buildContentForPersistence(); + const partialContent = buildContentForPersistence( + contentPartsState, + TOOLS_WITH_UI, + currentThinkingSteps + ); try { const savedMessage = await appendMessage(currentThreadId, { role: "assistant", @@ -941,6 +915,330 @@ export default function NewChatPage() { ] ); + const handleResume = useCallback( + async ( + decisions: Array<{ + type: string; + message?: string; + edited_action?: { name: string; args: Record }; + }> + ) => { + if (!pendingInterrupt) return; + const { threadId: resumeThreadId, assistantMsgId } = pendingInterrupt; + setPendingInterrupt(null); + setIsRunning(true); + + const token = getBearerToken(); + if (!token) { + toast.error("Not authenticated. Please log in again."); + setIsRunning(false); + return; + } + + const controller = new AbortController(); + abortControllerRef.current = controller; + + const currentThinkingSteps = new Map( + (messageThinkingSteps.get(assistantMsgId) ?? []).map((s) => [s.id, s]) + ); + + const contentPartsState: ContentPartsState = { + contentParts: [], + currentTextPartIndex: -1, + toolCallIndices: new Map(), + }; + const { contentParts, toolCallIndices } = contentPartsState; + + const existingMsg = messages.find((m) => m.id === assistantMsgId); + if (existingMsg && Array.isArray(existingMsg.content)) { + for (const part of existingMsg.content) { + if (typeof part === "object" && part !== null) { + const p = part as Record; + if (p.type === "text") { + contentParts.push({ type: "text", text: String(p.text ?? "") }); + contentPartsState.currentTextPartIndex = contentParts.length - 1; + } else if (p.type === "tool-call") { + toolCallIndices.set(String(p.toolCallId), contentParts.length); + contentParts.push({ + type: "tool-call", + toolCallId: String(p.toolCallId), + toolName: String(p.toolName), + args: (p.args as Record) ?? {}, + result: p.result as unknown, + }); + contentPartsState.currentTextPartIndex = -1; + } + } + } + } + + // Merge edited args if present to fix race condition + if (decisions.length > 0 && decisions[0].type === "edit" && decisions[0].edited_action) { + const editedAction = decisions[0].edited_action; + for (const part of contentParts) { + if (part.type === "tool-call" && part.toolName === editedAction.name) { + part.args = { ...part.args, ...editedAction.args }; + break; + } + } + } + + const decisionType = decisions[0]?.type as "approve" | "reject" | undefined; + if (decisionType) { + for (const part of contentParts) { + if ( + part.type === "tool-call" && + typeof part.result === "object" && + part.result !== null && + "__interrupt__" in (part.result as Record) + ) { + part.result = { + ...(part.result as Record), + __decided__: decisionType, + }; + } + } + } + + try { + const backendUrl = process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL || "http://localhost:8000"; + const response = await fetch(`${backendUrl}/api/v1/threads/${resumeThreadId}/resume`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${token}`, + }, + body: JSON.stringify({ + search_space_id: searchSpaceId, + decisions, + }), + signal: controller.signal, + }); + + if (!response.ok) { + throw new Error(`Backend error: ${response.status}`); + } + + for await (const parsed of readSSEStream(response)) { + switch (parsed.type) { + case "text-delta": + appendText(contentPartsState, parsed.delta); + setMessages((prev) => + prev.map((m) => + m.id === assistantMsgId + ? { ...m, content: buildContentForUI(contentPartsState, TOOLS_WITH_UI) } + : m + ) + ); + break; + + case "tool-input-start": + addToolCall(contentPartsState, TOOLS_WITH_UI, parsed.toolCallId, parsed.toolName, {}); + setMessages((prev) => + prev.map((m) => + m.id === assistantMsgId + ? { ...m, content: buildContentForUI(contentPartsState, TOOLS_WITH_UI) } + : m + ) + ); + break; + + case "tool-input-available": + if (toolCallIndices.has(parsed.toolCallId)) { + updateToolCall(contentPartsState, parsed.toolCallId, { + args: parsed.input || {}, + }); + } else { + addToolCall( + contentPartsState, + TOOLS_WITH_UI, + parsed.toolCallId, + parsed.toolName, + parsed.input || {} + ); + } + setMessages((prev) => + prev.map((m) => + m.id === assistantMsgId + ? { ...m, content: buildContentForUI(contentPartsState, TOOLS_WITH_UI) } + : m + ) + ); + break; + + case "tool-output-available": + updateToolCall(contentPartsState, parsed.toolCallId, { + result: parsed.output, + }); + setMessages((prev) => + prev.map((m) => + m.id === assistantMsgId + ? { ...m, content: buildContentForUI(contentPartsState, TOOLS_WITH_UI) } + : m + ) + ); + break; + + case "data-thinking-step": { + const stepData = parsed.data as ThinkingStepData; + if (stepData?.id) { + currentThinkingSteps.set(stepData.id, stepData); + setMessageThinkingSteps((prev) => { + const newMap = new Map(prev); + newMap.set(assistantMsgId, Array.from(currentThinkingSteps.values())); + return newMap; + }); + } + break; + } + + case "data-interrupt-request": { + const interruptData = parsed.data as Record; + const actionRequests = (interruptData.action_requests ?? []) as Array<{ + name: string; + args: Record; + }>; + for (const action of actionRequests) { + const existingIdx = Array.from(toolCallIndices.entries()).find(([, idx]) => { + const part = contentParts[idx]; + return part?.type === "tool-call" && part.toolName === action.name; + }); + if (existingIdx) { + updateToolCall(contentPartsState, existingIdx[0], { + result: { + __interrupt__: true, + ...interruptData, + }, + }); + } else { + const tcId = `interrupt-${action.name}`; + addToolCall(contentPartsState, TOOLS_WITH_UI, tcId, action.name, action.args); + updateToolCall(contentPartsState, tcId, { + result: { + __interrupt__: true, + ...interruptData, + }, + }); + } + } + setMessages((prev) => + prev.map((m) => + m.id === assistantMsgId + ? { ...m, content: buildContentForUI(contentPartsState, TOOLS_WITH_UI) } + : m + ) + ); + setPendingInterrupt({ + threadId: resumeThreadId, + assistantMsgId, + interruptData, + }); + break; + } + + case "error": + throw new Error(parsed.errorText || "Server error"); + } + } + + const finalContent = buildContentForPersistence( + contentPartsState, + TOOLS_WITH_UI, + currentThinkingSteps + ); + if (contentParts.length > 0) { + try { + const savedMessage = await appendMessage(resumeThreadId, { + role: "assistant", + content: finalContent, + }); + const newMsgId = `msg-${savedMessage.id}`; + setMessages((prev) => + prev.map((m) => (m.id === assistantMsgId ? { ...m, id: newMsgId } : m)) + ); + setMessageThinkingSteps((prev) => { + const steps = prev.get(assistantMsgId); + if (steps) { + const newMap = new Map(prev); + newMap.delete(assistantMsgId); + newMap.set(newMsgId, steps); + return newMap; + } + return prev; + }); + } catch (err) { + console.error("Failed to persist resumed assistant message:", err); + } + } + } catch (error) { + if (error instanceof Error && error.name === "AbortError") { + return; + } + console.error("[NewChatPage] Resume error:", error); + toast.error("Failed to resume. Please try again."); + } finally { + setIsRunning(false); + abortControllerRef.current = null; + } + }, + [pendingInterrupt, messages, searchSpaceId, messageThinkingSteps] + ); + + useEffect(() => { + const handler = (e: Event) => { + const detail = (e as CustomEvent).detail as { + decisions: Array<{ + type: string; + message?: string; + edited_action?: { name: string; args: Record }; + }>; + }; + if (detail?.decisions && pendingInterrupt) { + const decision = detail.decisions[0]; + const decisionType = decision?.type as "approve" | "reject" | "edit"; + + setMessages((prev) => + prev.map((m) => { + if (m.id !== pendingInterrupt.assistantMsgId) return m; + const parts = m.content as unknown as Array>; + const newContent = parts.map((part) => { + if ( + part.type === "tool-call" && + typeof part.result === "object" && + part.result !== null && + "__interrupt__" in part.result + ) { + // For edit decisions, also update the displayed args + if (decisionType === "edit" && decision.edited_action) { + return { + ...part, + args: decision.edited_action.args, // Update displayed args + result: { + ...(part.result as Record), + __decided__: decisionType, + }, + }; + } + return { + ...part, + result: { + ...(part.result as Record), + __decided__: decisionType, + }, + }; + } + return part; + }); + return { ...m, content: newContent as unknown as ThreadMessageLike["content"] }; + }) + ); + handleResume(detail.decisions); + } + }; + window.addEventListener("hitl-decision", handler); + return () => window.removeEventListener("hitl-decision", handler); + }, [handleResume, pendingInterrupt]); + // Convert message (pass through since already in correct format) const convertMessage = useCallback( (message: ThreadMessageLike): ThreadMessageLike => message, @@ -1026,77 +1324,12 @@ export default function NewChatPage() { const assistantMsgId = `msg-assistant-${Date.now()}`; const currentThinkingSteps = new Map(); - // Content parts tracking (same as onNew) - type ContentPart = - | { type: "text"; text: string } - | { - type: "tool-call"; - toolCallId: string; - toolName: string; - args: Record; - result?: unknown; - }; - const contentParts: ContentPart[] = []; - let currentTextPartIndex = -1; - const toolCallIndices = new Map(); - - const appendText = (delta: string) => { - if (currentTextPartIndex >= 0 && contentParts[currentTextPartIndex]?.type === "text") { - (contentParts[currentTextPartIndex] as { type: "text"; text: string }).text += delta; - } else { - contentParts.push({ type: "text", text: delta }); - currentTextPartIndex = contentParts.length - 1; - } - }; - - const addToolCall = (toolCallId: string, toolName: string, args: Record) => { - if (TOOLS_WITH_UI.has(toolName)) { - contentParts.push({ type: "tool-call", toolCallId, toolName, args }); - toolCallIndices.set(toolCallId, contentParts.length - 1); - currentTextPartIndex = -1; - } - }; - - const updateToolCall = ( - toolCallId: string, - update: { args?: Record; result?: unknown } - ) => { - const index = toolCallIndices.get(toolCallId); - if (index !== undefined && contentParts[index]?.type === "tool-call") { - const tc = contentParts[index] as ContentPart & { type: "tool-call" }; - if (update.args) tc.args = update.args; - if (update.result !== undefined) tc.result = update.result; - } - }; - - const buildContentForUI = (): ThreadMessageLike["content"] => { - const filtered = contentParts.filter((part) => { - if (part.type === "text") return part.text.length > 0; - if (part.type === "tool-call") return TOOLS_WITH_UI.has(part.toolName); - return false; - }); - return filtered.length > 0 - ? (filtered as ThreadMessageLike["content"]) - : [{ type: "text", text: "" }]; - }; - - const buildContentForPersistence = (): unknown[] => { - const parts: unknown[] = []; - if (currentThinkingSteps.size > 0) { - parts.push({ - type: "thinking-steps", - steps: Array.from(currentThinkingSteps.values()), - }); - } - for (const part of contentParts) { - if (part.type === "text" && part.text.length > 0) { - parts.push(part); - } else if (part.type === "tool-call" && TOOLS_WITH_UI.has(part.toolName)) { - parts.push(part); - } - } - return parts.length > 0 ? parts : [{ type: "text", text: "" }]; + const contentPartsState: ContentPartsState = { + contentParts: [], + currentTextPartIndex: -1, + toolCallIndices: new Map(), }; + const { contentParts, toolCallIndices } = contentPartsState; // Add placeholder messages to UI // Always add back the user message (with new query for edit, or original content for reload) @@ -1140,113 +1373,95 @@ export default function NewChatPage() { throw new Error(`Backend error: ${response.status}`); } - if (!response.body) { - throw new Error("No response body"); - } + for await (const parsed of readSSEStream(response)) { + switch (parsed.type) { + case "text-delta": + appendText(contentPartsState, parsed.delta); + setMessages((prev) => + prev.map((m) => + m.id === assistantMsgId + ? { ...m, content: buildContentForUI(contentPartsState, TOOLS_WITH_UI) } + : m + ) + ); + break; - // Parse SSE stream (same logic as onNew) - const reader = response.body.getReader(); - const decoder = new TextDecoder(); - let buffer = ""; + case "tool-input-start": + addToolCall(contentPartsState, TOOLS_WITH_UI, parsed.toolCallId, parsed.toolName, {}); + setMessages((prev) => + prev.map((m) => + m.id === assistantMsgId + ? { ...m, content: buildContentForUI(contentPartsState, TOOLS_WITH_UI) } + : m + ) + ); + break; - try { - while (true) { - const { done, value } = await reader.read(); - if (done) break; + case "tool-input-available": + if (toolCallIndices.has(parsed.toolCallId)) { + updateToolCall(contentPartsState, parsed.toolCallId, { args: parsed.input || {} }); + } else { + addToolCall( + contentPartsState, + TOOLS_WITH_UI, + parsed.toolCallId, + parsed.toolName, + parsed.input || {} + ); + } + setMessages((prev) => + prev.map((m) => + m.id === assistantMsgId + ? { ...m, content: buildContentForUI(contentPartsState, TOOLS_WITH_UI) } + : m + ) + ); + break; - buffer += decoder.decode(value, { stream: true }); - const events = buffer.split(/\r?\n\r?\n/); - buffer = events.pop() || ""; - - for (const event of events) { - const lines = event.split(/\r?\n/); - for (const line of lines) { - if (!line.startsWith("data: ")) continue; - const data = line.slice(6).trim(); - if (!data || data === "[DONE]") continue; - - try { - const parsed = JSON.parse(data); - - switch (parsed.type) { - case "text-delta": - appendText(parsed.delta); - setMessages((prev) => - prev.map((m) => - m.id === assistantMsgId ? { ...m, content: buildContentForUI() } : m - ) - ); - break; - - case "tool-input-start": - addToolCall(parsed.toolCallId, parsed.toolName, {}); - setMessages((prev) => - prev.map((m) => - m.id === assistantMsgId ? { ...m, content: buildContentForUI() } : m - ) - ); - break; - - case "tool-input-available": - if (toolCallIndices.has(parsed.toolCallId)) { - updateToolCall(parsed.toolCallId, { args: parsed.input || {} }); - } else { - addToolCall(parsed.toolCallId, parsed.toolName, parsed.input || {}); - } - setMessages((prev) => - prev.map((m) => - m.id === assistantMsgId ? { ...m, content: buildContentForUI() } : m - ) - ); - break; - - case "tool-output-available": - updateToolCall(parsed.toolCallId, { result: parsed.output }); - if (parsed.output?.status === "pending" && parsed.output?.podcast_id) { - const idx = toolCallIndices.get(parsed.toolCallId); - if (idx !== undefined) { - const part = contentParts[idx]; - if (part?.type === "tool-call" && part.toolName === "generate_podcast") { - setActivePodcastTaskId(String(parsed.output.podcast_id)); - } - } - } - setMessages((prev) => - prev.map((m) => - m.id === assistantMsgId ? { ...m, content: buildContentForUI() } : m - ) - ); - break; - - case "data-thinking-step": { - const stepData = parsed.data as ThinkingStepData; - if (stepData?.id) { - currentThinkingSteps.set(stepData.id, stepData); - setMessageThinkingSteps((prev) => { - const newMap = new Map(prev); - newMap.set(assistantMsgId, Array.from(currentThinkingSteps.values())); - return newMap; - }); - } - break; - } - - case "error": - throw new Error(parsed.errorText || "Server error"); + case "tool-output-available": + updateToolCall(contentPartsState, parsed.toolCallId, { result: parsed.output }); + if (parsed.output?.status === "pending" && parsed.output?.podcast_id) { + const idx = toolCallIndices.get(parsed.toolCallId); + if (idx !== undefined) { + const part = contentParts[idx]; + if (part?.type === "tool-call" && part.toolName === "generate_podcast") { + setActivePodcastTaskId(String(parsed.output.podcast_id)); } - } catch (e) { - if (e instanceof SyntaxError) continue; - throw e; } } + setMessages((prev) => + prev.map((m) => + m.id === assistantMsgId + ? { ...m, content: buildContentForUI(contentPartsState, TOOLS_WITH_UI) } + : m + ) + ); + break; + + case "data-thinking-step": { + const stepData = parsed.data as ThinkingStepData; + if (stepData?.id) { + currentThinkingSteps.set(stepData.id, stepData); + setMessageThinkingSteps((prev) => { + const newMap = new Map(prev); + newMap.set(assistantMsgId, Array.from(currentThinkingSteps.values())); + return newMap; + }); + } + break; } + + case "error": + throw new Error(parsed.errorText || "Server error"); } - } finally { - reader.releaseLock(); } // Persist messages after streaming completes - const finalContent = buildContentForPersistence(); + const finalContent = buildContentForPersistence( + contentPartsState, + TOOLS_WITH_UI, + currentThinkingSteps + ); if (contentParts.length > 0) { try { // Persist user message (for both edit and reload modes, since backend deleted it) @@ -1427,17 +1642,24 @@ export default function NewChatPage() { return ( + + + + {/* Disabled for now */} -
- } - /> +
+
+ } + /> +
+
); diff --git a/surfsense_web/app/globals.css b/surfsense_web/app/globals.css index 3b87e4356..4095fc660 100644 --- a/surfsense_web/app/globals.css +++ b/surfsense_web/app/globals.css @@ -187,6 +187,23 @@ button { background-color: hsl(var(--muted-foreground) / 0.4); } +/* Human-in-the-loop approval card animations */ +@keyframes pulse-subtle { + 0%, + 100% { + opacity: 1; + box-shadow: 0 0 0 0 rgb(0 0 0 / 0.15); + } + 50% { + opacity: 1; + box-shadow: 0 0 20px 4px rgb(0 0 0 / 0.12); + } +} + +.animate-pulse-subtle { + animation: pulse-subtle 2s cubic-bezier(0.4, 0, 0.6, 1) infinite; +} + /* Integrations section — vertical column auto-scroll */ @keyframes integrations-scroll-up { 0% { @@ -208,3 +225,5 @@ button { @source '../node_modules/@llamaindex/chat-ui/**/*.{ts,tsx}'; @source '../node_modules/streamdown/dist/*.js'; +@source '../node_modules/@streamdown/code/dist/*.js'; +@source '../node_modules/@streamdown/math/dist/*.js'; diff --git a/surfsense_web/app/layout.tsx b/surfsense_web/app/layout.tsx index 047baa27b..5e8fa394f 100644 --- a/surfsense_web/app/layout.tsx +++ b/surfsense_web/app/layout.tsx @@ -2,6 +2,7 @@ import type { Metadata, Viewport } from "next"; import "./globals.css"; import { RootProvider } from "fumadocs-ui/provider/next"; import { Roboto } from "next/font/google"; +import { AnnouncementToastProvider } from "@/components/announcements/AnnouncementToastProvider"; import { ElectricProvider } from "@/components/providers/ElectricProvider"; import { GlobalLoadingProvider } from "@/components/providers/GlobalLoadingProvider"; import { I18nProvider } from "@/components/providers/I18nProvider"; @@ -124,6 +125,7 @@ export default function RootLayout({ + diff --git a/surfsense_web/atoms/chat/current-thread.atom.ts b/surfsense_web/atoms/chat/current-thread.atom.ts index 5de11eb92..aee5bf7b4 100644 --- a/surfsense_web/atoms/chat/current-thread.atom.ts +++ b/surfsense_web/atoms/chat/current-thread.atom.ts @@ -1,5 +1,6 @@ import { atom } from "jotai"; import type { ChatVisibility } from "@/lib/chat/thread-persistence"; +import { reportPanelAtom, reportPanelOpenAtom } from "./report-panel.atom"; // TODO: Update `hasComments` to true when the first comment is created on a thread. // Currently it only updates on thread load. The gutter still works because @@ -39,6 +40,8 @@ export const showCommentsGutterAtom = atom((get) => { const thread = get(currentThreadAtom); // Hide gutter if comments are collapsed if (thread.commentsCollapsed) return false; + // Hide gutter if report panel is open (report panel takes the right side) + if (get(reportPanelOpenAtom)) return false; return ( thread.visibility === "SEARCH_SPACE" && (thread.hasComments || thread.addingCommentToMessageId !== null) @@ -59,6 +62,8 @@ export const setThreadVisibilityAtom = atom(null, (get, set, newVisibility: Chat export const resetCurrentThreadAtom = atom(null, (_, set) => { set(currentThreadAtom, initialState); + // Also close the report panel when resetting the thread + set(reportPanelAtom, { isOpen: false, reportId: null, title: null, wordCount: null }); }); /** Atom to read whether comments panel is collapsed */ diff --git a/surfsense_web/atoms/chat/report-panel.atom.ts b/surfsense_web/atoms/chat/report-panel.atom.ts new file mode 100644 index 000000000..0abb0c0ff --- /dev/null +++ b/surfsense_web/atoms/chat/report-panel.atom.ts @@ -0,0 +1,52 @@ +import { atom } from "jotai"; + +interface ReportPanelState { + isOpen: boolean; + reportId: number | null; + title: string | null; + wordCount: number | null; + /** When set, uses public endpoints for fetching report data (public shared chat) */ + shareToken: string | null; +} + +const initialState: ReportPanelState = { + isOpen: false, + reportId: null, + title: null, + wordCount: null, + shareToken: null, +}; + +/** Core atom holding the report panel state */ +export const reportPanelAtom = atom(initialState); + +/** Derived read-only atom for checking if panel is open */ +export const reportPanelOpenAtom = atom((get) => get(reportPanelAtom).isOpen); + +/** Action atom to open the report panel with a specific report */ +export const openReportPanelAtom = atom( + null, + ( + _get, + set, + { + reportId, + title, + wordCount, + shareToken, + }: { reportId: number; title: string; wordCount?: number; shareToken?: string | null } + ) => { + set(reportPanelAtom, { + isOpen: true, + reportId, + title, + wordCount: wordCount ?? null, + shareToken: shareToken ?? null, + }); + } +); + +/** Action atom to close the report panel */ +export const closeReportPanelAtom = atom(null, (_, set) => { + set(reportPanelAtom, initialState); +}); diff --git a/surfsense_web/components/announcements/AnnouncementToastProvider.tsx b/surfsense_web/components/announcements/AnnouncementToastProvider.tsx new file mode 100644 index 000000000..cf955fe0f --- /dev/null +++ b/surfsense_web/components/announcements/AnnouncementToastProvider.tsx @@ -0,0 +1,90 @@ +"use client"; + +import { Megaphone } from "lucide-react"; +import { useEffect, useRef } from "react"; +import { toast } from "sonner"; +import type { Announcement } from "@/contracts/types/announcement.types"; +import { announcements } from "@/lib/announcements/announcements-data"; +import { + isAnnouncementToasted, + markAnnouncementRead, + markAnnouncementToasted, +} from "@/lib/announcements/announcements-storage"; + +/** Map announcement category to the Sonner toast method */ +const categoryToVariant: Record = { + update: "info", + feature: "success", + maintenance: "warning", + info: "info", +}; + +/** Show a single announcement as a toast */ +function showAnnouncementToast(announcement: Announcement) { + const variant = categoryToVariant[announcement.category] ?? "info"; + + const options = { + description: truncateText(announcement.description, 120), + duration: 12000, + icon: , + action: announcement.link + ? { + label: announcement.link.label, + onClick: () => { + if (announcement.link?.url.startsWith("http")) { + window.open(announcement.link.url, "_blank"); + } else if (announcement.link?.url) { + window.location.href = announcement.link.url; + } + }, + } + : undefined, + onDismiss: () => { + markAnnouncementRead(announcement.id); + }, + }; + + toast[variant](announcement.title, options); + markAnnouncementToasted(announcement.id); +} + +/** + * Global provider that shows important announcements as toast notifications. + * + * Place this component once at the root layout level (alongside ). + * On mount, it checks for unread important announcements that haven't been + * shown as toasts yet, and displays them with a short stagger delay. + */ +export function AnnouncementToastProvider() { + const hasChecked = useRef(false); + + useEffect(() => { + // Only run once per page load + if (hasChecked.current) return; + hasChecked.current = true; + + // Small delay to let the page settle before showing toasts + const timer = setTimeout(() => { + const importantUntoasted = announcements.filter( + (a) => a.isImportant && !isAnnouncementToasted(a.id) + ); + + // Show each important announcement as a toast with stagger + for (let i = 0; i < importantUntoasted.length; i++) { + const announcement = importantUntoasted[i]; + setTimeout(() => showAnnouncementToast(announcement), i * 800); + } + }, 1500); // Initial delay for page to settle + + return () => clearTimeout(timer); + }, []); + + // This component renders nothing — it only triggers side effects + return null; +} + +/** Truncate text to a maximum length, adding ellipsis if needed */ +function truncateText(text: string, maxLength: number): string { + if (text.length <= maxLength) return text; + return `${text.slice(0, maxLength).trimEnd()}...`; +} diff --git a/surfsense_web/components/assistant-ui/inline-mention-editor.tsx b/surfsense_web/components/assistant-ui/inline-mention-editor.tsx index b8194c91f..b4e37d324 100644 --- a/surfsense_web/components/assistant-ui/inline-mention-editor.tsx +++ b/surfsense_web/components/assistant-ui/inline-mention-editor.tsx @@ -126,38 +126,39 @@ export const InlineMentionEditor = forwardRef { if (!editorRef.current) return ""; - let text = ""; - const walker = document.createTreeWalker( - editorRef.current, - NodeFilter.SHOW_TEXT | NodeFilter.SHOW_ELEMENT, - { - acceptNode: (node) => { - // Skip chip elements entirely - if (node.nodeType === Node.ELEMENT_NODE) { - const el = node as Element; - if (el.hasAttribute(CHIP_DATA_ATTR)) { - return NodeFilter.FILTER_REJECT; // Skip this subtree - } - return NodeFilter.FILTER_SKIP; // Continue into children - } - return NodeFilter.FILTER_ACCEPT; - }, - } - ); - - let node: Node | null = walker.nextNode(); - while (node) { + const extractText = (node: Node): string => { if (node.nodeType === Node.TEXT_NODE) { - text += node.textContent; + return node.textContent ?? ""; } - node = walker.nextNode(); - } - return text.trim(); + if (node.nodeType === Node.ELEMENT_NODE) { + const element = node as Element; + + // Preserve mention chips as inline @title tokens. + if (element.hasAttribute(CHIP_DATA_ATTR)) { + const title = element.querySelector("[data-mention-title='true']")?.textContent?.trim(); + if (title) { + return `@${title}`; + } + return ""; + } + + let result = ""; + for (const child of Array.from(element.childNodes)) { + result += extractText(child); + } + return result; + } + + return ""; + }; + + return extractText(editorRef.current).trim(); }, []); // Get all mentioned documents diff --git a/surfsense_web/components/assistant-ui/markdown-text.tsx b/surfsense_web/components/assistant-ui/markdown-text.tsx index cc8cec5d9..779d6fa5c 100644 --- a/surfsense_web/components/assistant-ui/markdown-text.tsx +++ b/surfsense_web/components/assistant-ui/markdown-text.tsx @@ -10,11 +10,46 @@ import { } from "@assistant-ui/react-markdown"; import { CheckIcon, CopyIcon } from "lucide-react"; import { type FC, memo, type ReactNode, useState } from "react"; +import rehypeKatex from "rehype-katex"; import remarkGfm from "remark-gfm"; +import remarkMath from "remark-math"; +import "katex/dist/katex.min.css"; import { InlineCitation } from "@/components/assistant-ui/inline-citation"; import { TooltipIconButton } from "@/components/assistant-ui/tooltip-icon-button"; import { cn } from "@/lib/utils"; +/** + * Convert all LaTeX delimiter styles to the dollar-sign syntax + * that remark-math understands. LLMs use various delimiters + * (\(...\), \[...\], \begin{equation}, etc.) and we need to + * normalise them all to $ / $$ before the markdown parser runs. + */ +function convertLatexDelimiters(content: string): string { + // 1. Block math: \[...\] → $$...$$ + content = content.replace(/\\\[([\s\S]*?)\\\]/g, (_, inner) => `$$${inner}$$`); + // 2. Inline math: \(...\) → $...$ + content = content.replace(/\\\(([\s\S]*?)\\\)/g, (_, inner) => `$${inner}$`); + // 3. Block: \begin{equation}...\end{equation} → $$...$$ + content = content.replace( + /\\begin\{equation\}([\s\S]*?)\\end\{equation\}/g, + (_, inner) => `$$${inner}$$` + ); + // 4. Block: \begin{displaymath}...\end{displaymath} → $$...$$ + content = content.replace( + /\\begin\{displaymath\}([\s\S]*?)\\end\{displaymath\}/g, + (_, inner) => `$$${inner}$$` + ); + // 5. Inline: \begin{math}...\end{math} → $...$ + content = content.replace(/\\begin\{math\}([\s\S]*?)\\end\{math\}/g, (_, inner) => `$${inner}$`); + // 6. Strip backtick wrapping around math: `$$...$$` → $$...$$ and `$...$` → $...$ + content = content.replace(/`(\${1,2})((?:(?!\1).)+)\1`/g, "$1$2$1"); + + // Ensure markdown headings (## ...) always start on their own line. + content = content.replace(/([^\n])(#{1,6}\s)/g, "$1\n\n$2"); + + return content; +} + // Citation pattern: [citation:CHUNK_ID] or [citation:doc-CHUNK_ID] // Also matches Chinese brackets 【】 and handles zero-width spaces that LLM sometimes inserts const CITATION_REGEX = /[[【]\u200B?citation:(doc-)?(\d+)\u200B?[\]】]/g; @@ -59,7 +94,8 @@ function parseTextWithCitations(text: string): ReactNode[] { // Reset regex state CITATION_REGEX.lastIndex = 0; - while ((match = CITATION_REGEX.exec(text)) !== null) { + match = CITATION_REGEX.exec(text); + while (match !== null) { // Add text before the citation if (match.index > lastIndex) { parts.push(text.substring(lastIndex, match.index)); @@ -80,6 +116,7 @@ function parseTextWithCitations(text: string): ReactNode[] { lastIndex = match.index + match[0].length; instanceIndex++; + match = CITATION_REGEX.exec(text); } // Add any remaining text after the last citation @@ -93,9 +130,11 @@ function parseTextWithCitations(text: string): ReactNode[] { const MarkdownTextImpl = () => { return ( ); }; diff --git a/surfsense_web/components/homepage/footer-new.tsx b/surfsense_web/components/homepage/footer-new.tsx index e52ab09b9..4bbff0cbd 100644 --- a/surfsense_web/components/homepage/footer-new.tsx +++ b/surfsense_web/components/homepage/footer-new.tsx @@ -33,6 +33,10 @@ export function FooterNew() { title: "Contact Us", href: "/contact", }, + { + title: "Announcements", + href: "/announcements", + }, ]; const socials = [ diff --git a/surfsense_web/components/homepage/hero-section.tsx b/surfsense_web/components/homepage/hero-section.tsx index b1e8d02bf..426e6c043 100644 --- a/surfsense_web/components/homepage/hero-section.tsx +++ b/surfsense_web/components/homepage/hero-section.tsx @@ -1,10 +1,11 @@ "use client"; import { useFeatureFlagVariantKey } from "@posthog/react"; import { AnimatePresence, motion } from "motion/react"; -import Image from "next/image"; import Link from "next/link"; -import React, { useEffect, useRef, useState } from "react"; +import type React from "react"; +import { useEffect, useRef, useState } from "react"; import Balancer from "react-wrap-balancer"; +import { WalkthroughScroll } from "@/components/ui/walkthrough-scroll"; import { AUTH_TYPE, BACKEND_URL } from "@/lib/env-config"; import { trackLoginAttempt } from "@/lib/posthog/events"; import { cn } from "@/lib/utils"; @@ -40,41 +41,37 @@ export function HeroSection() { return (
Then chat with it in real-time, even alongside your team.

-
+
-
-
- {/* Light mode image */} - header - {/* Dark mode image */} - header -
+
+
); @@ -236,24 +211,23 @@ const BackgroundGrids = () => { ); }; -const CollisionMechanism = React.forwardRef< - HTMLDivElement, - { - containerRef: React.RefObject; - parentRef: React.RefObject; - beamOptions?: { - initialX?: number; - translateX?: number; - initialY?: number; - translateY?: number; - rotate?: number; - className?: string; - duration?: number; - delay?: number; - repeatDelay?: number; - }; - } ->(({ parentRef, containerRef, beamOptions = {} }, ref) => { +const CollisionMechanism = ({ + parentRef, + beamOptions = {}, +}: { + parentRef: React.RefObject; + beamOptions?: { + initialX?: number; + translateX?: number; + initialY?: number; + translateY?: number; + rotate?: number; + className?: string; + duration?: number; + delay?: number; + repeatDelay?: number; + }; +}) => { const beamRef = useRef(null); const [collision, setCollision] = useState<{ detected: boolean; @@ -264,14 +238,14 @@ const CollisionMechanism = React.forwardRef< useEffect(() => { const checkCollision = () => { - if (beamRef.current && containerRef.current && parentRef.current && !cycleCollisionDetected) { + if (beamRef.current && parentRef.current && !cycleCollisionDetected) { const beamRect = beamRef.current.getBoundingClientRect(); - const containerRect = containerRef.current.getBoundingClientRect(); const parentRect = parentRef.current.getBoundingClientRect(); + const rightEdge = parentRect.right; - if (beamRect.bottom >= containerRect.top) { - const relativeX = beamRect.left - parentRect.left + beamRect.width / 2; - const relativeY = beamRect.bottom - parentRect.top; + if (beamRect.right >= rightEdge - 20) { + const relativeX = parentRect.width - 20; + const relativeY = beamRect.top - parentRect.top + beamRect.height / 2; setCollision({ detected: true, @@ -288,7 +262,7 @@ const CollisionMechanism = React.forwardRef< const animationInterval = setInterval(checkCollision, 100); return () => clearInterval(animationInterval); - }, [cycleCollisionDetected, containerRef]); + }, [cycleCollisionDetected, parentRef]); useEffect(() => { if (collision.detected && collision.coordinates) { @@ -354,9 +328,7 @@ const CollisionMechanism = React.forwardRef< ); -}); - -CollisionMechanism.displayName = "CollisionMechanism"; +}; const Explosion = ({ ...props }: React.HTMLProps) => { const spans = Array.from({ length: 20 }, (_, index) => ({ diff --git a/surfsense_web/components/homepage/navbar.tsx b/surfsense_web/components/homepage/navbar.tsx index 670e3c810..fd118b50a 100644 --- a/surfsense_web/components/homepage/navbar.tsx +++ b/surfsense_web/components/homepage/navbar.tsx @@ -4,6 +4,7 @@ import { IconBrandGithub, IconBrandReddit, IconMenu2, + IconSpeakerphone, IconX, } from "@tabler/icons-react"; import { AnimatePresence, motion } from "motion/react"; @@ -12,6 +13,7 @@ import { useEffect, useState } from "react"; import { SignInButton } from "@/components/auth/sign-in-button"; import { Logo } from "@/components/Logo"; import { ThemeTogglerComponent } from "@/components/theme/theme-toggle"; +import { useAnnouncements } from "@/hooks/use-announcements"; import { useGithubStars } from "@/hooks/use-github-stars"; import { cn } from "@/lib/utils"; @@ -47,7 +49,11 @@ export const Navbar = () => { const DesktopNav = ({ navItems, isScrolled }: any) => { const [hovered, setHovered] = useState(null); + const [mounted, setMounted] = useState(false); const { compactFormat: githubStars, loading: loadingGithubStars } = useGithubStars(); + const { unreadCount } = useAnnouncements(); + + useEffect(() => setMounted(true), []); return ( { @@ -118,6 +124,17 @@ const DesktopNav = ({ navItems, isScrolled }: any) => { )} + + + {mounted && unreadCount > 0 && ( + + {unreadCount > 99 ? "99+" : unreadCount} + + )} +
@@ -127,7 +144,11 @@ const DesktopNav = ({ navItems, isScrolled }: any) => { const MobileNav = ({ navItems, isScrolled }: any) => { const [open, setOpen] = useState(false); + const [mounted, setMounted] = useState(false); const { compactFormat: githubStars, loading: loadingGithubStars } = useGithubStars(); + const { unreadCount } = useAnnouncements(); + + useEffect(() => setMounted(true), []); return ( { )} + + + {mounted && unreadCount > 0 && ( + + {unreadCount > 99 ? "99+" : unreadCount} + + )} +
diff --git a/surfsense_web/components/homepage/use-cases-grid.tsx b/surfsense_web/components/homepage/use-cases-grid.tsx new file mode 100644 index 000000000..b835703aa --- /dev/null +++ b/surfsense_web/components/homepage/use-cases-grid.tsx @@ -0,0 +1,124 @@ +"use client"; + +import { AnimatePresence, motion } from "motion/react"; +import { ExpandedGifOverlay, useExpandedGif } from "@/components/ui/expanded-gif-overlay"; + +const useCases = [ + { + title: "Search & Citation", + description: "Ask questions and get Perplexity-style cited responses from your knowledge base.", + src: "/homepage/hero_tutorial/BSNCGif.gif", + }, + { + title: "Document Mention QNA", + description: "Mention specific documents in your queries for targeted answers.", + src: "/homepage/hero_tutorial/BQnaGif_compressed.gif", + }, + { + title: "Report Generation", + description: "Generate and export reports in many formats.", + src: "/homepage/hero_tutorial/ReportGenGif_compressed.gif", + }, + { + title: "Podcast Generation", + description: "Turn your knowledge into podcasts in under 20 seconds.", + src: "/homepage/hero_tutorial/PodcastGenGif.gif", + }, + { + title: "Image Generation", + description: "Generate images directly from your conversations.", + src: "/homepage/hero_tutorial/ImageGenGif.gif", + }, + { + title: "Realtime Chat", + description: "Chat together in realtime with your team.", + src: "/homepage/hero_realtime/RealTimeChatGif.gif", + }, + { + title: "Realtime Comments", + description: "Add comments and tag teammates on any message.", + src: "/homepage/hero_realtime/RealTimeCommentsFlow.gif", + }, +]; + +function UseCaseCard({ + title, + description, + src, + className, +}: { + title: string; + description: string; + src: string; + className?: string; +}) { + const { expanded, open, close } = useExpandedGif(); + + return ( + <> + +
+ {title} +
+
+

{title}

+

{description}

+
+
+ + + {expanded && } + + + ); +} + +export function UseCasesGrid() { + return ( +
+
+

+ What You Can Do +

+
+ + {/* Row 1: 2 larger cards */} +
+ {useCases.slice(0, 2).map((useCase) => ( + + ))} +
+ + {/* Row 2: 3 equal cards */} +
+ {useCases.slice(2, 5).map((useCase) => ( + + ))} +
+ + {/* Row 3: 2 cards */} +
+ {useCases.slice(5).map((useCase) => ( + + ))} +
+ +

+ And more coming soon. +

+
+ ); +} diff --git a/surfsense_web/components/layout/providers/LayoutDataProvider.tsx b/surfsense_web/components/layout/providers/LayoutDataProvider.tsx index ed96b84ca..aa6d2984d 100644 --- a/surfsense_web/components/layout/providers/LayoutDataProvider.tsx +++ b/surfsense_web/components/layout/providers/LayoutDataProvider.tsx @@ -2,7 +2,15 @@ import { useQuery, useQueryClient } from "@tanstack/react-query"; import { useAtomValue, useSetAtom } from "jotai"; -import { AlertTriangle, Inbox, LogOut, PencilIcon, SquareLibrary, Trash2 } from "lucide-react"; +import { + AlertTriangle, + Inbox, + LogOut, + Megaphone, + PencilIcon, + SquareLibrary, + Trash2, +} from "lucide-react"; import { useParams, usePathname, useRouter } from "next/navigation"; import { useTranslations } from "next-intl"; import { useTheme } from "next-themes"; @@ -23,6 +31,7 @@ import { } from "@/components/ui/dialog"; import { Input } from "@/components/ui/input"; import { isPageLimitExceededMetadata } from "@/contracts/types/inbox.types"; +import { useAnnouncements } from "@/hooks/use-announcements"; import { useInbox } from "@/hooks/use-inbox"; import { searchSpacesApiService } from "@/lib/apis/search-spaces-api.service"; import { logout } from "@/lib/auth-utils"; @@ -65,6 +74,9 @@ export function LayoutDataProvider({ const queryClient = useQueryClient(); const { theme, setTheme } = useTheme(); + // Announcements + const { unreadCount: announcementUnreadCount } = useAnnouncements(); + // Atoms const { data: user } = useAtomValue(currentUserAtom); const { data: searchSpacesData, refetch: refetchSearchSpaces } = useAtomValue(searchSpacesAtom); @@ -293,8 +305,15 @@ export function LayoutDataProvider({ icon: SquareLibrary, isActive: pathname?.includes("/documents"), }, + { + title: "Announcements", + url: "/announcements", + icon: Megaphone, + isActive: pathname?.includes("/announcements"), + badge: announcementUnreadCount > 0 ? formatInboxCount(announcementUnreadCount) : undefined, + }, ], - [searchSpaceId, pathname, isInboxSidebarOpen, totalUnreadCount] + [searchSpaceId, pathname, isInboxSidebarOpen, totalUnreadCount, announcementUnreadCount] ); // Handlers diff --git a/surfsense_web/components/layout/ui/icon-rail/SearchSpaceAvatar.tsx b/surfsense_web/components/layout/ui/icon-rail/SearchSpaceAvatar.tsx index 134c109dd..dd10709ce 100644 --- a/surfsense_web/components/layout/ui/icon-rail/SearchSpaceAvatar.tsx +++ b/surfsense_web/components/layout/ui/icon-rail/SearchSpaceAvatar.tsx @@ -126,7 +126,7 @@ export function SearchSpaceAvatar({ type="button" onClick={onClick} className={cn( - "relative flex items-center justify-center rounded-lg font-semibold text-white transition-all", + "relative flex items-center justify-center rounded-lg font-semibold text-white transition-all select-none", "hover:opacity-90 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring", sizeClasses, isActive && "ring-2 ring-primary ring-offset-1 ring-offset-background" diff --git a/surfsense_web/components/layout/ui/sidebar/SidebarUserProfile.tsx b/surfsense_web/components/layout/ui/sidebar/SidebarUserProfile.tsx index a30b3e626..3ba7bb2a0 100644 --- a/surfsense_web/components/layout/ui/sidebar/SidebarUserProfile.tsx +++ b/surfsense_web/components/layout/ui/sidebar/SidebarUserProfile.tsx @@ -111,7 +111,7 @@ function UserAvatar({ return (
{initials} diff --git a/surfsense_web/components/markdown-viewer.tsx b/surfsense_web/components/markdown-viewer.tsx index 91f97830a..297c850e0 100644 --- a/surfsense_web/components/markdown-viewer.tsx +++ b/surfsense_web/components/markdown-viewer.tsx @@ -1,23 +1,87 @@ +import { createCodePlugin } from "@streamdown/code"; +import { createMathPlugin } from "@streamdown/math"; import Image from "next/image"; import { Streamdown, type StreamdownProps } from "streamdown"; +import "katex/dist/katex.min.css"; import { cn } from "@/lib/utils"; +const code = createCodePlugin({ + themes: ["nord", "nord"], +}); + +const math = createMathPlugin({ + singleDollarTextMath: true, +}); + interface MarkdownViewerProps { content: string; className?: string; } +/** + * If the entire content is wrapped in a single ```markdown or ```md + * code fence, strip the fence so the inner markdown renders properly. + */ +function stripOuterMarkdownFence(content: string): string { + const trimmed = content.trim(); + const match = trimmed.match(/^```(?:markdown|md)?\s*\n([\s\S]+?)\n```\s*$/); + return match ? match[1] : content; +} + +/** + * Convert all LaTeX delimiter styles to the double-dollar syntax + * that Streamdown's @streamdown/math plugin understands. + * + * Streamdown math conventions (different from remark-math!): + * $$...$$ on the SAME line → inline math + * $$\n...\n$$ on SEPARATE lines → block (display) math + * + * Conversions performed: + * \[...\] → $$\n ... \n$$ (block math) + * \(...\) → $$...$$ (inline math, same line) + * \begin{equation}...\end{equation} → $$\n ... \n$$ (block math) + * \begin{displaymath}...\end{displaymath} → $$\n ... \n$$ (block math) + * \begin{math}...\end{math} → $$...$$ (inline math, same line) + * `$$ … $$` → $$ … $$ (strip wrapping backtick code) + * `$ … $` → $ … $ (strip wrapping backtick code) + * $...$ → $$...$$ (normalise single-$ to double-$$) + */ +function convertLatexDelimiters(content: string): string { + // 1. Block math: \[...\] → $$\n...\n$$ (display math on separate lines) + content = content.replace(/\\\[([\s\S]*?)\\\]/g, (_, inner) => `\n$$\n${inner.trim()}\n$$\n`); + // 2. Inline math: \(...\) → $$...$$ (inline math on same line) + content = content.replace(/\\\(([\s\S]*?)\\\)/g, (_, inner) => `$$${inner.trim()}$$`); + // 3. Block: \begin{equation}...\end{equation} → $$\n...\n$$ + content = content.replace( + /\\begin\{equation\}([\s\S]*?)\\end\{equation\}/g, + (_, inner) => `\n$$\n${inner.trim()}\n$$\n` + ); + // 4. Block: \begin{displaymath}...\end{displaymath} → $$\n...\n$$ + content = content.replace( + /\\begin\{displaymath\}([\s\S]*?)\\end\{displaymath\}/g, + (_, inner) => `\n$$\n${inner.trim()}\n$$\n` + ); + // 5. Inline: \begin{math}...\end{math} → $$...$$ + content = content.replace( + /\\begin\{math\}([\s\S]*?)\\end\{math\}/g, + (_, inner) => `$$${inner.trim()}$$` + ); + // 6. Strip backtick wrapping around math: `$$...$$` → $$...$$ and `$...$` → $...$ + content = content.replace(/`(\${1,2})((?:(?!\1).)+)\1`/g, "$1$2$1"); + // 7. Normalise single-dollar $...$ to double-dollar $$...$$ so they render + // reliably in Streamdown (single-$ has strict no-space rules that often fail). + // We match $…$ where the content starts with a backslash (LaTeX command) + // to avoid converting currency like $50. + content = content.replace( + /(? `$$${inner.trim()}$$` + ); + return content; +} + export function MarkdownViewer({ content, className }: MarkdownViewerProps) { + const processedContent = convertLatexDelimiters(stripOuterMarkdownFence(content)); const components: StreamdownProps["components"] = { - // Define custom components for markdown elements - callout: ({ children, ...props }) => ( -
- {children} -
- ), p: ({ children, ...props }) => (

{children} @@ -71,42 +135,41 @@ export function MarkdownViewer({ content, className }: MarkdownViewerProps) { /> ), table: ({ ...props }) => ( -

- +
+
), - th: ({ ...props }) =>
, - td: ({ ...props }) => , - code: ({ className, children, ...props }) => { - const match = /language-(\w+)/.exec(className || ""); - const isInline = !match; - - if (isInline) { - return ( - - {children} - - ); - } - - // For code blocks, let Streamdown handle syntax highlighting - return ( - - {children} - - ); - }, + th: ({ ...props }) => ( + + ), + td: ({ ...props }) => ( + + ), }; return (
*]:!border-none [&_[data-streamdown=code-block]>*]:![box-shadow:none]", + "[&_[data-streamdown=code-block-download-button]]:!hidden", className )} > - - {content} + + {processedContent}
); diff --git a/surfsense_web/components/new-chat/chat-share-button.tsx b/surfsense_web/components/new-chat/chat-share-button.tsx index c9e693b21..ece5f7fae 100644 --- a/surfsense_web/components/new-chat/chat-share-button.tsx +++ b/surfsense_web/components/new-chat/chat-share-button.tsx @@ -138,6 +138,26 @@ export function ChatShareButton({ thread, onVisibilityChange, className }: ChatS return (
+ {/* Globe indicator when public snapshots exist - clicks to settings */} + {hasPublicSnapshots && ( + + + + + + {snapshotCount === 1 + ? "This chat has a public link" + : `This chat has ${snapshotCount} public links`} + + + )} + @@ -242,26 +262,6 @@ export function ChatShareButton({ thread, onVisibilityChange, className }: ChatS
- - {/* Globe indicator when public snapshots exist - clicks to settings */} - {hasPublicSnapshots && ( - - - - - - {snapshotCount === 1 - ? "This chat has a public link" - : `This chat has ${snapshotCount} public links`} - - - )} ); } diff --git a/surfsense_web/components/public-chat/public-chat-view.tsx b/surfsense_web/components/public-chat/public-chat-view.tsx index a1e6008ff..bfe370bf0 100644 --- a/surfsense_web/components/public-chat/public-chat-view.tsx +++ b/surfsense_web/components/public-chat/public-chat-view.tsx @@ -2,8 +2,10 @@ import { AssistantRuntimeProvider } from "@assistant-ui/react"; import { Navbar } from "@/components/homepage/navbar"; +import { ReportPanel } from "@/components/report-panel/report-panel"; import { DisplayImageToolUI } from "@/components/tool-ui/display-image"; import { GeneratePodcastToolUI } from "@/components/tool-ui/generate-podcast"; +import { GenerateReportToolUI } from "@/components/tool-ui/generate-report"; import { LinkPreviewToolUI } from "@/components/tool-ui/link-preview"; import { ScrapeWebpageToolUI } from "@/components/tool-ui/scrape-webpage"; import { Spinner } from "@/components/ui/spinner"; @@ -42,12 +44,16 @@ export function PublicChatView({ shareToken }: PublicChatViewProps) { {/* Tool UIs for rendering tool results */} + -
- } /> +
+
+ } /> +
+
diff --git a/surfsense_web/components/report-panel/report-panel.tsx b/surfsense_web/components/report-panel/report-panel.tsx new file mode 100644 index 000000000..84b271511 --- /dev/null +++ b/surfsense_web/components/report-panel/report-panel.tsx @@ -0,0 +1,487 @@ +"use client"; + +import { useAtomValue, useSetAtom } from "jotai"; +import { ChevronDownIcon, XIcon } from "lucide-react"; +import { useCallback, useEffect, useRef, useState } from "react"; +import { z } from "zod"; +import { closeReportPanelAtom, reportPanelAtom } from "@/atoms/chat/report-panel.atom"; +import { MarkdownViewer } from "@/components/markdown-viewer"; +import { Button } from "@/components/ui/button"; +import { Drawer, DrawerContent, DrawerHandle } from "@/components/ui/drawer"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; +import { useMediaQuery } from "@/hooks/use-media-query"; +import { baseApiService } from "@/lib/apis/base-api.service"; +import { authenticatedFetch } from "@/lib/auth-utils"; + +/** + * Zod schema for a single version entry + */ +const VersionInfoSchema = z.object({ + id: z.number(), + created_at: z.string().nullish(), +}); + +/** + * Zod schema for the report content API response + */ +const ReportContentResponseSchema = z.object({ + id: z.number(), + title: z.string(), + content: z.string().nullish(), + report_metadata: z + .object({ + status: z.enum(["ready", "failed"]).nullish(), + error_message: z.string().nullish(), + word_count: z.number().nullish(), + char_count: z.number().nullish(), + section_count: z.number().nullish(), + }) + .nullish(), + report_group_id: z.number().nullish(), + versions: z.array(VersionInfoSchema).nullish(), +}); + +type ReportContentResponse = z.infer; +type VersionInfo = z.infer; + +/** + * Shimmer loading skeleton for report panel + */ +function ReportPanelSkeleton() { + return ( +
+ {/* Title skeleton */} +
+ + {/* Paragraph 1 */} +
+
+
+
+
+
+ + {/* Heading */} +
+ + {/* Paragraph 2 */} +
+
+
+
+
+ + {/* Heading */} +
+ + {/* Paragraph 3 */} +
+
+
+
+
+
+ ); +} + +/** + * Inner content component used by both desktop panel and mobile drawer + */ +function ReportPanelContent({ + reportId, + title, + onClose, + insideDrawer = false, + shareToken, +}: { + reportId: number; + title: string; + onClose?: () => void; + /** When true, adjusts dropdown behavior to work inside a Vaul drawer on mobile */ + insideDrawer?: boolean; + /** When set, uses public endpoint for fetching report data (public shared chat) */ + shareToken?: string | null; +}) { + const [reportContent, setReportContent] = useState(null); + const [isLoading, setIsLoading] = useState(true); + const [error, setError] = useState(null); + const [copied, setCopied] = useState(false); + const [exporting, setExporting] = useState<"pdf" | "docx" | "md" | null>(null); + + // Version state + const [activeReportId, setActiveReportId] = useState(reportId); + const [versions, setVersions] = useState([]); + + // Reset active version when the external reportId changes (e.g. clicking a different card) + useEffect(() => { + setActiveReportId(reportId); + }, [reportId]); + + // Fetch report content (re-runs when activeReportId changes for version switching) + useEffect(() => { + let cancelled = false; + const fetchContent = async () => { + setIsLoading(true); + setError(null); + try { + const url = shareToken + ? `/api/v1/public/${shareToken}/reports/${activeReportId}/content` + : `/api/v1/reports/${activeReportId}/content`; + const rawData = await baseApiService.get(url); + if (cancelled) return; + const parsed = ReportContentResponseSchema.safeParse(rawData); + if (parsed.success) { + // Check if the report was marked as failed in metadata + if (parsed.data.report_metadata?.status === "failed") { + setError(parsed.data.report_metadata?.error_message || "Report generation failed"); + } else { + setReportContent(parsed.data); + // Update versions from the response + if (parsed.data.versions && parsed.data.versions.length > 0) { + setVersions(parsed.data.versions); + } + } + } else { + console.warn("Invalid report content response:", parsed.error.issues); + setError("Invalid response format"); + } + } catch (err) { + if (cancelled) return; + console.error("Error fetching report content:", err); + setError(err instanceof Error ? err.message : "Failed to load report"); + } finally { + if (!cancelled) setIsLoading(false); + } + }; + + fetchContent(); + return () => { + cancelled = true; + }; + }, [activeReportId, shareToken]); + + // Copy markdown content + const handleCopy = useCallback(async () => { + if (!reportContent?.content) return; + try { + await navigator.clipboard.writeText(reportContent.content); + setCopied(true); + setTimeout(() => setCopied(false), 2000); + } catch (err) { + console.error("Failed to copy:", err); + } + }, [reportContent?.content]); + + // Export report + const handleExport = useCallback( + async (format: "pdf" | "docx" | "md") => { + setExporting(format); + const safeTitle = + title + .replace(/[^a-zA-Z0-9 _-]/g, "_") + .trim() + .slice(0, 80) || "report"; + try { + if (format === "md") { + // Download markdown content directly as a .md file + if (!reportContent?.content) return; + const blob = new Blob([reportContent.content], { + type: "text/markdown;charset=utf-8", + }); + const url = URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = `${safeTitle}.md`; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + URL.revokeObjectURL(url); + } else { + const response = await authenticatedFetch( + `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/reports/${activeReportId}/export?format=${format}`, + { method: "GET" } + ); + + if (!response.ok) { + throw new Error(`Export failed: ${response.status}`); + } + + const blob = await response.blob(); + const url = URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = `${safeTitle}.${format}`; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + URL.revokeObjectURL(url); + } + } catch (err) { + console.error(`Export ${format} failed:`, err); + } finally { + setExporting(null); + } + }, + [activeReportId, title, reportContent?.content] + ); + + // Show full-page skeleton only on initial load (no data loaded yet). + // Once we have versions/content from a prior fetch, keep the action bar visible. + const hasLoadedBefore = versions.length > 0 || reportContent !== null; + + if (isLoading && !hasLoadedBefore) { + return ( + <> + {/* Minimal top bar with close button even during initial load */} +
+ {onClose && ( + + )} +
+ + + ); + } + + const activeVersionIndex = versions.findIndex((v) => v.id === activeReportId); + + return ( + <> + {/* Action bar — always visible after initial load */} +
+
+ {/* Copy button */} + + + {/* Export dropdown */} + + + + + + handleExport("md")}> + Download Markdown + + {/* PDF/DOCX export requires server-side conversion via authenticated endpoint. + Hide for public viewers who have no auth token. */} + {!shareToken && ( + <> + handleExport("pdf")} + disabled={exporting !== null} + > + Download PDF + + handleExport("docx")} + disabled={exporting !== null} + > + Download DOCX + + + )} + + + + {/* Version switcher — only shown when multiple versions exist */} + {versions.length > 1 && + (insideDrawer ? ( + /* Mobile: compact dropdown */ + + + + + + {versions.map((v, i) => ( + setActiveReportId(v.id)} + className={v.id === activeReportId ? "bg-accent font-medium" : ""} + > + Version {i + 1} + + ))} + + + ) : ( + /* Desktop: inline version buttons */ +
+
+ {versions.map((v, i) => ( + + ))} +
+ + {activeVersionIndex + 1} of {versions.length} + +
+ ))} +
+ {onClose && ( + + )} +
+ + {/* Report content — skeleton/error/content shown only in this area */} +
+ {isLoading ? ( + + ) : error || !reportContent ? ( +
+
+

Failed to load report

+

{error || "An unknown error occurred"}

+
+
+ ) : ( +
+ {reportContent.content ? ( + + ) : ( +

No content available.

+ )} +
+ )} +
+ + ); +} + +/** + * Desktop report panel — renders as a right-side flex sibling + */ +function DesktopReportPanel() { + const panelState = useAtomValue(reportPanelAtom); + const closePanel = useSetAtom(closeReportPanelAtom); + const panelRef = useRef(null); + + // Close panel on Escape key + useEffect(() => { + const handleKeyDown = (e: KeyboardEvent) => { + if (e.key === "Escape") { + closePanel(); + } + }; + document.addEventListener("keydown", handleKeyDown); + return () => document.removeEventListener("keydown", handleKeyDown); + }, [closePanel]); + + if (!panelState.isOpen || !panelState.reportId) return null; + + return ( +
+ +
+ ); +} + +/** + * Mobile report drawer — uses Vaul (same pattern as comment sheet) + */ +function MobileReportDrawer() { + const panelState = useAtomValue(reportPanelAtom); + const closePanel = useSetAtom(closeReportPanelAtom); + + if (!panelState.reportId) return null; + + return ( + { + if (!open) closePanel(); + }} + shouldScaleBackground={false} + > + + +
+ +
+
+
+ ); +} + +/** + * ReportPanel — responsive report viewer + * + * On desktop (lg+): Renders as a right-side split panel (flex sibling to the chat thread) + * On mobile/tablet: Renders as a Vaul bottom drawer + * + * When open on desktop, the comments gutter is automatically suppressed + * (handled via showCommentsGutterAtom in current-thread.atom.ts) + */ +export function ReportPanel() { + const panelState = useAtomValue(reportPanelAtom); + const isDesktop = useMediaQuery("(min-width: 1024px)"); + + // Don't render anything if panel is not open + if (!panelState.isOpen || !panelState.reportId) return null; + + if (isDesktop) { + return ; + } + + return ; +} diff --git a/surfsense_web/components/tool-ui/article/index.tsx b/surfsense_web/components/tool-ui/article/index.tsx index 5669ea832..43ea7c4c9 100644 --- a/surfsense_web/components/tool-ui/article/index.tsx +++ b/surfsense_web/components/tool-ui/article/index.tsx @@ -8,7 +8,8 @@ import { FileTextIcon, UserIcon, } from "lucide-react"; -import { Component, type ReactNode, useCallback } from "react"; +import Image from "next/image"; +import { Component, type ReactNode, useCallback, useState } from "react"; import { z } from "zod"; import { Card, CardContent } from "@/components/ui/card"; import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; @@ -126,6 +127,30 @@ function formatWordCount(count: number): string { return `${count} words`; } +/** + * Favicon component that fetches the site icon via Google's favicon service, + * falling back to BookOpenIcon on error. + */ +function SiteFavicon({ domain }: { domain: string }) { + const [failed, setFailed] = useState(false); + + if (failed) { + return ; + } + + return ( + {`${domain} setFailed(true)} + unoptimized + /> + ); +} + /** * Article card component for displaying scraped webpage content */ @@ -198,27 +223,35 @@ export function Article({ }} > {/* Header */} - -
- {/* Icon */} -
- -
+ +
+ {/* Favicon / Icon */} + {domain ? ( +
+ +
+ ) : ( +
+ +
+ )} {/* Content */}
{/* Title */} -

+

{title}

{/* Description */} {description && ( -

{description}

+

+ {description} +

)} {/* Metadata row */} -
+
{domain && ( @@ -274,13 +307,6 @@ export function Article({ )}
- - {/* External link indicator */} - {href && ( -
- -
- )}
{/* Response actions */} diff --git a/surfsense_web/components/tool-ui/create-notion-page.tsx b/surfsense_web/components/tool-ui/create-notion-page.tsx new file mode 100644 index 000000000..e148c71ba --- /dev/null +++ b/surfsense_web/components/tool-ui/create-notion-page.tsx @@ -0,0 +1,503 @@ +"use client"; + +import { makeAssistantToolUI } from "@assistant-ui/react"; +import { AlertTriangleIcon, CheckIcon, Loader2Icon, PencilIcon, XIcon } from "lucide-react"; +import { useMemo, useState } from "react"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Textarea } from "@/components/ui/textarea"; + +interface InterruptResult { + __interrupt__: true; + __decided__?: "approve" | "reject" | "edit"; + action_requests: Array<{ + name: string; + args: Record; + description?: string; + }>; + review_configs: Array<{ + action_name: string; + allowed_decisions: Array<"approve" | "edit" | "reject">; + }>; + interrupt_type?: string; + message?: string; + context?: { + accounts?: Array<{ + id: number; + name: string; + workspace_id: string | null; + workspace_name: string; + workspace_icon: string; + }>; + parent_pages?: Record< + number, + Array<{ + page_id: string; + title: string; + document_id: number; + }> + >; + error?: string; + }; +} + +interface SuccessResult { + status: "success"; + page_id: string; + title: string; + url: string; + content_preview?: string; + content_length?: number; + message?: string; +} + +interface ErrorResult { + status: "error"; + message: string; +} + +type CreateNotionPageResult = InterruptResult | SuccessResult | ErrorResult; + +function isInterruptResult(result: unknown): result is InterruptResult { + return ( + typeof result === "object" && + result !== null && + "__interrupt__" in result && + (result as InterruptResult).__interrupt__ === true + ); +} + +function isErrorResult(result: unknown): result is ErrorResult { + return ( + typeof result === "object" && + result !== null && + "status" in result && + (result as ErrorResult).status === "error" + ); +} + +function ApprovalCard({ + args, + interruptData, + onDecision, +}: { + args: Record; + interruptData: InterruptResult; + onDecision: (decision: { + type: "approve" | "reject" | "edit"; + message?: string; + edited_action?: { name: string; args: Record }; + }) => void; +}) { + const [decided, setDecided] = useState<"approve" | "reject" | "edit" | null>( + interruptData.__decided__ ?? null + ); + const [isEditing, setIsEditing] = useState(false); + const [editedArgs, setEditedArgs] = useState>(args); + + const accounts = interruptData.context?.accounts ?? []; + const parentPages = interruptData.context?.parent_pages ?? {}; + + const defaultAccountId = useMemo(() => { + if (args.connector_id) return String(args.connector_id); + if (accounts.length === 1) return String(accounts[0].id); + return ""; + }, [args.connector_id, accounts]); + + const [selectedAccountId, setSelectedAccountId] = useState(defaultAccountId); + const [selectedParentPageId, setSelectedParentPageId] = useState( + args.parent_page_id ? String(args.parent_page_id) : "__none__" + ); + + const availableParentPages = useMemo(() => { + if (!selectedAccountId) return []; + return parentPages[Number(selectedAccountId)] ?? []; + }, [selectedAccountId, parentPages]); + + const isTitleValid = useMemo(() => { + const currentTitle = isEditing ? editedArgs.title : args.title; + return currentTitle && typeof currentTitle === "string" && currentTitle.trim().length > 0; + }, [isEditing, editedArgs.title, args.title]); + + const reviewConfig = interruptData.review_configs[0]; + const allowedDecisions = reviewConfig?.allowed_decisions ?? ["approve", "reject"]; + const canEdit = allowedDecisions.includes("edit"); + + return ( +
+
+
+ +
+
+

+ Create Notion Page +

+

+ {isEditing ? "You can edit the arguments below" : "Requires your approval to proceed"} +

+
+
+ + {/* Context section - account and parent page selection */} + {!decided && interruptData.context && ( +
+ {interruptData.context.error ? ( +

{interruptData.context.error}

+ ) : ( + <> + {accounts.length > 0 && ( +
+
+ Notion Account * +
+ +
+ )} + + {selectedAccountId && ( +
+
+ Parent Page (optional) +
+ + {availableParentPages.length === 0 && selectedAccountId && ( +

+ No pages available. Page will be created at workspace root. +

+ )} +
+ )} + + )} +
+ )} + + {/* Display mode - show args as read-only */} + {!isEditing && ( +
+ {args.title != null && ( +
+

Title

+

{String(args.title)}

+
+ )} + {args.content != null && ( +
+

Content

+

+ {String(args.content)} +

+
+ )} +
+ )} + + {/* Edit mode - show editable form fields */} + {isEditing && !decided && ( +
+
+ + setEditedArgs({ ...editedArgs, title: e.target.value })} + placeholder="Enter page title" + className={!isTitleValid ? "border-destructive" : ""} + /> + {!isTitleValid && ( +

Title is required and cannot be empty

+ )} +
+
+ +