From a9de890cd863c83b1ac4f675b3445e0acda356ea Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 20 Jan 2026 16:04:17 +0200 Subject: [PATCH 01/44] Add chat_session_state table migration --- .../73_add_chat_session_state_table.py | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 surfsense_backend/alembic/versions/73_add_chat_session_state_table.py diff --git a/surfsense_backend/alembic/versions/73_add_chat_session_state_table.py b/surfsense_backend/alembic/versions/73_add_chat_session_state_table.py new file mode 100644 index 000000000..12e99f306 --- /dev/null +++ b/surfsense_backend/alembic/versions/73_add_chat_session_state_table.py @@ -0,0 +1,75 @@ +"""Add chat_session_state table for live collaboration + +Revision ID: 73 +Revises: 72 + +Creates chat_session_state table to track AI responding state per thread. +Enables real-time sync via Electric SQL for shared chat collaboration. +""" + +from collections.abc import Sequence + +from alembic import op + +revision: str = "73" +down_revision: str | None = "72" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Create chat_session_state table with Electric SQL replication.""" + op.execute( + """ + CREATE TABLE IF NOT EXISTS chat_session_state ( + id SERIAL PRIMARY KEY, + thread_id INTEGER NOT NULL REFERENCES new_chat_threads(id) ON DELETE CASCADE, + ai_responding_to_user_id UUID REFERENCES "user"(id) ON DELETE SET NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (thread_id) + ) + """ + ) + + op.execute( + "CREATE INDEX IF NOT EXISTS idx_chat_session_state_thread_id ON chat_session_state(thread_id)" + ) + + op.execute("ALTER TABLE chat_session_state REPLICA IDENTITY FULL;") + + op.execute( + """ + DO $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_publication_tables + WHERE pubname = 'electric_publication_default' + AND tablename = 'chat_session_state' + ) THEN + ALTER PUBLICATION electric_publication_default ADD TABLE chat_session_state; + END IF; + END + $$; + """ + ) + + +def downgrade() -> None: + """Drop chat_session_state table and remove from Electric SQL replication.""" + op.execute( + """ + DO $$ + BEGIN + IF EXISTS ( + SELECT 1 FROM pg_publication_tables + WHERE pubname = 'electric_publication_default' + AND tablename = 'chat_session_state' + ) THEN + ALTER PUBLICATION electric_publication_default DROP TABLE chat_session_state; + END IF; + END + $$; + """ + ) + + op.execute("DROP TABLE IF EXISTS chat_session_state;") From 7d35419b882da4c206af249637029ffea707b031 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 20 Jan 2026 16:17:54 +0200 Subject: [PATCH 02/44] Add ChatSessionState model --- surfsense_backend/app/db.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py index 38e27ecf2..35b512c5e 100644 --- a/surfsense_backend/app/db.py +++ b/surfsense_backend/app/db.py @@ -472,6 +472,38 @@ class ChatCommentMention(BaseModel, TimestampMixin): mentioned_user = relationship("User") +class ChatSessionState(BaseModel): + """ + Tracks real-time session state for shared chat collaboration. + One record per thread, synced via Electric SQL. + """ + + __tablename__ = "chat_session_state" + + thread_id = Column( + Integer, + ForeignKey("new_chat_threads.id", ondelete="CASCADE"), + nullable=False, + unique=True, + index=True, + ) + ai_responding_to_user_id = Column( + UUID(as_uuid=True), + ForeignKey("user.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + updated_at = Column( + TIMESTAMP(timezone=True), + nullable=False, + default=lambda: datetime.now(UTC), + onupdate=lambda: datetime.now(UTC), + ) + + thread = relationship("NewChatThread") + ai_responding_to_user = relationship("User") + + class Document(BaseModel, TimestampMixin): __tablename__ = "documents" From 55a07c064e8e0d56f99cb0d9c2094d591235169f Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 20 Jan 2026 16:24:37 +0200 Subject: [PATCH 03/44] Add chat session state schemas --- .../app/schemas/chat_session_state.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 surfsense_backend/app/schemas/chat_session_state.py diff --git a/surfsense_backend/app/schemas/chat_session_state.py b/surfsense_backend/app/schemas/chat_session_state.py new file mode 100644 index 000000000..6eca0e26f --- /dev/null +++ b/surfsense_backend/app/schemas/chat_session_state.py @@ -0,0 +1,29 @@ +""" +Pydantic schemas for chat session state (live collaboration). +""" + +from datetime import datetime +from uuid import UUID + +from pydantic import BaseModel, ConfigDict + + +class RespondingUser(BaseModel): + """The user that the AI is currently responding to.""" + + id: UUID + display_name: str | None = None + email: str + + model_config = ConfigDict(from_attributes=True) + + +class ChatSessionStateResponse(BaseModel): + """Current session state for a chat thread.""" + + id: int + thread_id: int + responding_to: RespondingUser | None = None + updated_at: datetime + + model_config = ConfigDict(from_attributes=True) From d7b0b90a01c40d941440e84078c3944d95e61326 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 20 Jan 2026 16:31:40 +0200 Subject: [PATCH 04/44] Add chat session state service --- .../services/chat_session_state_service.py | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 surfsense_backend/app/services/chat_session_state_service.py diff --git a/surfsense_backend/app/services/chat_session_state_service.py b/surfsense_backend/app/services/chat_session_state_service.py new file mode 100644 index 000000000..d82fff3a7 --- /dev/null +++ b/surfsense_backend/app/services/chat_session_state_service.py @@ -0,0 +1,65 @@ +""" +Service layer for chat session state (live collaboration). +""" + +from datetime import UTC, datetime +from uuid import UUID + +from sqlalchemy import select +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import selectinload + +from app.db import ChatSessionState + + +async def get_session_state( + session: AsyncSession, + thread_id: int, +) -> ChatSessionState | None: + """Get the current session state for a thread.""" + result = await session.execute( + select(ChatSessionState) + .options(selectinload(ChatSessionState.ai_responding_to_user)) + .filter(ChatSessionState.thread_id == thread_id) + ) + return result.scalar_one_or_none() + + +async def set_ai_responding( + session: AsyncSession, + thread_id: int, + user_id: UUID, +) -> ChatSessionState: + """Mark AI as responding to a specific user. Uses upsert for atomicity.""" + now = datetime.now(UTC) + upsert_query = insert(ChatSessionState).values( + thread_id=thread_id, + ai_responding_to_user_id=user_id, + updated_at=now, + ) + upsert_query = upsert_query.on_conflict_do_update( + index_elements=["thread_id"], + set_={ + "ai_responding_to_user_id": user_id, + "updated_at": now, + }, + ) + await session.execute(upsert_query) + await session.commit() + + return await get_session_state(session, thread_id) + + +async def clear_ai_responding( + session: AsyncSession, + thread_id: int, +) -> ChatSessionState | None: + """Clear AI responding state when response is complete.""" + state = await get_session_state(session, thread_id) + if state: + state.ai_responding_to_user_id = None + state.updated_at = datetime.now(UTC) + await session.commit() + await session.refresh(state) + return state From dc628198cec68f60b9878b5bd0eb26e6699d0848 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 20 Jan 2026 16:40:38 +0200 Subject: [PATCH 05/44] Integrate session state into chat streaming --- surfsense_backend/app/routes/new_chat_routes.py | 1 + .../app/tasks/chat/stream_new_chat.py | 14 +++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/surfsense_backend/app/routes/new_chat_routes.py b/surfsense_backend/app/routes/new_chat_routes.py index 8fddc55c4..1ef3d0822 100644 --- a/surfsense_backend/app/routes/new_chat_routes.py +++ b/surfsense_backend/app/routes/new_chat_routes.py @@ -990,6 +990,7 @@ async def handle_new_chat( search_space_id=request.search_space_id, chat_id=request.chat_id, session=session, + user_id=user.id, llm_config_id=llm_config_id, attachments=request.attachments, mentioned_document_ids=request.mentioned_document_ids, diff --git a/surfsense_backend/app/tasks/chat/stream_new_chat.py b/surfsense_backend/app/tasks/chat/stream_new_chat.py index 85a524108..984cecc6d 100644 --- a/surfsense_backend/app/tasks/chat/stream_new_chat.py +++ b/surfsense_backend/app/tasks/chat/stream_new_chat.py @@ -11,6 +11,7 @@ Supports loading LLM configurations from: import json from collections.abc import AsyncGenerator +from uuid import UUID from langchain_core.messages import HumanMessage from sqlalchemy.ext.asyncio import AsyncSession @@ -27,6 +28,10 @@ from app.agents.new_chat.llm_config import ( ) from app.db import Document, SurfsenseDocsDocument from app.schemas.new_chat import ChatAttachment +from app.services.chat_session_state_service import ( + clear_ai_responding, + set_ai_responding, +) from app.services.connector_service import ConnectorService from app.services.new_streaming_service import VercelStreamingService @@ -149,6 +154,7 @@ async def stream_new_chat( search_space_id: int, chat_id: int, session: AsyncSession, + user_id: UUID, llm_config_id: int = -1, attachments: list[ChatAttachment] | None = None, mentioned_document_ids: list[int] | None = None, @@ -166,8 +172,8 @@ async def stream_new_chat( search_space_id: The search space ID chat_id: The chat ID (used as LangGraph thread_id for memory) session: The database session + user_id: The ID of the user sending the message llm_config_id: The LLM configuration ID (default: -1 for first global config) - messages: Optional chat history from frontend (list of ChatMessage) attachments: Optional attachments with extracted content mentioned_document_ids: Optional list of document IDs mentioned with @ in the chat mentioned_surfsense_doc_ids: Optional list of SurfSense doc IDs mentioned with @ in the chat @@ -181,6 +187,8 @@ async def stream_new_chat( current_text_id: str | None = None try: + # Mark AI as responding to this user for live collaboration + await set_ai_responding(session, chat_id, user_id) # Load LLM config - supports both YAML (negative IDs) and database (positive IDs) agent_config: AgentConfig | None = None @@ -1144,3 +1152,7 @@ async def stream_new_chat( yield streaming_service.format_finish_step() yield streaming_service.format_finish() yield streaming_service.format_done() + + finally: + # Clear AI responding state for live collaboration + await clear_ai_responding(session, chat_id) From 6a31e79edeb2da2fa1770fe666b9c84e379158e1 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 20 Jan 2026 16:52:18 +0200 Subject: [PATCH 06/44] Add chat session state types for live collaboration --- .../types/chat-session-state.types.ts | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 surfsense_web/contracts/types/chat-session-state.types.ts diff --git a/surfsense_web/contracts/types/chat-session-state.types.ts b/surfsense_web/contracts/types/chat-session-state.types.ts new file mode 100644 index 000000000..cf73859e6 --- /dev/null +++ b/surfsense_web/contracts/types/chat-session-state.types.ts @@ -0,0 +1,24 @@ +import { z } from "zod"; + +/** + * Chat session state for live collaboration. + * Tracks which user the AI is currently responding to. + */ +export const chatSessionState = z.object({ + id: z.number(), + thread_id: z.number(), + ai_responding_to_user_id: z.string().uuid().nullable(), + updated_at: z.string(), +}); + +/** + * User currently being responded to by the AI. + */ +export const respondingUser = z.object({ + id: z.string().uuid(), + display_name: z.string().nullable(), + email: z.string(), +}); + +export type ChatSessionState = z.infer; +export type RespondingUser = z.infer; From 69f2460d18a748e569c880df5d1188fbf593a80c Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 20 Jan 2026 18:26:58 +0200 Subject: [PATCH 07/44] Refactor chat session state hook to use useShape --- surfsense_web/hooks/use-chat-session-state.ts | 28 +++++++++++++++++++ surfsense_web/lib/electric/client.ts | 1 - 2 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 surfsense_web/hooks/use-chat-session-state.ts diff --git a/surfsense_web/hooks/use-chat-session-state.ts b/surfsense_web/hooks/use-chat-session-state.ts new file mode 100644 index 000000000..0272464df --- /dev/null +++ b/surfsense_web/hooks/use-chat-session-state.ts @@ -0,0 +1,28 @@ +"use client"; + +import { useShape } from "@electric-sql/react"; +import type { ChatSessionState } from "@/contracts/types/chat-session-state.types"; + +const ELECTRIC_URL = process.env.NEXT_PUBLIC_ELECTRIC_URL || "http://localhost:5133"; + +export function useChatSessionState(threadId: number | null) { + const { data, isLoading, isError, error } = useShape({ + url: `${ELECTRIC_URL}/v1/shape`, + params: { + table: "chat_session_state", + where: `thread_id = ${threadId}`, + }, + // Skip fetching if no threadId + ...(threadId ? {} : { url: undefined as unknown as string }), + }); + + const sessionState = data?.[0] ?? null; + + return { + sessionState, + isAiResponding: !!sessionState?.ai_responding_to_user_id, + respondingToUserId: sessionState?.ai_responding_to_user_id ?? null, + loading: isLoading, + error: isError ? error : null, + }; +} diff --git a/surfsense_web/lib/electric/client.ts b/surfsense_web/lib/electric/client.ts index 514185d23..d900ddb0a 100644 --- a/surfsense_web/lib/electric/client.ts +++ b/surfsense_web/lib/electric/client.ts @@ -228,7 +228,6 @@ export async function initElectric(userId: string): Promise { CREATE INDEX IF NOT EXISTS idx_documents_search_space_type ON documents(search_space_id, document_type); `); - // Create the chat_comment_mentions table schema in PGlite await db.exec(` CREATE TABLE IF NOT EXISTS chat_comment_mentions ( id INTEGER PRIMARY KEY, From b56c70c401c4e395617384d2b3b4fc13c91c0cec Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 20 Jan 2026 18:33:29 +0200 Subject: [PATCH 08/44] Add ChatSessionStatus component with animation --- .../assistant-ui/chat-session-status.tsx | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 surfsense_web/components/assistant-ui/chat-session-status.tsx diff --git a/surfsense_web/components/assistant-ui/chat-session-status.tsx b/surfsense_web/components/assistant-ui/chat-session-status.tsx new file mode 100644 index 000000000..aef4aaeb0 --- /dev/null +++ b/surfsense_web/components/assistant-ui/chat-session-status.tsx @@ -0,0 +1,49 @@ +"use client"; + +import type { FC } from "react"; +import { Loader2 } from "lucide-react"; +import { cn } from "@/lib/utils"; + +interface ChatSessionStatusProps { + isAiResponding: boolean; + respondingToUserId: string | null; + currentUserId: string | null; + members: Array<{ + user_id: string; + user_display_name: string | null; + user_email: string; + }>; + className?: string; +} + +export const ChatSessionStatus: FC = ({ + isAiResponding, + respondingToUserId, + currentUserId, + members, + className, +}) => { + if (!isAiResponding || !respondingToUserId) { + return null; + } + + if (respondingToUserId === currentUserId) { + return null; + } + + const respondingUser = members.find((m) => m.user_id === respondingToUserId); + const displayName = respondingUser?.user_display_name || respondingUser?.user_email || "another user"; + + return ( +
+ + Currently responding to {displayName} +
+ ); +}; From 22ead877fa80fbcf13883fa27379eb0e9e17a3f7 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 20 Jan 2026 18:39:50 +0200 Subject: [PATCH 09/44] Integrate ChatSessionStatus and blocking logic into Composer --- .../assistant-ui/chat-session-status.tsx | 4 +-- .../components/assistant-ui/thread.tsx | 30 ++++++++++++++++--- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/surfsense_web/components/assistant-ui/chat-session-status.tsx b/surfsense_web/components/assistant-ui/chat-session-status.tsx index aef4aaeb0..62f7c33ce 100644 --- a/surfsense_web/components/assistant-ui/chat-session-status.tsx +++ b/surfsense_web/components/assistant-ui/chat-session-status.tsx @@ -10,8 +10,8 @@ interface ChatSessionStatusProps { currentUserId: string | null; members: Array<{ user_id: string; - user_display_name: string | null; - user_email: string; + user_display_name?: string | null; + user_email?: string | null; }>; className?: string; } diff --git a/surfsense_web/components/assistant-ui/thread.tsx b/surfsense_web/components/assistant-ui/thread.tsx index eaf30fc96..1d83b5a60 100644 --- a/surfsense_web/components/assistant-ui/thread.tsx +++ b/surfsense_web/components/assistant-ui/thread.tsx @@ -31,6 +31,7 @@ import { mentionedDocumentIdsAtom, mentionedDocumentsAtom, } from "@/atoms/chat/mentioned-documents.atom"; +import { membersAtom } from "@/atoms/members/members-query.atoms"; import { globalNewLLMConfigsAtom, llmPreferencesAtom, @@ -39,6 +40,7 @@ import { import { currentUserAtom } from "@/atoms/user/user-query.atoms"; import { AssistantMessage } from "@/components/assistant-ui/assistant-message"; import { ComposerAddAttachment, ComposerAttachments } from "@/components/assistant-ui/attachment"; +import { ChatSessionStatus } from "@/components/assistant-ui/chat-session-status"; import { ConnectorIndicator } from "@/components/assistant-ui/connector-popup"; import { InlineMentionEditor, @@ -59,6 +61,7 @@ import { import type { ThinkingStep } from "@/components/tool-ui/deepagent-thinking"; import { Button } from "@/components/ui/button"; import type { Document } from "@/contracts/types/document.types"; +import { useChatSessionState } from "@/hooks/use-chat-session-state"; import { cn } from "@/lib/utils"; interface ThreadProps { @@ -215,7 +218,7 @@ const Composer: FC = () => { const editorRef = useRef(null); const editorContainerRef = useRef(null); const documentPickerRef = useRef(null); - const { search_space_id } = useParams(); + const { search_space_id, chat_id } = useParams(); const setMentionedDocumentIds = useSetAtom(mentionedDocumentIdsAtom); const composerRuntime = useComposerRuntime(); const hasAutoFocusedRef = useRef(false); @@ -223,6 +226,18 @@ const Composer: FC = () => { const isThreadEmpty = useAssistantState(({ thread }) => thread.isEmpty); const isThreadRunning = useAssistantState(({ thread }) => thread.isRunning); + // Live collaboration: track AI responding state + const { data: currentUser } = useAtomValue(currentUserAtom); + const { data: members } = useAtomValue(membersAtom); + const threadId = useMemo(() => { + if (Array.isArray(chat_id) && chat_id.length > 0) { + return Number.parseInt(chat_id[0], 10) || null; + } + return typeof chat_id === "string" ? Number.parseInt(chat_id, 10) || null : null; + }, [chat_id]); + const { isAiResponding, respondingToUserId } = useChatSessionState(threadId); + const isBlockedByOtherUser = isAiResponding && respondingToUserId !== currentUser?.id; + // Auto-focus editor on new chat page after mount useEffect(() => { if (isThreadEmpty && !hasAutoFocusedRef.current && editorRef.current) { @@ -298,9 +313,9 @@ const Composer: FC = () => { [showDocumentPopover] ); - // Submit message (blocked during streaming or when document picker is open) + // Submit message (blocked during streaming, document picker open, or AI responding to another user) const handleSubmit = useCallback(() => { - if (isThreadRunning) { + if (isThreadRunning || isBlockedByOtherUser) { return; } if (!showDocumentPopover) { @@ -315,6 +330,7 @@ const Composer: FC = () => { }, [ showDocumentPopover, isThreadRunning, + isBlockedByOtherUser, composerRuntime, setMentionedDocuments, setMentionedDocumentIds, @@ -374,7 +390,13 @@ const Composer: FC = () => { ); return ( - + + {/* Inline editor with @mention support */} From 17f8c993df801043f96c8d2116b559a7d9249dcc Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 20 Jan 2026 19:48:28 +0200 Subject: [PATCH 10/44] Simplify chat session state hook and disable send button when blocked --- .../components/assistant-ui/thread.tsx | 29 ++++++++++++------- surfsense_web/hooks/use-chat-session-state.ts | 6 ++-- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/surfsense_web/components/assistant-ui/thread.tsx b/surfsense_web/components/assistant-ui/thread.tsx index 1d83b5a60..d65372c24 100644 --- a/surfsense_web/components/assistant-ui/thread.tsx +++ b/surfsense_web/components/assistant-ui/thread.tsx @@ -226,7 +226,7 @@ const Composer: FC = () => { const isThreadEmpty = useAssistantState(({ thread }) => thread.isEmpty); const isThreadRunning = useAssistantState(({ thread }) => thread.isRunning); - // Live collaboration: track AI responding state + // Live collaboration state const { data: currentUser } = useAtomValue(currentUserAtom); const { data: members } = useAtomValue(membersAtom); const threadId = useMemo(() => { @@ -439,13 +439,17 @@ const Composer: FC = () => { />, document.body )} - + ); }; -const ComposerAction: FC = () => { +interface ComposerActionProps { + isBlockedByOtherUser?: boolean; +} + +const ComposerAction: FC = ({ isBlockedByOtherUser = false }) => { // Check if any attachments are still being processed (running AND progress < 100) // When progress is 100, processing is done but waiting for send() const hasProcessingAttachments = useAssistantState(({ composer }) => @@ -480,7 +484,8 @@ const ComposerAction: FC = () => { return userConfigs?.some((c) => c.id === agentLlmId) ?? false; }, [preferences, globalConfigs, userConfigs]); - const isSendDisabled = hasProcessingAttachments || isComposerEmpty || !hasModelConfigured; + const isSendDisabled = + hasProcessingAttachments || isComposerEmpty || !hasModelConfigured || isBlockedByOtherUser; return (
@@ -509,13 +514,15 @@ const ComposerAction: FC = () => { ({ url: `${ELECTRIC_URL}/v1/shape`, @@ -12,8 +16,6 @@ export function useChatSessionState(threadId: number | null) { table: "chat_session_state", where: `thread_id = ${threadId}`, }, - // Skip fetching if no threadId - ...(threadId ? {} : { url: undefined as unknown as string }), }); const sessionState = data?.[0] ?? null; From 0245e4bea96074501815f262d167f9ed0d66dbc1 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 21 Jan 2026 15:17:23 +0200 Subject: [PATCH 11/44] Add Electric SQL replication for messages and comments --- ..._messages_comments_electric_replication.py | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 surfsense_backend/alembic/versions/74_add_messages_comments_electric_replication.py diff --git a/surfsense_backend/alembic/versions/74_add_messages_comments_electric_replication.py b/surfsense_backend/alembic/versions/74_add_messages_comments_electric_replication.py new file mode 100644 index 000000000..c06b4ed3d --- /dev/null +++ b/surfsense_backend/alembic/versions/74_add_messages_comments_electric_replication.py @@ -0,0 +1,94 @@ +"""Add new_chat_messages and chat_comments to Electric SQL publication + +Revision ID: 74 +Revises: 73 + +Enables real-time sync for chat messages and comments via Electric SQL. +""" + +from collections.abc import Sequence + +from alembic import op + +revision: str = "74" +down_revision: str | None = "73" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Add new_chat_messages and chat_comments to Electric SQL replication.""" + # Set REPLICA IDENTITY FULL for Electric SQL sync + op.execute("ALTER TABLE new_chat_messages REPLICA IDENTITY FULL;") + op.execute("ALTER TABLE chat_comments REPLICA IDENTITY FULL;") + + # Add new_chat_messages to Electric publication + op.execute( + """ + DO $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_publication_tables + WHERE pubname = 'electric_publication_default' + AND tablename = 'new_chat_messages' + ) THEN + ALTER PUBLICATION electric_publication_default ADD TABLE new_chat_messages; + END IF; + END + $$; + """ + ) + + # Add chat_comments to Electric publication + op.execute( + """ + DO $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_publication_tables + WHERE pubname = 'electric_publication_default' + AND tablename = 'chat_comments' + ) THEN + ALTER PUBLICATION electric_publication_default ADD TABLE chat_comments; + END IF; + END + $$; + """ + ) + + +def downgrade() -> None: + """Remove new_chat_messages and chat_comments from Electric SQL replication.""" + op.execute( + """ + DO $$ + BEGIN + IF EXISTS ( + SELECT 1 FROM pg_publication_tables + WHERE pubname = 'electric_publication_default' + AND tablename = 'new_chat_messages' + ) THEN + ALTER PUBLICATION electric_publication_default DROP TABLE new_chat_messages; + END IF; + END + $$; + """ + ) + + op.execute( + """ + DO $$ + BEGIN + IF EXISTS ( + SELECT 1 FROM pg_publication_tables + WHERE pubname = 'electric_publication_default' + AND tablename = 'chat_comments' + ) THEN + ALTER PUBLICATION electric_publication_default DROP TABLE chat_comments; + END IF; + END + $$; + """ + ) + + # Note: Not reverting REPLICA IDENTITY as it doesn't harm normal operations From 3765d0a868cecd832a060ebe5c57ee1ca1da7666 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 21 Jan 2026 17:57:11 +0200 Subject: [PATCH 12/44] Update migration 74 for live collaboration tables --- ...ve_collaboration_tables_electric_replication.py} | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) rename surfsense_backend/alembic/versions/{74_add_messages_comments_electric_replication.py => 74_add_live_collaboration_tables_electric_replication.py} (82%) diff --git a/surfsense_backend/alembic/versions/74_add_messages_comments_electric_replication.py b/surfsense_backend/alembic/versions/74_add_live_collaboration_tables_electric_replication.py similarity index 82% rename from surfsense_backend/alembic/versions/74_add_messages_comments_electric_replication.py rename to surfsense_backend/alembic/versions/74_add_live_collaboration_tables_electric_replication.py index c06b4ed3d..c98ddd332 100644 --- a/surfsense_backend/alembic/versions/74_add_messages_comments_electric_replication.py +++ b/surfsense_backend/alembic/versions/74_add_live_collaboration_tables_electric_replication.py @@ -1,9 +1,14 @@ -"""Add new_chat_messages and chat_comments to Electric SQL publication +"""Add live collaboration tables to Electric SQL publication Revision ID: 74 Revises: 73 -Enables real-time sync for chat messages and comments via Electric SQL. +Enables real-time sync for live collaboration features: +- new_chat_messages: Live message sync between users +- chat_comments: Live comment updates + +Note: User/member info is fetched via API (membersAtom) for client-side joins, +not via Electric SQL, to keep where clauses optimized and reduce complexity. """ from collections.abc import Sequence @@ -17,7 +22,7 @@ depends_on: str | Sequence[str] | None = None def upgrade() -> None: - """Add new_chat_messages and chat_comments to Electric SQL replication.""" + """Add live collaboration tables to Electric SQL replication.""" # Set REPLICA IDENTITY FULL for Electric SQL sync op.execute("ALTER TABLE new_chat_messages REPLICA IDENTITY FULL;") op.execute("ALTER TABLE chat_comments REPLICA IDENTITY FULL;") @@ -58,7 +63,7 @@ def upgrade() -> None: def downgrade() -> None: - """Remove new_chat_messages and chat_comments from Electric SQL replication.""" + """Remove live collaboration tables from Electric SQL replication.""" op.execute( """ DO $$ From 73ff26119434f045da174b9645a53ce1aa02ba2c Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 21 Jan 2026 17:58:17 +0200 Subject: [PATCH 13/44] Add raw message/comment types and reduce members stale time --- .../atoms/members/members-query.atoms.ts | 2 +- .../contracts/types/chat-comments.types.ts | 14 ++++++++++++++ .../contracts/types/chat-messages.types.ts | 16 ++++++++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 surfsense_web/contracts/types/chat-messages.types.ts diff --git a/surfsense_web/atoms/members/members-query.atoms.ts b/surfsense_web/atoms/members/members-query.atoms.ts index 8ed56ef0c..f486dc02b 100644 --- a/surfsense_web/atoms/members/members-query.atoms.ts +++ b/surfsense_web/atoms/members/members-query.atoms.ts @@ -9,7 +9,7 @@ export const membersAtom = atomWithQuery((get) => { return { queryKey: cacheKeys.members.all(searchSpaceId?.toString() ?? ""), enabled: !!searchSpaceId, - staleTime: 5 * 60 * 1000, // 5 minutes + staleTime: 3 * 1000, // 3 seconds - short staleness for live collaboration queryFn: async () => { if (!searchSpaceId) { return []; diff --git a/surfsense_web/contracts/types/chat-comments.types.ts b/surfsense_web/contracts/types/chat-comments.types.ts index 92b3ff060..c3b32be9d 100644 --- a/surfsense_web/contracts/types/chat-comments.types.ts +++ b/surfsense_web/contracts/types/chat-comments.types.ts @@ -1,5 +1,18 @@ import { z } from "zod"; +/** + * Raw comment + */ +export const rawComment = z.object({ + id: z.number(), + message_id: z.number(), + parent_id: z.number().nullable(), + author_id: z.string().nullable(), + content: z.string(), + created_at: z.string(), + updated_at: z.string(), +}); + export const author = z.object({ id: z.string().uuid(), display_name: z.string().nullable(), @@ -122,6 +135,7 @@ export const getMentionsResponse = z.object({ total_count: z.number(), }); +export type RawComment = z.infer; export type Author = z.infer; export type CommentReply = z.infer; export type Comment = z.infer; diff --git a/surfsense_web/contracts/types/chat-messages.types.ts b/surfsense_web/contracts/types/chat-messages.types.ts new file mode 100644 index 000000000..01edef3f2 --- /dev/null +++ b/surfsense_web/contracts/types/chat-messages.types.ts @@ -0,0 +1,16 @@ +import { z } from "zod"; + +/** + * Raw message from database (Electric SQL sync) + */ +export const rawMessage = z.object({ + id: z.number(), + thread_id: z.number(), + role: z.enum(["user", "assistant", "system"]), + content: z.unknown(), + author_id: z.string().nullable(), + created_at: z.string(), + updated_at: z.string(), +}); + +export type RawMessage = z.infer; From dd781fa3d531a7cb19b2f63b4980a0ecc418ea11 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 21 Jan 2026 17:58:35 +0200 Subject: [PATCH 14/44] Add live sync hooks for messages and comments --- surfsense_web/hooks/use-chat-messages-live.ts | 77 +++++++++ surfsense_web/hooks/use-comments-live.ts | 154 ++++++++++++++++++ 2 files changed, 231 insertions(+) create mode 100644 surfsense_web/hooks/use-chat-messages-live.ts create mode 100644 surfsense_web/hooks/use-comments-live.ts diff --git a/surfsense_web/hooks/use-chat-messages-live.ts b/surfsense_web/hooks/use-chat-messages-live.ts new file mode 100644 index 000000000..39341d479 --- /dev/null +++ b/surfsense_web/hooks/use-chat-messages-live.ts @@ -0,0 +1,77 @@ +"use client"; + +import { useShape } from "@electric-sql/react"; +import { useAtomValue } from "jotai"; +import { useMemo } from "react"; +import { membersAtom } from "@/atoms/members/members-query.atoms"; +import type { RawMessage } from "@/contracts/types/chat-messages.types"; +import type { Membership } from "@/contracts/types/members.types"; +import type { MessageRecord } from "@/lib/chat/thread-persistence"; + +const ELECTRIC_URL = process.env.NEXT_PUBLIC_ELECTRIC_URL || "http://localhost:5133"; + +/** + * Member info for building author data - derived from Membership + */ +type MemberInfo = Pick; + +/** + * Hook to get live chat messages for real-time sync. + * Uses Electric SQL for messages + membersAtom (API) for author info. + */ +export function useChatMessagesLive(threadId: number | null) { + + const { + data: messagesData, + isLoading: messagesLoading, + isError: messagesError, + error: messagesErrorDetails, + } = useShape({ + url: `${ELECTRIC_URL}/v1/shape`, + params: { + table: "new_chat_messages", + where: `thread_id = ${threadId}`, + }, + }); + + + const { data: membersData, isLoading: membersLoading } = useAtomValue(membersAtom); + + + const messages = useMemo(() => { + if (!messagesData) return []; + + // Build member lookup map + const memberMap = new Map(); + if (membersData) { + for (const member of membersData) { + memberMap.set(member.user_id, { + user_display_name: member.user_display_name, + user_avatar_url: member.user_avatar_url, + }); + } + } + + // Transform raw messages to MessageRecord with author info + return [...messagesData].map((msg): MessageRecord => { + const author = msg.author_id ? memberMap.get(msg.author_id) : null; + return { + id: msg.id, + thread_id: msg.thread_id, + role: msg.role, + content: msg.content, + created_at: msg.created_at, + author_id: msg.author_id, + author_display_name: author?.user_display_name ?? null, + author_avatar_url: author?.user_avatar_url ?? null, + }; + }); + }, [messagesData, membersData]); + + return { + messages, + isLoading: messagesLoading || membersLoading, + isError: messagesError, + error: messagesError ? messagesErrorDetails : null, + }; +} diff --git a/surfsense_web/hooks/use-comments-live.ts b/surfsense_web/hooks/use-comments-live.ts new file mode 100644 index 000000000..f4d922888 --- /dev/null +++ b/surfsense_web/hooks/use-comments-live.ts @@ -0,0 +1,154 @@ +"use client"; + +import { useShape } from "@electric-sql/react"; +import { useAtomValue } from "jotai"; +import { useMemo } from "react"; +import { membersAtom, myAccessAtom } from "@/atoms/members/members-query.atoms"; +import { currentUserAtom } from "@/atoms/user/user-query.atoms"; +import type { Comment, CommentReply, Author } from "@/contracts/types/chat-comments.types"; +import type { Membership } from "@/contracts/types/members.types"; +import type { RawComment } from "@/contracts/types/chat-comments.types"; + +const ELECTRIC_URL = process.env.NEXT_PUBLIC_ELECTRIC_URL || "http://localhost:5133"; + +// Regex pattern to match @[uuid] mentions (matches backend MENTION_PATTERN) +const MENTION_PATTERN = /@\[([0-9a-fA-F-]{36})\]/g; + +/** + * Member info for building author objects - derived from Membership + */ +type MemberInfo = Pick; + +/** + * Render mentions in content by replacing @[uuid] with @{DisplayName} + */ +function renderMentions(content: string, memberMap: Map): string { + return content.replace(MENTION_PATTERN, (match, uuid) => { + const member = memberMap.get(uuid); + if (member?.user_display_name) { + return `@{${member.user_display_name}}`; + } + return match; + }); +} + +/** + * Hook to get live comments for a specific message. + * Uses Electric SQL for comments + membersAtom (API) for author info. + * Returns data matching the existing Comment type. + */ +export function useCommentsLive(messageId: number | null) { + const { + data: commentsData, + isLoading: commentsLoading, + isError: commentsError, + error: commentsErrorDetails, + } = useShape({ + url: `${ELECTRIC_URL}/v1/shape`, + params: { + table: "chat_comments", + where: `message_id = ${messageId}`, + }, + }); + + const { data: membersData, isLoading: membersLoading } = useAtomValue(membersAtom); + const { data: currentUser } = useAtomValue(currentUserAtom); + const { data: myAccess } = useAtomValue(myAccessAtom); + + const comments = useMemo(() => { + if (!commentsData) return []; + + // Build member lookup map + const memberMap = new Map(); + if (membersData) { + for (const member of membersData) { + memberMap.set(member.user_id, { + user_display_name: member.user_display_name, + user_avatar_url: member.user_avatar_url, + user_email: member.user_email, + }); + } + } + + const currentUserId = currentUser?.id; + const isOwnerOrAdmin = myAccess?.is_owner ?? false; + + // Build author object from member data + const buildAuthor = (authorId: string | null): Author | null => { + if (!authorId) return null; + const member = memberMap.get(authorId); + if (!member) return null; + return { + id: authorId, + display_name: member.user_display_name ?? null, + avatar_url: member.user_avatar_url ?? null, + email: member.user_email ?? "", + }; + }; + + // Transform raw comment to CommentReply + const transformToReply = (raw: RawComment): CommentReply => { + const isEdited = raw.created_at !== raw.updated_at; + const isAuthor = currentUserId === raw.author_id; + + return { + id: raw.id, + content: raw.content, + content_rendered: renderMentions(raw.content, memberMap), + author: buildAuthor(raw.author_id), + created_at: raw.created_at, + updated_at: raw.updated_at, + is_edited: isEdited, + can_edit: isAuthor, + can_delete: isAuthor || isOwnerOrAdmin, + }; + }; + + // Separate top-level comments and replies + const topLevelRaw: RawComment[] = []; + const repliesMap = new Map(); + + for (const raw of commentsData) { + if (raw.parent_id === null) { + topLevelRaw.push(raw); + } else { + const replies = repliesMap.get(raw.parent_id) || []; + replies.push(raw); + repliesMap.set(raw.parent_id, replies); + } + } + + // Transform top-level comments to Comment type + const transformToComment = (raw: RawComment): Comment => { + const isEdited = raw.created_at !== raw.updated_at; + const isAuthor = currentUserId === raw.author_id; + const rawReplies = repliesMap.get(raw.id) || []; + const replies = rawReplies.map(transformToReply); + + return { + id: raw.id, + message_id: raw.message_id, + content: raw.content, + content_rendered: renderMentions(raw.content, memberMap), + author: buildAuthor(raw.author_id), + created_at: raw.created_at, + updated_at: raw.updated_at, + is_edited: isEdited, + can_edit: isAuthor, + can_delete: isAuthor || isOwnerOrAdmin, + reply_count: replies.length, + replies, + }; + }; + + return topLevelRaw.map(transformToComment); + }, [commentsData, membersData, currentUser?.id, myAccess?.is_owner]); + + return { + comments, + commentCount: commentsData?.length ?? 0, + isLoading: commentsLoading || membersLoading, + isError: commentsError, + error: commentsError ? commentsErrorDetails : null, + }; +} From f7ee03ddf370110dd42614d545d3b3cf05b797cb Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 21 Jan 2026 18:10:03 +0200 Subject: [PATCH 15/44] Integrate live message sync on chat page --- .../new-chat/[[...chat_id]]/page.tsx | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx index 43c33ba5a..865dd6e68 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx @@ -33,6 +33,7 @@ import { GeneratePodcastToolUI } from "@/components/tool-ui/generate-podcast"; import { LinkPreviewToolUI } from "@/components/tool-ui/link-preview"; import { ScrapeWebpageToolUI } from "@/components/tool-ui/scrape-webpage"; // import { WriteTodosToolUI } from "@/components/tool-ui/write-todos"; +import { useChatMessagesLive } from "@/hooks/use-chat-messages-live"; import { getBearerToken } from "@/lib/auth-utils"; import { createAttachmentAdapter, extractAttachmentContent } from "@/lib/chat/attachment-adapter"; import { @@ -257,6 +258,9 @@ export default function NewChatPage() { // Get current user for author info in shared chats const { data: currentUser } = useAtomValue(currentUserAtom); + // Live sync for other users' messages (shared chats) + const { messages: liveMessages } = useChatMessagesLive(threadId); + // Create the attachment adapter for file processing const attachmentAdapter = useMemo(() => createAttachmentAdapter(), []); @@ -409,6 +413,45 @@ export default function NewChatPage() { }); }, [currentThread, setCurrentThreadState]); + // Live sync: Merge messages from other users (shared chats) + useEffect(() => { + if (!liveMessages.length || !currentUser?.id || isRunning) return; + + // Get IDs of messages we already have locally + const localMessageIds = new Set( + messages + .map((m) => { + // Extract numeric ID from "msg-{id}" format + const match = m.id?.match(/^msg-(\d+)$/); + return match ? Number.parseInt(match[1], 10) : null; + }) + .filter((id): id is number => id !== null) + ); + + // Find live messages from OTHER users that we don't have locally + const newOtherUserMessages = liveMessages.filter((liveMsg) => { + // Skip if we already have this message + if (localMessageIds.has(liveMsg.id)) return false; + // Skip if this is our own message (we added it optimistically) + if (liveMsg.author_id === currentUser.id) return false; + return true; + }); + + if (newOtherUserMessages.length > 0) { + // Convert and add new messages from other users + const converted = newOtherUserMessages.map(convertToThreadMessage); + setMessages((prev) => { + // Merge and sort by ID to maintain order + const merged = [...prev, ...converted]; + return merged.sort((a, b) => { + const aId = Number.parseInt((a.id ?? "").replace(/^msg-/, ""), 10) || 0; + const bId = Number.parseInt((b.id ?? "").replace(/^msg-/, ""), 10) || 0; + return aId - bId; + }); + }); + } + }, [liveMessages, currentUser?.id, messages, isRunning]); + // Cancel ongoing request const cancelRun = useCallback(async () => { if (abortControllerRef.current) { From f3467262940e956e9e443dc944901e763ff77a26 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Wed, 21 Jan 2026 18:13:02 +0200 Subject: [PATCH 16/44] Integrate live comment sync in comment components --- .../components/assistant-ui/assistant-message.tsx | 10 +++------- .../comment-panel-container.tsx | 15 +++++++-------- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/surfsense_web/components/assistant-ui/assistant-message.tsx b/surfsense_web/components/assistant-ui/assistant-message.tsx index 681dc315a..513242d1b 100644 --- a/surfsense_web/components/assistant-ui/assistant-message.tsx +++ b/surfsense_web/components/assistant-ui/assistant-message.tsx @@ -25,7 +25,7 @@ import { TooltipIconButton } from "@/components/assistant-ui/tooltip-icon-button import { CommentPanelContainer } from "@/components/chat-comments/comment-panel-container/comment-panel-container"; import { CommentSheet } from "@/components/chat-comments/comment-sheet/comment-sheet"; import { CommentTrigger } from "@/components/chat-comments/comment-trigger/comment-trigger"; -import { useComments } from "@/hooks/use-comments"; +import { useCommentsLive } from "@/hooks/use-comments-live"; import { useMediaQuery } from "@/hooks/use-media-query"; import { cn } from "@/lib/utils"; @@ -115,12 +115,8 @@ export const AssistantMessage: FC = () => { const isLastMessage = useAssistantState(({ message }) => message?.isLast ?? false); const isMessageStreaming = isThreadRunning && isLastMessage; - const { data: commentsData } = useComments({ - messageId: dbMessageId ?? 0, - enabled: !!dbMessageId, - }); - - const commentCount = commentsData?.total_count ?? 0; + // Live sync for real-time comment count + const { commentCount } = useCommentsLive(dbMessageId); const hasComments = commentCount > 0; const isAddingComment = dbMessageId !== null && addingCommentToMessageId === dbMessageId; const showCommentPanel = hasComments || isAddingComment; diff --git a/surfsense_web/components/chat-comments/comment-panel-container/comment-panel-container.tsx b/surfsense_web/components/chat-comments/comment-panel-container/comment-panel-container.tsx index 197ac0798..8281ca8fd 100644 --- a/surfsense_web/components/chat-comments/comment-panel-container/comment-panel-container.tsx +++ b/surfsense_web/components/chat-comments/comment-panel-container/comment-panel-container.tsx @@ -10,7 +10,7 @@ import { } from "@/atoms/chat-comments/comments-mutation.atoms"; import { membersAtom } from "@/atoms/members/members-query.atoms"; import { currentUserAtom } from "@/atoms/user/user-query.atoms"; -import { useComments } from "@/hooks/use-comments"; +import { useCommentsLive } from "@/hooks/use-comments-live"; import { CommentPanel } from "../comment-panel/comment-panel"; import type { CommentPanelContainerProps } from "./types"; import { transformComment, transformMember } from "./utils"; @@ -21,10 +21,10 @@ export function CommentPanelContainer({ maxHeight, variant = "desktop", }: CommentPanelContainerProps) { - const { data: commentsData, isLoading: isCommentsLoading } = useComments({ - messageId, - enabled: isOpen, - }); + // Live sync for real-time comment updates + const { comments: liveComments, isLoading: isCommentsLoading } = useCommentsLive( + isOpen ? messageId : null + ); const [{ data: membersData, isLoading: isMembersLoading }] = useAtom(membersAtom); const [{ data: currentUser }] = useAtom(currentUserAtom); @@ -35,9 +35,8 @@ export function CommentPanelContainer({ const [{ mutate: deleteComment, isPending: isDeleting }] = useAtom(deleteCommentMutationAtom); const commentThreads = useMemo(() => { - if (!commentsData?.comments) return []; - return commentsData.comments.map(transformComment); - }, [commentsData]); + return liveComments.map(transformComment); + }, [liveComments]); const members = useMemo(() => { if (!membersData) return []; From 8a83424be54ac46aa63bd8f114a0059a18932c4b Mon Sep 17 00:00:00 2001 From: Manoj Aggarwal Date: Wed, 21 Jan 2026 15:21:06 -0800 Subject: [PATCH 17/44] Add support for obsidian to surfsense --- .../versions/74_add_obsidian_connector.py | 33 ++ surfsense_backend/app/config/__init__.py | 15 + surfsense_backend/app/db.py | 2 + .../routes/search_source_connectors_routes.py | 73 +++ .../app/schemas/obsidian_auth_credentials.py | 59 +++ .../app/tasks/celery_tasks/connector_tasks.py | 43 ++ .../app/tasks/connector_indexers/__init__.py | 2 + .../connector_indexers/obsidian_indexer.py | 489 ++++++++++++++++++ .../components/obsidian-connect-form.tsx | 448 ++++++++++++++++ .../connect-forms/connector-benefits.ts | 8 + .../connector-popup/connect-forms/index.tsx | 3 + .../components/obsidian-config.tsx | 191 +++++++ .../connector-configs/index.tsx | 3 + .../constants/connector-constants.ts | 7 + surfsense_web/contracts/enums/connector.ts | 1 + .../contracts/enums/connectorIcons.tsx | 2 + surfsense_web/public/connectors/obsidian.svg | 12 + 17 files changed, 1391 insertions(+) create mode 100644 surfsense_backend/alembic/versions/74_add_obsidian_connector.py create mode 100644 surfsense_backend/app/schemas/obsidian_auth_credentials.py create mode 100644 surfsense_backend/app/tasks/connector_indexers/obsidian_indexer.py create mode 100644 surfsense_web/components/assistant-ui/connector-popup/connect-forms/components/obsidian-connect-form.tsx create mode 100644 surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/obsidian-config.tsx create mode 100644 surfsense_web/public/connectors/obsidian.svg diff --git a/surfsense_backend/alembic/versions/74_add_obsidian_connector.py b/surfsense_backend/alembic/versions/74_add_obsidian_connector.py new file mode 100644 index 000000000..b34cd07a0 --- /dev/null +++ b/surfsense_backend/alembic/versions/74_add_obsidian_connector.py @@ -0,0 +1,33 @@ +"""Add Obsidian connector enums + +Revision ID: 74_add_obsidian_connector +Revises: 73_add_user_memories_table +Create Date: 2026-01-21 + +""" + +from collections.abc import Sequence + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "74_add_obsidian_connector" +down_revision: str | None = "73_add_user_memories_table" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + # Add OBSIDIAN_CONNECTOR to documenttype enum + op.execute("ALTER TYPE documenttype ADD VALUE IF NOT EXISTS 'OBSIDIAN_CONNECTOR'") + + # Add OBSIDIAN_CONNECTOR to searchsourceconnectortype enum + op.execute( + "ALTER TYPE searchsourceconnectortype ADD VALUE IF NOT EXISTS 'OBSIDIAN_CONNECTOR'" + ) + + +def downgrade() -> None: + # Note: PostgreSQL doesn't support removing enum values directly. + # The values will remain in the enum type but won't be used. + pass diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index 448e2c253..a59b23391 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -61,6 +61,21 @@ class Config: "FFmpeg is not installed on the system. Please install it to use the Surfsense Podcaster." ) + # Deployment Mode (self-hosted or cloud) + # self-hosted: Full access to local file system connectors (Obsidian, etc.) + # cloud: Only cloud-based connectors available + DEPLOYMENT_MODE = os.getenv("SURFSENSE_DEPLOYMENT_MODE", "self-hosted") + + @classmethod + def is_self_hosted(cls) -> bool: + """Check if running in self-hosted mode.""" + return cls.DEPLOYMENT_MODE == "self-hosted" + + @classmethod + def is_cloud(cls) -> bool: + """Check if running in cloud mode.""" + return cls.DEPLOYMENT_MODE == "cloud" + # Database DATABASE_URL = os.getenv("DATABASE_URL") diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py index c56d19355..723aaaf8b 100644 --- a/surfsense_backend/app/db.py +++ b/surfsense_backend/app/db.py @@ -53,6 +53,7 @@ class DocumentType(str, Enum): ELASTICSEARCH_CONNECTOR = "ELASTICSEARCH_CONNECTOR" BOOKSTACK_CONNECTOR = "BOOKSTACK_CONNECTOR" CIRCLEBACK = "CIRCLEBACK" + OBSIDIAN_CONNECTOR = "OBSIDIAN_CONNECTOR" NOTE = "NOTE" @@ -80,6 +81,7 @@ class SearchSourceConnectorType(str, Enum): WEBCRAWLER_CONNECTOR = "WEBCRAWLER_CONNECTOR" BOOKSTACK_CONNECTOR = "BOOKSTACK_CONNECTOR" CIRCLEBACK_CONNECTOR = "CIRCLEBACK_CONNECTOR" + OBSIDIAN_CONNECTOR = "OBSIDIAN_CONNECTOR" # Self-hosted only - Local Obsidian vault indexing MCP_CONNECTOR = "MCP_CONNECTOR" # Model Context Protocol - User-defined API tools diff --git a/surfsense_backend/app/routes/search_source_connectors_routes.py b/surfsense_backend/app/routes/search_source_connectors_routes.py index f6319653f..4fb8698ec 100644 --- a/surfsense_backend/app/routes/search_source_connectors_routes.py +++ b/surfsense_backend/app/routes/search_source_connectors_routes.py @@ -868,6 +868,25 @@ async def index_connector_content( ) response_message = "Web page indexing started in the background." + elif connector.connector_type == SearchSourceConnectorType.OBSIDIAN_CONNECTOR: + from app.config import config as app_config + from app.tasks.celery_tasks.connector_tasks import index_obsidian_vault_task + + # Obsidian connector only available in self-hosted mode + if not app_config.is_self_hosted(): + raise HTTPException( + status_code=400, + detail="Obsidian connector is only available in self-hosted mode", + ) + + logger.info( + f"Triggering Obsidian vault indexing for connector {connector_id} into search space {search_space_id} from {indexing_from} to {indexing_to}" + ) + index_obsidian_vault_task.delay( + connector_id, search_space_id, str(user.id), indexing_from, indexing_to + ) + response_message = "Obsidian vault indexing started in the background." + else: raise HTTPException( status_code=400, @@ -2073,6 +2092,60 @@ async def run_bookstack_indexing( ) +# Add new helper functions for Obsidian indexing +async def run_obsidian_indexing_with_new_session( + connector_id: int, + search_space_id: int, + user_id: str, + start_date: str, + end_date: str, +): + """Wrapper to run Obsidian indexing with its own database session.""" + logger.info( + f"Background task started: Indexing Obsidian connector {connector_id} into space {search_space_id} from {start_date} to {end_date}" + ) + async with async_session_maker() as session: + await run_obsidian_indexing( + session, connector_id, search_space_id, user_id, start_date, end_date + ) + logger.info( + f"Background task finished: Indexing Obsidian connector {connector_id}" + ) + + +async def run_obsidian_indexing( + session: AsyncSession, + connector_id: int, + search_space_id: int, + user_id: str, + start_date: str, + end_date: str, +): + """ + Background task to run Obsidian vault indexing. + + Args: + session: Database session + connector_id: ID of the Obsidian connector + search_space_id: ID of the search space + user_id: ID of the user + start_date: Start date for indexing + end_date: End date for indexing + """ + from app.tasks.connector_indexers import index_obsidian_vault + + await _run_indexing_with_notifications( + session=session, + connector_id=connector_id, + search_space_id=search_space_id, + user_id=user_id, + start_date=start_date, + end_date=end_date, + indexing_function=index_obsidian_vault, + update_timestamp_func=_update_connector_timestamp_by_id, + ) + + # ============================================================================= # MCP Connector Routes # ============================================================================= diff --git a/surfsense_backend/app/schemas/obsidian_auth_credentials.py b/surfsense_backend/app/schemas/obsidian_auth_credentials.py new file mode 100644 index 000000000..ab178eac8 --- /dev/null +++ b/surfsense_backend/app/schemas/obsidian_auth_credentials.py @@ -0,0 +1,59 @@ +""" +Obsidian Connector Credentials Schema. + +Obsidian is a local-first note-taking app that stores notes as markdown files. +This connector supports indexing from local file system (self-hosted only). +""" + +from pydantic import BaseModel, field_validator + + +class ObsidianAuthCredentialsBase(BaseModel): + """ + Credentials/configuration for the Obsidian connector. + + Since Obsidian vaults are local directories, this schema primarily + holds the vault path and configuration options rather than API tokens. + """ + + vault_path: str + vault_name: str | None = None + exclude_folders: list[str] | None = None + include_attachments: bool = False + + @field_validator("vault_path") + @classmethod + def validate_vault_path(cls, v: str) -> str: + """Ensure vault path is provided and stripped of whitespace.""" + if not v or not v.strip(): + raise ValueError("Vault path is required") + return v.strip() + + @field_validator("exclude_folders", mode="before") + @classmethod + def parse_exclude_folders(cls, v): + """Parse exclude_folders from string if needed.""" + if v is None: + return [".trash", ".obsidian", "templates"] + if isinstance(v, str): + return [f.strip() for f in v.split(",") if f.strip()] + return v + + def to_dict(self) -> dict: + """Convert credentials to dictionary for storage.""" + return { + "vault_path": self.vault_path, + "vault_name": self.vault_name, + "exclude_folders": self.exclude_folders, + "include_attachments": self.include_attachments, + } + + @classmethod + def from_dict(cls, data: dict) -> "ObsidianAuthCredentialsBase": + """Create credentials from dictionary.""" + return cls( + vault_path=data.get("vault_path", ""), + vault_name=data.get("vault_name"), + exclude_folders=data.get("exclude_folders"), + include_attachments=data.get("include_attachments", False), + ) diff --git a/surfsense_backend/app/tasks/celery_tasks/connector_tasks.py b/surfsense_backend/app/tasks/celery_tasks/connector_tasks.py index 78fa595de..b44d7c9a4 100644 --- a/surfsense_backend/app/tasks/celery_tasks/connector_tasks.py +++ b/surfsense_backend/app/tasks/celery_tasks/connector_tasks.py @@ -759,3 +759,46 @@ async def _index_bookstack_pages( await run_bookstack_indexing( session, connector_id, search_space_id, user_id, start_date, end_date ) + + +@celery_app.task(name="index_obsidian_vault", bind=True) +def index_obsidian_vault_task( + self, + connector_id: int, + search_space_id: int, + user_id: str, + start_date: str, + end_date: str, +): + """Celery task to index Obsidian vault notes.""" + import asyncio + + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + try: + loop.run_until_complete( + _index_obsidian_vault( + connector_id, search_space_id, user_id, start_date, end_date + ) + ) + finally: + loop.close() + + +async def _index_obsidian_vault( + connector_id: int, + search_space_id: int, + user_id: str, + start_date: str, + end_date: str, +): + """Index Obsidian vault with new session.""" + from app.routes.search_source_connectors_routes import ( + run_obsidian_indexing, + ) + + async with get_celery_session_maker()() as session: + await run_obsidian_indexing( + session, connector_id, search_space_id, user_id, start_date, end_date + ) diff --git a/surfsense_backend/app/tasks/connector_indexers/__init__.py b/surfsense_backend/app/tasks/connector_indexers/__init__.py index 95e57ddf2..d1f89f333 100644 --- a/surfsense_backend/app/tasks/connector_indexers/__init__.py +++ b/surfsense_backend/app/tasks/connector_indexers/__init__.py @@ -44,6 +44,7 @@ from .luma_indexer import index_luma_events # Documentation and knowledge management from .notion_indexer import index_notion_pages +from .obsidian_indexer import index_obsidian_vault from .slack_indexer import index_slack_messages from .webcrawler_indexer import index_crawled_urls @@ -65,6 +66,7 @@ __all__ = [ # noqa: RUF022 "index_linear_issues", # Documentation and knowledge management "index_notion_pages", + "index_obsidian_vault", "index_crawled_urls", # Communication platforms "index_slack_messages", diff --git a/surfsense_backend/app/tasks/connector_indexers/obsidian_indexer.py b/surfsense_backend/app/tasks/connector_indexers/obsidian_indexer.py new file mode 100644 index 000000000..15c88938a --- /dev/null +++ b/surfsense_backend/app/tasks/connector_indexers/obsidian_indexer.py @@ -0,0 +1,489 @@ +""" +Obsidian connector indexer. + +Indexes markdown notes from a local Obsidian vault. +This connector is only available in self-hosted mode. +""" + +import os +import re +from datetime import datetime, UTC +from pathlib import Path + +import yaml +from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.ext.asyncio import AsyncSession + +from app.config import config +from app.db import Document, DocumentType, SearchSourceConnectorType +from app.services.llm_service import get_user_long_context_llm +from app.services.task_logging_service import TaskLoggingService +from app.utils.document_converters import ( + create_document_chunks, + generate_content_hash, + generate_document_summary, + generate_unique_identifier_hash, +) + +from .base import ( + build_document_metadata_string, + check_document_by_unique_identifier, + get_connector_by_id, + get_current_timestamp, + logger, + update_connector_last_indexed, +) + + +def parse_frontmatter(content: str) -> tuple[dict | None, str]: + """ + Parse YAML frontmatter from markdown content. + + Args: + content: The full markdown content + + Returns: + Tuple of (frontmatter dict or None, content without frontmatter) + """ + if not content.startswith("---"): + return None, content + + # Find the closing --- + end_match = re.search(r"\n---\n", content[3:]) + if not end_match: + return None, content + + frontmatter_str = content[3 : end_match.start() + 3] + remaining_content = content[end_match.end() + 3 :] + + try: + frontmatter = yaml.safe_load(frontmatter_str) + return frontmatter, remaining_content.strip() + except yaml.YAMLError: + return None, content + + +def extract_wiki_links(content: str) -> list[str]: + """ + Extract [[wiki-style links]] from content. + + Args: + content: Markdown content + + Returns: + List of linked note names + """ + # Match [[link]] or [[link|alias]] + pattern = r"\[\[([^\]|]+)(?:\|[^\]]+)?\]\]" + matches = re.findall(pattern, content) + return list(set(matches)) + + +def extract_tags(content: str) -> list[str]: + """ + Extract #tags from content (both inline and frontmatter). + + Args: + content: Markdown content + + Returns: + List of tags (without # prefix) + """ + # Match #tag but not ## headers + pattern = r"(? list[dict]: + """ + Scan an Obsidian vault for markdown files. + + Args: + vault_path: Path to the Obsidian vault + exclude_folders: List of folder names to exclude + + Returns: + List of file info dicts with path, name, modified time + """ + if exclude_folders is None: + exclude_folders = [".trash", ".obsidian", "templates"] + + vault = Path(vault_path) + if not vault.exists(): + raise ValueError(f"Vault path does not exist: {vault_path}") + + files = [] + for md_file in vault.rglob("*.md"): + # Check if file is in an excluded folder + relative_path = md_file.relative_to(vault) + parts = relative_path.parts + + if any(excluded in parts for excluded in exclude_folders): + continue + + try: + stat = md_file.stat() + files.append( + { + "path": str(md_file), + "relative_path": str(relative_path), + "name": md_file.stem, + "modified_at": datetime.fromtimestamp(stat.st_mtime, tz=UTC), + "created_at": datetime.fromtimestamp(stat.st_ctime, tz=UTC), + "size": stat.st_size, + } + ) + except OSError as e: + logger.warning(f"Could not stat file {md_file}: {e}") + + return files + + +async def index_obsidian_vault( + session: AsyncSession, + connector_id: int, + search_space_id: int, + user_id: str, + start_date: str | None = None, + end_date: str | None = None, + update_last_indexed: bool = True, +) -> tuple[int, str | None]: + """ + Index notes from a local Obsidian vault. + + This indexer is only available in self-hosted mode as it requires + direct file system access to the user's Obsidian vault. + + Args: + session: Database session + connector_id: ID of the Obsidian connector + search_space_id: ID of the search space to store documents in + user_id: ID of the user + start_date: Start date for filtering (YYYY-MM-DD format) - optional + end_date: End date for filtering (YYYY-MM-DD format) - optional + update_last_indexed: Whether to update the last_indexed_at timestamp + + Returns: + Tuple containing (number of documents indexed, error message or None) + """ + task_logger = TaskLoggingService(session, search_space_id) + + # Check if self-hosted mode + if not config.is_self_hosted(): + return 0, "Obsidian connector is only available in self-hosted mode" + + # Log task start + log_entry = await task_logger.log_task_start( + task_name="obsidian_vault_indexing", + source="connector_indexing_task", + message=f"Starting Obsidian vault indexing for connector {connector_id}", + metadata={ + "connector_id": connector_id, + "user_id": str(user_id), + "start_date": start_date, + "end_date": end_date, + }, + ) + + try: + # Get the connector + await task_logger.log_task_progress( + log_entry, + f"Retrieving Obsidian connector {connector_id} from database", + {"stage": "connector_retrieval"}, + ) + + connector = await get_connector_by_id( + session, connector_id, SearchSourceConnectorType.OBSIDIAN_CONNECTOR + ) + + if not connector: + await task_logger.log_task_failure( + log_entry, + f"Connector with ID {connector_id} not found or is not an Obsidian connector", + "Connector not found", + {"error_type": "ConnectorNotFound"}, + ) + return ( + 0, + f"Connector with ID {connector_id} not found or is not an Obsidian connector", + ) + + # Get vault path from connector config + vault_path = connector.config.get("vault_path") + if not vault_path: + await task_logger.log_task_failure( + log_entry, + "Vault path not configured for this connector", + "Missing vault path", + {"error_type": "MissingVaultPath"}, + ) + return 0, "Vault path not configured for this connector" + + # Validate vault path exists + if not os.path.exists(vault_path): + await task_logger.log_task_failure( + log_entry, + f"Vault path does not exist: {vault_path}", + "Vault path not found", + {"error_type": "VaultNotFound", "vault_path": vault_path}, + ) + return 0, f"Vault path does not exist: {vault_path}" + + # Get configuration options + exclude_folders = connector.config.get( + "exclude_folders", [".trash", ".obsidian", "templates"] + ) + vault_name = connector.config.get("vault_name") or os.path.basename(vault_path) + + await task_logger.log_task_progress( + log_entry, + f"Scanning Obsidian vault: {vault_name}", + {"stage": "vault_scan", "vault_path": vault_path}, + ) + + # Scan vault for markdown files + try: + files = scan_vault(vault_path, exclude_folders) + except Exception as e: + await task_logger.log_task_failure( + log_entry, + f"Failed to scan vault: {e}", + "Vault scan error", + {"error_type": "VaultScanError"}, + ) + return 0, f"Failed to scan vault: {e}" + + logger.info(f"Found {len(files)} markdown files in vault") + + await task_logger.log_task_progress( + log_entry, + f"Found {len(files)} markdown files to process", + {"stage": "files_discovered", "file_count": len(files)}, + ) + + # Filter by date if provided + if start_date: + start_dt = datetime.strptime(start_date, "%Y-%m-%d").replace(tzinfo=UTC) + files = [f for f in files if f["modified_at"] >= start_dt] + + if end_date: + end_dt = datetime.strptime(end_date, "%Y-%m-%d").replace(tzinfo=UTC) + files = [f for f in files if f["modified_at"] <= end_dt] + + # Get LLM for summarization + long_context_llm = await get_user_long_context_llm(session, user_id) + + indexed_count = 0 + skipped_count = 0 + + for file_info in files: + try: + file_path = file_info["path"] + relative_path = file_info["relative_path"] + + # Read file content + try: + with open(file_path, encoding="utf-8") as f: + content = f.read() + except UnicodeDecodeError: + logger.warning(f"Could not decode file {file_path}, skipping") + skipped_count += 1 + continue + + if not content.strip(): + logger.debug(f"Empty file {file_path}, skipping") + skipped_count += 1 + continue + + # Parse frontmatter and extract metadata + frontmatter, body_content = parse_frontmatter(content) + wiki_links = extract_wiki_links(content) + tags = extract_tags(content) + + # Get title from frontmatter or filename + title = file_info["name"] + if frontmatter: + title = frontmatter.get("title", title) + # Also extract tags from frontmatter + fm_tags = frontmatter.get("tags", []) + if isinstance(fm_tags, list): + tags = list(set(tags + fm_tags)) + elif isinstance(fm_tags, str): + tags = list(set(tags + [fm_tags])) + + # Generate unique identifier using vault name and relative path + unique_identifier = f"{vault_name}:{relative_path}" + unique_identifier_hash = generate_unique_identifier_hash( + DocumentType.OBSIDIAN_CONNECTOR, + unique_identifier, + search_space_id, + ) + + # Check for existing document + existing_document = await check_document_by_unique_identifier( + session, unique_identifier_hash + ) + + # Generate content hash + content_hash = generate_content_hash(content) + + # Build metadata + document_metadata = { + "vault_name": vault_name, + "file_path": relative_path, + "tags": tags, + "outgoing_links": wiki_links, + "frontmatter": frontmatter, + "modified_at": file_info["modified_at"].isoformat(), + "created_at": file_info["created_at"].isoformat(), + "word_count": len(body_content.split()), + } + + # Build document content with metadata + metadata_sections = [ + ( + "METADATA", + [ + f"Title: {title}", + f"Vault: {vault_name}", + f"Path: {relative_path}", + f"Tags: {', '.join(tags) if tags else 'None'}", + f"Links to: {', '.join(wiki_links) if wiki_links else 'None'}", + ], + ), + ("CONTENT", [body_content]), + ] + document_string = build_document_metadata_string(metadata_sections) + + if existing_document: + # Check if content has changed + if existing_document.content_hash == content_hash: + logger.debug(f"Note {title} unchanged, skipping") + skipped_count += 1 + continue + + # Update existing document + logger.info(f"Updating note: {title}") + + # Generate new summary if content changed + if long_context_llm: + new_summary = await generate_document_summary( + content=document_string, + llm=long_context_llm, + ) + existing_document.summary = new_summary + + existing_document.content = document_string + existing_document.content_hash = content_hash + existing_document.document_metadata = document_metadata + existing_document.updated_at = get_current_timestamp() + + # Update embedding + embedding = config.embedding_model_instance.embed(document_string) + existing_document.embedding = embedding + + # Update chunks + await create_document_chunks( + session=session, + document=existing_document, + content=document_string, + chunker=config.chunker_instance, + embedding_model=config.embedding_model_instance, + ) + + indexed_count += 1 + + else: + # Create new document + logger.info(f"Indexing new note: {title}") + + # Generate summary + summary = "" + if long_context_llm: + summary = await generate_document_summary( + content=document_string, + llm=long_context_llm, + ) + + # Generate embedding + embedding = config.embedding_model_instance.embed(document_string) + + # Create document + new_document = Document( + search_space_id=search_space_id, + title=title, + url=f"obsidian://{vault_name}/{relative_path}", + document_type=DocumentType.OBSIDIAN_CONNECTOR, + content=document_string, + content_hash=content_hash, + unique_identifier_hash=unique_identifier_hash, + document_metadata=document_metadata, + summary=summary, + embedding=embedding, + connector_id=connector_id, + ) + + session.add(new_document) + await session.flush() + + # Create chunks + await create_document_chunks( + session=session, + document=new_document, + content=document_string, + chunker=config.chunker_instance, + embedding_model=config.embedding_model_instance, + ) + + indexed_count += 1 + + except Exception as e: + logger.exception( + f"Error processing file {file_info.get('path', 'unknown')}: {e}" + ) + skipped_count += 1 + continue + + # Update connector's last indexed timestamp + await update_connector_last_indexed(session, connector, update_last_indexed) + + # Commit all changes + await session.commit() + + await task_logger.log_task_success( + log_entry, + f"Successfully indexed {indexed_count} Obsidian notes (skipped {skipped_count})", + { + "indexed_count": indexed_count, + "skipped_count": skipped_count, + "total_files": len(files), + }, + ) + + return indexed_count, None + + except SQLAlchemyError as e: + logger.exception(f"Database error during Obsidian indexing: {e}") + await session.rollback() + await task_logger.log_task_failure( + log_entry, + f"Database error during Obsidian indexing: {e}", + "Database error", + {"error_type": "SQLAlchemyError"}, + ) + return 0, f"Database error: {e}" + + except Exception as e: + logger.exception(f"Error during Obsidian indexing: {e}") + await task_logger.log_task_failure( + log_entry, + f"Error during Obsidian indexing: {e}", + "Unexpected error", + {"error_type": type(e).__name__}, + ) + return 0, str(e) diff --git a/surfsense_web/components/assistant-ui/connector-popup/connect-forms/components/obsidian-connect-form.tsx b/surfsense_web/components/assistant-ui/connector-popup/connect-forms/components/obsidian-connect-form.tsx new file mode 100644 index 000000000..acdbc4c1f --- /dev/null +++ b/surfsense_web/components/assistant-ui/connector-popup/connect-forms/components/obsidian-connect-form.tsx @@ -0,0 +1,448 @@ +"use client"; + +import { zodResolver } from "@hookform/resolvers/zod"; +import { FolderOpen, Info } from "lucide-react"; +import type { FC } from "react"; +import { useRef, useState } from "react"; +import { useForm } from "react-hook-form"; +import * as z from "zod"; +import { + Accordion, + AccordionContent, + AccordionItem, + AccordionTrigger, +} from "@/components/ui/accordion"; +import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; +import { + Form, + FormControl, + FormDescription, + FormField, + FormItem, + FormLabel, + FormMessage, +} from "@/components/ui/form"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Switch } from "@/components/ui/switch"; +import { EnumConnectorName } from "@/contracts/enums/connector"; +import { getConnectorBenefits } from "../connector-benefits"; +import type { ConnectFormProps } from "../index"; + +const obsidianConnectorFormSchema = z.object({ + name: z.string().min(3, { + message: "Connector name must be at least 3 characters.", + }), + vault_path: z.string().min(1, { + message: "Vault path is required.", + }), + vault_name: z.string().min(1, { + message: "Vault name is required.", + }), + exclude_folders: z.string().optional(), + include_attachments: z.boolean(), +}); + +type ObsidianConnectorFormValues = z.infer; + +export const ObsidianConnectForm: FC = ({ onSubmit, isSubmitting }) => { + const isSubmittingRef = useRef(false); + const [periodicEnabled, setPeriodicEnabled] = useState(true); + const [frequencyMinutes, setFrequencyMinutes] = useState("60"); + const form = useForm({ + resolver: zodResolver(obsidianConnectorFormSchema), + defaultValues: { + name: "Obsidian Vault", + vault_path: "", + vault_name: "", + exclude_folders: ".obsidian,.trash", + include_attachments: false, + }, + }); + + const handleSubmit = async (values: ObsidianConnectorFormValues) => { + // Prevent multiple submissions + if (isSubmittingRef.current || isSubmitting) { + return; + } + + isSubmittingRef.current = true; + try { + // Parse exclude_folders into an array + const excludeFolders = values.exclude_folders + ? values.exclude_folders + .split(",") + .map((f) => f.trim()) + .filter(Boolean) + : [".obsidian", ".trash"]; + + await onSubmit({ + name: values.name, + connector_type: EnumConnectorName.OBSIDIAN_CONNECTOR, + config: { + vault_path: values.vault_path, + vault_name: values.vault_name, + exclude_folders: excludeFolders, + include_attachments: values.include_attachments, + }, + is_indexable: true, + is_active: true, + last_indexed_at: null, + periodic_indexing_enabled: periodicEnabled, + indexing_frequency_minutes: periodicEnabled ? Number.parseInt(frequencyMinutes, 10) : null, + next_scheduled_at: null, + periodicEnabled, + frequencyMinutes, + }); + } finally { + isSubmittingRef.current = false; + } + }; + + return ( +
+ + +
+ Self-Hosted Only + + This connector requires direct file system access and only works with self-hosted + SurfSense installations. + +
+ + +
+
+ + ( + + Connector Name + + + + + A friendly name to identify this connector. + + + + )} + /> + + ( + + Vault Path + + + + + The absolute path to your Obsidian vault on the server. This must be accessible + from the SurfSense backend. + + + + )} + /> + + ( + + Vault Name + + + + + A display name for your vault. This will be used in search results. + + + + )} + /> + + ( + + Exclude Folders + + + + + Comma-separated list of folder names to exclude from indexing. + + + + )} + /> + + ( + +
+ Include Attachments + + Index attachment folders and embedded files (images, PDFs, etc.) + +
+ + + + + )} + /> + + {/* Indexing Configuration */} +
+

Indexing Configuration

+ + {/* Periodic Sync Config */} +
+
+
+

Enable Periodic Sync

+

+ Automatically re-index at regular intervals +

+
+ +
+ + {periodicEnabled && ( +
+
+ + +
+
+ )} +
+
+ + +
+ + {/* What you get section */} + {getConnectorBenefits(EnumConnectorName.OBSIDIAN_CONNECTOR) && ( +
+

+ What you get with Obsidian integration: +

+
    + {getConnectorBenefits(EnumConnectorName.OBSIDIAN_CONNECTOR)?.map((benefit) => ( +
  • {benefit}
    • + ))} +
+
+ )} + + {/* Documentation Section */} + + + + Documentation + + +
+

How it works

+

+ The Obsidian connector scans your local Obsidian vault directory and indexes all + Markdown files. It preserves your note structure and extracts metadata from YAML + frontmatter. +

+
    +
  • + The connector parses frontmatter metadata (title, tags, aliases, dates, etc.) +
    • +
  • Wiki-style links ([[note]]) are extracted and preserved
    • +
  • Inline tags (#tag) are recognized and indexed
    • +
  • Content is chunked intelligently for optimal search results
    • +
  • + Subsequent indexing runs use content hashing to skip unchanged files for faster + sync +
    • +
+
+ +
+
+

Setup

+ + + + File System Access Required + + + The SurfSense backend must have read access to your Obsidian vault directory. For + Docker deployments, mount your vault as a volume. + + + +
+
+

+ Step 1: Locate your vault +

+
    +
  1. Open Obsidian and go to Settings → About
    2. +
  2. Look for "Vault path" to find the location
    3. +
  3. Or right-click any note and select "Reveal in Finder/Explorer"
    4. +
+
+ +
+

+ Step 2: Mount vault for Docker +

+

+ If running SurfSense in Docker, add a volume mount: +

+ 
+											{`volumes:
+  - /path/to/your/vault:/app/obsidian_vaults/my-vault:ro`}
+
+

+ Then use /app/obsidian_vaults/my-vault as your vault path. +

+
+ +
+

+ Step 3: Configure exclusions +

+

+ Common folders to exclude: +

+
    +
  • + .obsidian - Obsidian config (always recommended) +
    • +
  • + .trash - Obsidian's trash folder +
    • +
  • + templates - If you have a templates folder +
    • +
  • + daily-notes - If you want to exclude daily notes +
    • +
+
+
+
+
+ +
+
+

What Gets Indexed

+ + + Indexed Content + +

The Obsidian connector indexes:

+
    +
  • All Markdown files (.md) in your vault
    • +
  • YAML frontmatter metadata (title, tags, aliases, dates)
    • +
  • Wiki-style links between notes
    • +
  • Inline tags throughout your notes
    • +
  • Full note content with proper chunking
    • +
+ + +
+
+ + + +
+ ); +}; diff --git a/surfsense_web/components/assistant-ui/connector-popup/connect-forms/connector-benefits.ts b/surfsense_web/components/assistant-ui/connector-popup/connect-forms/connector-benefits.ts index 9a74132ed..392de4bc8 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connect-forms/connector-benefits.ts +++ b/surfsense_web/components/assistant-ui/connector-popup/connect-forms/connector-benefits.ts @@ -108,6 +108,14 @@ export function getConnectorBenefits(connectorType: string): string[] | null { "Real-time updates via webhook integration", "No manual indexing required - meetings are added automatically", ], + OBSIDIAN_CONNECTOR: [ + "Search through all your Obsidian notes and knowledge base", + "Access note content with YAML frontmatter metadata preserved", + "Wiki-style links ([[note]]) and #tags are indexed", + "Connect your personal knowledge base directly to your search space", + "Incremental sync - only changed files are re-indexed", + "Full support for your vault's folder structure", + ], }; return benefits[connectorType] || null; diff --git a/surfsense_web/components/assistant-ui/connector-popup/connect-forms/index.tsx b/surfsense_web/components/assistant-ui/connector-popup/connect-forms/index.tsx index 4710f9ad5..ffaeb1478 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connect-forms/index.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connect-forms/index.tsx @@ -7,6 +7,7 @@ import { GithubConnectForm } from "./components/github-connect-form"; import { LinkupApiConnectForm } from "./components/linkup-api-connect-form"; import { LumaConnectForm } from "./components/luma-connect-form"; import { MCPConnectForm } from "./components/mcp-connect-form"; +import { ObsidianConnectForm } from "./components/obsidian-connect-form"; import { SearxngConnectForm } from "./components/searxng-connect-form"; import { TavilyApiConnectForm } from "./components/tavily-api-connect-form"; @@ -58,6 +59,8 @@ export function getConnectFormComponent(connectorType: string): ConnectFormCompo return CirclebackConnectForm; case "MCP_CONNECTOR": return MCPConnectForm; + case "OBSIDIAN_CONNECTOR": + return ObsidianConnectForm; // Add other connector types here as needed default: return null; diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/obsidian-config.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/obsidian-config.tsx new file mode 100644 index 000000000..f79f2953a --- /dev/null +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/obsidian-config.tsx @@ -0,0 +1,191 @@ +"use client"; + +import { FolderOpen } from "lucide-react"; +import type { FC } from "react"; +import { useEffect, useState } from "react"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { Switch } from "@/components/ui/switch"; +import type { ConnectorConfigProps } from "../index"; + +export interface ObsidianConfigProps extends ConnectorConfigProps { + onNameChange?: (name: string) => void; +} + +export const ObsidianConfig: FC = ({ + connector, + onConfigChange, + onNameChange, +}) => { + const [vaultPath, setVaultPath] = useState( + (connector.config?.vault_path as string) || "" + ); + const [vaultName, setVaultName] = useState( + (connector.config?.vault_name as string) || "" + ); + const [excludeFolders, setExcludeFolders] = useState(() => { + const folders = connector.config?.exclude_folders; + if (Array.isArray(folders)) { + return folders.join(", "); + } + return (folders as string) || ".obsidian, .trash"; + }); + const [includeAttachments, setIncludeAttachments] = useState( + (connector.config?.include_attachments as boolean) || false + ); + const [name, setName] = useState(connector.name || ""); + + // Update values when connector changes + useEffect(() => { + const path = (connector.config?.vault_path as string) || ""; + const vName = (connector.config?.vault_name as string) || ""; + const folders = connector.config?.exclude_folders; + const attachments = (connector.config?.include_attachments as boolean) || false; + + setVaultPath(path); + setVaultName(vName); + setIncludeAttachments(attachments); + setName(connector.name || ""); + + if (Array.isArray(folders)) { + setExcludeFolders(folders.join(", ")); + } else if (typeof folders === "string") { + setExcludeFolders(folders); + } + }, [connector.config, connector.name]); + + const handleVaultPathChange = (value: string) => { + setVaultPath(value); + if (onConfigChange) { + onConfigChange({ + ...connector.config, + vault_path: value, + }); + } + }; + + const handleVaultNameChange = (value: string) => { + setVaultName(value); + if (onConfigChange) { + onConfigChange({ + ...connector.config, + vault_name: value, + }); + } + }; + + const handleExcludeFoldersChange = (value: string) => { + setExcludeFolders(value); + const foldersArray = value + .split(",") + .map((f) => f.trim()) + .filter(Boolean); + if (onConfigChange) { + onConfigChange({ + ...connector.config, + exclude_folders: foldersArray, + }); + } + }; + + const handleIncludeAttachmentsChange = (value: boolean) => { + setIncludeAttachments(value); + if (onConfigChange) { + onConfigChange({ + ...connector.config, + include_attachments: value, + }); + } + }; + + const handleNameChange = (value: string) => { + setName(value); + if (onNameChange) { + onNameChange(value); + } + }; + + return ( +
+ {/* Connector Name */} +
+
+ + handleNameChange(e.target.value)} + placeholder="My Obsidian Vault" + className="border-slate-400/20 focus-visible:border-slate-400/40" + /> +

+ A friendly name to identify this connector. +

+
+
+ + {/* Configuration */} +
+
+

+ + Vault Configuration +

+
+ +
+
+ + handleVaultPathChange(e.target.value)} + placeholder="/path/to/your/obsidian/vault" + className="border-slate-400/20 focus-visible:border-slate-400/40 font-mono" + /> +

+ The absolute path to your Obsidian vault on the server. +

+
+ +
+ + handleVaultNameChange(e.target.value)} + placeholder="My Knowledge Base" + className="border-slate-400/20 focus-visible:border-slate-400/40" + /> +

+ A display name for your vault in search results. +

+
+ +
+ + handleExcludeFoldersChange(e.target.value)} + placeholder=".obsidian, .trash, templates" + className="border-slate-400/20 focus-visible:border-slate-400/40 font-mono" + /> +

+ Comma-separated list of folder names to exclude from indexing. +

+
+ +
+
+ +

+ Index attachment folders and embedded files +

+
+ +
+
+
+
+ ); +}; diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/index.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/index.tsx index b493ce746..359343863 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/index.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/index.tsx @@ -15,6 +15,7 @@ import { JiraConfig } from "./components/jira-config"; import { LinkupApiConfig } from "./components/linkup-api-config"; import { LumaConfig } from "./components/luma-config"; import { MCPConfig } from "./components/mcp-config"; +import { ObsidianConfig } from "./components/obsidian-config"; import { SearxngConfig } from "./components/searxng-config"; import { SlackConfig } from "./components/slack-config"; import { TavilyApiConfig } from "./components/tavily-api-config"; @@ -73,6 +74,8 @@ export function getConnectorConfigComponent( return CirclebackConfig; case "MCP_CONNECTOR": return MCPConfig; + case "OBSIDIAN_CONNECTOR": + return ObsidianConfig; // OAuth connectors (Gmail, Calendar, Airtable, Notion) and others don't need special config UI default: return null; diff --git a/surfsense_web/components/assistant-ui/connector-popup/constants/connector-constants.ts b/surfsense_web/components/assistant-ui/connector-popup/constants/connector-constants.ts index 1ac3549df..e145a3d55 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/constants/connector-constants.ts +++ b/surfsense_web/components/assistant-ui/connector-popup/constants/connector-constants.ts @@ -166,6 +166,13 @@ export const OTHER_CONNECTORS = [ description: "Connect to MCP servers for AI tools", connectorType: EnumConnectorName.MCP_CONNECTOR, }, + { + id: "obsidian-connector", + title: "Obsidian", + description: "Index your Obsidian vault (self-hosted only)", + connectorType: EnumConnectorName.OBSIDIAN_CONNECTOR, + selfHostedOnly: true, + }, ] as const; // Re-export IndexingConfigState from schemas for backward compatibility diff --git a/surfsense_web/contracts/enums/connector.ts b/surfsense_web/contracts/enums/connector.ts index 7fe170eef..cd0703277 100644 --- a/surfsense_web/contracts/enums/connector.ts +++ b/surfsense_web/contracts/enums/connector.ts @@ -23,5 +23,6 @@ export enum EnumConnectorName { WEBCRAWLER_CONNECTOR = "WEBCRAWLER_CONNECTOR", YOUTUBE_CONNECTOR = "YOUTUBE_CONNECTOR", CIRCLEBACK_CONNECTOR = "CIRCLEBACK_CONNECTOR", + OBSIDIAN_CONNECTOR = "OBSIDIAN_CONNECTOR", MCP_CONNECTOR = "MCP_CONNECTOR", } diff --git a/surfsense_web/contracts/enums/connectorIcons.tsx b/surfsense_web/contracts/enums/connectorIcons.tsx index 9350b6a1e..f19f65162 100644 --- a/surfsense_web/contracts/enums/connectorIcons.tsx +++ b/surfsense_web/contracts/enums/connectorIcons.tsx @@ -66,6 +66,8 @@ export const getConnectorIcon = (connectorType: EnumConnectorName | string, clas return ; case EnumConnectorName.MCP_CONNECTOR: return MCP; + case EnumConnectorName.OBSIDIAN_CONNECTOR: + return Obsidian; // Additional cases for non-enum connector types case "YOUTUBE_CONNECTOR": return YouTube; diff --git a/surfsense_web/public/connectors/obsidian.svg b/surfsense_web/public/connectors/obsidian.svg new file mode 100644 index 000000000..9fe15c4a3 --- /dev/null +++ b/surfsense_web/public/connectors/obsidian.svg @@ -0,0 +1,12 @@ + + + + + + + + + + + + From 88a9a5bef2f6ee1a4d6fd38c504f4b1108b6deda Mon Sep 17 00:00:00 2001 From: Manoj Aggarwal Date: Wed, 21 Jan 2026 22:54:09 -0800 Subject: [PATCH 18/44] format --- .../versions/74_add_obsidian_connector.py | 4 +- .../agents/new_chat/tools/knowledge_base.py | 11 ++ .../app/services/connector_service.py | 91 +++++++++++++++ .../connector_indexers/obsidian_indexer.py | 107 +++++++++++------- .../components/obsidian-connect-form.tsx | 30 +++-- .../views/connector-connect-view.tsx | 9 +- .../utils/connector-document-mapping.ts | 1 + .../contracts/types/connector.types.ts | 1 + 8 files changed, 200 insertions(+), 54 deletions(-) diff --git a/surfsense_backend/alembic/versions/74_add_obsidian_connector.py b/surfsense_backend/alembic/versions/74_add_obsidian_connector.py index b34cd07a0..ea2b0c46e 100644 --- a/surfsense_backend/alembic/versions/74_add_obsidian_connector.py +++ b/surfsense_backend/alembic/versions/74_add_obsidian_connector.py @@ -1,7 +1,7 @@ """Add Obsidian connector enums Revision ID: 74_add_obsidian_connector -Revises: 73_add_user_memories_table +Revises: 73 Create Date: 2026-01-21 """ @@ -12,7 +12,7 @@ from alembic import op # revision identifiers, used by Alembic. revision: str = "74_add_obsidian_connector" -down_revision: str | None = "73_add_user_memories_table" +down_revision: str | None = "73" branch_labels: str | Sequence[str] | None = None depends_on: str | Sequence[str] | None = None diff --git a/surfsense_backend/app/agents/new_chat/tools/knowledge_base.py b/surfsense_backend/app/agents/new_chat/tools/knowledge_base.py index e91d865fa..84a0fdb3c 100644 --- a/surfsense_backend/app/agents/new_chat/tools/knowledge_base.py +++ b/surfsense_backend/app/agents/new_chat/tools/knowledge_base.py @@ -49,6 +49,7 @@ _ALL_CONNECTORS: list[str] = [ "BOOKSTACK_CONNECTOR", "CRAWLED_URL", "CIRCLEBACK", + "OBSIDIAN_CONNECTOR", ] @@ -508,6 +509,16 @@ async def search_knowledge_base_async( ) all_documents.extend(chunks) + elif connector == "OBSIDIAN_CONNECTOR": + _, chunks = await connector_service.search_obsidian( + user_query=query, + search_space_id=search_space_id, + top_k=top_k, + start_date=resolved_start_date, + end_date=resolved_end_date, + ) + all_documents.extend(chunks) + except Exception as e: print(f"Error searching connector {connector}: {e}") continue diff --git a/surfsense_backend/app/services/connector_service.py b/surfsense_backend/app/services/connector_service.py index 832aee4cc..dc43697e7 100644 --- a/surfsense_backend/app/services/connector_service.py +++ b/surfsense_backend/app/services/connector_service.py @@ -2780,3 +2780,94 @@ class ConnectorService: } return result_object, circleback_docs + + async def search_obsidian( + self, + user_query: str, + search_space_id: int, + top_k: int = 20, + start_date: datetime | None = None, + end_date: datetime | None = None, + ) -> tuple: + """ + Search for Obsidian vault notes and return both the source information and langchain documents. + + Uses combined chunk-level and document-level hybrid search with RRF fusion. + + Args: + user_query: The user's query + search_space_id: The search space ID to search in + top_k: Maximum number of results to return + start_date: Optional start date for filtering documents by updated_at + end_date: Optional end date for filtering documents by updated_at + + Returns: + tuple: (sources_info, langchain_documents) + """ + obsidian_docs = await self._combined_rrf_search( + query_text=user_query, + search_space_id=search_space_id, + document_type="OBSIDIAN_CONNECTOR", + top_k=top_k, + start_date=start_date, + end_date=end_date, + ) + + # Early return if no results + if not obsidian_docs: + return { + "id": 53, + "name": "Obsidian Vault", + "type": "OBSIDIAN_CONNECTOR", + "sources": [], + }, [] + + def _title_fn(doc_info: dict[str, Any], metadata: dict[str, Any]) -> str: + return doc_info.get("title", "Untitled Note") + + def _url_fn(doc_info: dict[str, Any], metadata: dict[str, Any]) -> str: + # Obsidian URL format: obsidian://vault_name/path + return doc_info.get("url", "") + + def _description_fn( + chunk: dict[str, Any], _doc_info: dict[str, Any], metadata: dict[str, Any] + ) -> str: + description = self._chunk_preview(chunk.get("content", ""), limit=200) + info_parts = [] + vault_name = metadata.get("vault_name") + tags = metadata.get("tags", []) + if vault_name: + info_parts.append(f"Vault: {vault_name}") + if tags and isinstance(tags, list) and len(tags) > 0: + info_parts.append(f"Tags: {', '.join(tags[:3])}") + if info_parts: + description = (description + " | " + " | ".join(info_parts)).strip(" |") + return description + + def _extra_fields_fn( + _chunk: dict[str, Any], _doc_info: dict[str, Any], metadata: dict[str, Any] + ) -> dict[str, Any]: + return { + "vault_name": metadata.get("vault_name", ""), + "file_path": metadata.get("file_path", ""), + "tags": metadata.get("tags", []), + "outgoing_links": metadata.get("outgoing_links", []), + } + + sources_list = self._build_chunk_sources_from_documents( + obsidian_docs, + title_fn=_title_fn, + url_fn=_url_fn, + description_fn=_description_fn, + extra_fields_fn=_extra_fields_fn, + ) + + # Create result object + result_object = { + "id": 53, + "name": "Obsidian Vault", + "type": "OBSIDIAN_CONNECTOR", + "sources": sources_list, + } + + return result_object, obsidian_docs diff --git a/surfsense_backend/app/tasks/connector_indexers/obsidian_indexer.py b/surfsense_backend/app/tasks/connector_indexers/obsidian_indexer.py index 15c88938a..4c4dab4c2 100644 --- a/surfsense_backend/app/tasks/connector_indexers/obsidian_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/obsidian_indexer.py @@ -7,7 +7,7 @@ This connector is only available in self-hosted mode. import os import re -from datetime import datetime, UTC +from datetime import UTC, datetime from pathlib import Path import yaml @@ -266,17 +266,40 @@ async def index_obsidian_vault( {"stage": "files_discovered", "file_count": len(files)}, ) - # Filter by date if provided - if start_date: - start_dt = datetime.strptime(start_date, "%Y-%m-%d").replace(tzinfo=UTC) - files = [f for f in files if f["modified_at"] >= start_dt] + # Filter by date if provided (handle "undefined" string from frontend) + # Also handle inverted dates (start > end) by skipping filtering + start_dt = None + end_dt = None - if end_date: + if start_date and start_date != "undefined": + start_dt = datetime.strptime(start_date, "%Y-%m-%d").replace(tzinfo=UTC) + + if end_date and end_date != "undefined": + # Make end_date inclusive (end of day) end_dt = datetime.strptime(end_date, "%Y-%m-%d").replace(tzinfo=UTC) - files = [f for f in files if f["modified_at"] <= end_dt] + end_dt = end_dt.replace(hour=23, minute=59, second=59) + + # Only apply date filtering if dates are valid and in correct order + if start_dt and end_dt and start_dt > end_dt: + logger.warning( + f"start_date ({start_date}) is after end_date ({end_date}), skipping date filter" + ) + else: + if start_dt: + files = [f for f in files if f["modified_at"] >= start_dt] + logger.info( + f"After start_date filter ({start_date}): {len(files)} files" + ) + if end_dt: + files = [f for f in files if f["modified_at"] <= end_dt] + logger.info(f"After end_date filter ({end_date}): {len(files)} files") + + logger.info(f"Processing {len(files)} files after date filtering") # Get LLM for summarization - long_context_llm = await get_user_long_context_llm(session, user_id) + long_context_llm = await get_user_long_context_llm( + session, user_id, search_space_id + ) indexed_count = 0 skipped_count = 0 @@ -312,9 +335,9 @@ async def index_obsidian_vault( # Also extract tags from frontmatter fm_tags = frontmatter.get("tags", []) if isinstance(fm_tags, list): - tags = list(set(tags + fm_tags)) + tags = list({*tags, *fm_tags}) elif isinstance(fm_tags, str): - tags = list(set(tags + [fm_tags])) + tags = list({*tags, fm_tags}) # Generate unique identifier using vault name and relative path unique_identifier = f"{vault_name}:{relative_path}" @@ -330,7 +353,7 @@ async def index_obsidian_vault( ) # Generate content hash - content_hash = generate_content_hash(content) + content_hash = generate_content_hash(content, search_space_id) # Build metadata document_metadata = { @@ -372,11 +395,19 @@ async def index_obsidian_vault( # Generate new summary if content changed if long_context_llm: - new_summary = await generate_document_summary( - content=document_string, - llm=long_context_llm, + new_summary, _ = await generate_document_summary( + document_string, + long_context_llm, + document_metadata, ) - existing_document.summary = new_summary + # Store summary in metadata + document_metadata["summary"] = new_summary + + # Add URL and connector_id to metadata + document_metadata["url"] = ( + f"obsidian://{vault_name}/{relative_path}" + ) + document_metadata["connector_id"] = connector_id existing_document.content = document_string existing_document.content_hash = content_hash @@ -387,14 +418,10 @@ async def index_obsidian_vault( embedding = config.embedding_model_instance.embed(document_string) existing_document.embedding = embedding - # Update chunks - await create_document_chunks( - session=session, - document=existing_document, - content=document_string, - chunker=config.chunker_instance, - embedding_model=config.embedding_model_instance, - ) + # Update chunks - delete old and create new + existing_document.chunks.clear() + new_chunks = await create_document_chunks(document_string) + existing_document.chunks = new_chunks indexed_count += 1 @@ -403,42 +430,42 @@ async def index_obsidian_vault( logger.info(f"Indexing new note: {title}") # Generate summary - summary = "" + summary_content = "" if long_context_llm: - summary = await generate_document_summary( - content=document_string, - llm=long_context_llm, + summary_content, _ = await generate_document_summary( + document_string, + long_context_llm, + document_metadata, ) # Generate embedding embedding = config.embedding_model_instance.embed(document_string) + # Add URL and summary to metadata + document_metadata["url"] = ( + f"obsidian://{vault_name}/{relative_path}" + ) + document_metadata["summary"] = summary_content + document_metadata["connector_id"] = connector_id + + # Create chunks + chunks = await create_document_chunks(document_string) + # Create document new_document = Document( search_space_id=search_space_id, title=title, - url=f"obsidian://{vault_name}/{relative_path}", document_type=DocumentType.OBSIDIAN_CONNECTOR, content=document_string, content_hash=content_hash, unique_identifier_hash=unique_identifier_hash, document_metadata=document_metadata, - summary=summary, embedding=embedding, - connector_id=connector_id, + chunks=chunks, + updated_at=get_current_timestamp(), ) session.add(new_document) - await session.flush() - - # Create chunks - await create_document_chunks( - session=session, - document=new_document, - content=document_string, - chunker=config.chunker_instance, - embedding_model=config.embedding_model_instance, - ) indexed_count += 1 diff --git a/surfsense_web/components/assistant-ui/connector-popup/connect-forms/components/obsidian-connect-form.tsx b/surfsense_web/components/assistant-ui/connector-popup/connect-forms/components/obsidian-connect-form.tsx index acdbc4c1f..064e10e2f 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connect-forms/components/obsidian-connect-form.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connect-forms/components/obsidian-connect-form.tsx @@ -362,8 +362,8 @@ export const ObsidianConnectForm: FC = ({ onSubmit, isSubmitti File System Access Required - The SurfSense backend must have read access to your Obsidian vault directory. For - Docker deployments, mount your vault as a volume. + The SurfSense backend must have read access to your Obsidian vault directory. + For Docker deployments, mount your vault as a volume. @@ -373,18 +373,34 @@ export const ObsidianConnectForm: FC = ({ onSubmit, isSubmitti Step 1: Locate your vault
    -
  1. Open Obsidian and go to Settings → About
    2. -
  2. Look for "Vault path" to find the location
    3. -
  3. Or right-click any note and select "Reveal in Finder/Explorer"
    4. +
  4. + macOS/Linux: Right-click any note in Obsidian → "Reveal in + Finder" to see the vault folder +
    5. +
  5. + Windows: Right-click any note → "Show in system explorer" +
    6. +
  6. + Or: Click the vault switcher (bottom-left icon) → "Open + folder" next to your vault name +

- Step 2: Mount vault for Docker + Step 2: Enter the path

- If running SurfSense in Docker, add a volume mount: + Running locally (no Docker): Use the direct path to your + vault: +

+ 
+											{`/Users/yourname/Documents/MyObsidianVault`}
+
+

+ Running in Docker: Mount your vault as a volume in + docker-compose.yml: 

 											{`volumes:
diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-connect-view.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-connect-view.tsx
index 7666b775f..49d1ebacc 100644
--- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-connect-view.tsx
+++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-connect-view.tsx
@@ -57,6 +57,7 @@ export const ConnectorConnectView: FC = ({
 			LUMA_CONNECTOR: "luma-connect-form",
 			CIRCLEBACK_CONNECTOR: "circleback-connect-form",
 			MCP_CONNECTOR: "mcp-connect-form",
+			OBSIDIAN_CONNECTOR: "obsidian-connect-form",
 		};
 		const formId = formIdMap[connectorType];
 		if (formId) {
@@ -141,12 +142,10 @@ export const ConnectorConnectView: FC = ({
 							
 							Connecting
 						
+					) : connectorType === "MCP_CONNECTOR" ? (
+						"Connect"
 					) : (
-						<>
-							{connectorType === "MCP_CONNECTOR"
-								? "Connect"
-								: `Connect ${getConnectorTypeDisplay(connectorType)}`}
-						
+						`Connect ${getConnectorTypeDisplay(connectorType)}`
 					)}
diff --git a/surfsense_web/components/assistant-ui/connector-popup/utils/connector-document-mapping.ts b/surfsense_web/components/assistant-ui/connector-popup/utils/connector-document-mapping.ts index 433a51e8c..ab66a943a 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/utils/connector-document-mapping.ts +++ b/surfsense_web/components/assistant-ui/connector-popup/utils/connector-document-mapping.ts @@ -26,6 +26,7 @@ export const CONNECTOR_TO_DOCUMENT_TYPE: Record = { ELASTICSEARCH_CONNECTOR: "ELASTICSEARCH_CONNECTOR", BOOKSTACK_CONNECTOR: "BOOKSTACK_CONNECTOR", CIRCLEBACK_CONNECTOR: "CIRCLEBACK", + OBSIDIAN_CONNECTOR: "OBSIDIAN_CONNECTOR", // Special mappings (connector type differs from document type) GOOGLE_DRIVE_CONNECTOR: "GOOGLE_DRIVE_FILE", diff --git a/surfsense_web/contracts/types/connector.types.ts b/surfsense_web/contracts/types/connector.types.ts index afd20b474..0e298443e 100644 --- a/surfsense_web/contracts/types/connector.types.ts +++ b/surfsense_web/contracts/types/connector.types.ts @@ -27,6 +27,7 @@ export const searchSourceConnectorTypeEnum = z.enum([ "BOOKSTACK_CONNECTOR", "CIRCLEBACK_CONNECTOR", "MCP_CONNECTOR", + "OBSIDIAN_CONNECTOR", ]); export const searchSourceConnector = z.object({ From 3ab9cc84854c8e050ad76b7c45228154777b6a74 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 22 Jan 2026 15:05:00 +0200 Subject: [PATCH 19/44] Restore API-based comments and remove unstable live sync integration --- .../new-chat/[[...chat_id]]/page.tsx | 43 ------------------- .../assistant-ui/assistant-message.tsx | 10 +++-- .../comment-panel-container.tsx | 15 ++++--- .../contracts/types/chat-messages.types.ts | 2 +- surfsense_web/hooks/use-chat-messages-live.ts | 7 ++- 5 files changed, 22 insertions(+), 55 deletions(-) diff --git a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx index 865dd6e68..43c33ba5a 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx @@ -33,7 +33,6 @@ import { GeneratePodcastToolUI } from "@/components/tool-ui/generate-podcast"; import { LinkPreviewToolUI } from "@/components/tool-ui/link-preview"; import { ScrapeWebpageToolUI } from "@/components/tool-ui/scrape-webpage"; // import { WriteTodosToolUI } from "@/components/tool-ui/write-todos"; -import { useChatMessagesLive } from "@/hooks/use-chat-messages-live"; import { getBearerToken } from "@/lib/auth-utils"; import { createAttachmentAdapter, extractAttachmentContent } from "@/lib/chat/attachment-adapter"; import { @@ -258,9 +257,6 @@ export default function NewChatPage() { // Get current user for author info in shared chats const { data: currentUser } = useAtomValue(currentUserAtom); - // Live sync for other users' messages (shared chats) - const { messages: liveMessages } = useChatMessagesLive(threadId); - // Create the attachment adapter for file processing const attachmentAdapter = useMemo(() => createAttachmentAdapter(), []); @@ -413,45 +409,6 @@ export default function NewChatPage() { }); }, [currentThread, setCurrentThreadState]); - // Live sync: Merge messages from other users (shared chats) - useEffect(() => { - if (!liveMessages.length || !currentUser?.id || isRunning) return; - - // Get IDs of messages we already have locally - const localMessageIds = new Set( - messages - .map((m) => { - // Extract numeric ID from "msg-{id}" format - const match = m.id?.match(/^msg-(\d+)$/); - return match ? Number.parseInt(match[1], 10) : null; - }) - .filter((id): id is number => id !== null) - ); - - // Find live messages from OTHER users that we don't have locally - const newOtherUserMessages = liveMessages.filter((liveMsg) => { - // Skip if we already have this message - if (localMessageIds.has(liveMsg.id)) return false; - // Skip if this is our own message (we added it optimistically) - if (liveMsg.author_id === currentUser.id) return false; - return true; - }); - - if (newOtherUserMessages.length > 0) { - // Convert and add new messages from other users - const converted = newOtherUserMessages.map(convertToThreadMessage); - setMessages((prev) => { - // Merge and sort by ID to maintain order - const merged = [...prev, ...converted]; - return merged.sort((a, b) => { - const aId = Number.parseInt((a.id ?? "").replace(/^msg-/, ""), 10) || 0; - const bId = Number.parseInt((b.id ?? "").replace(/^msg-/, ""), 10) || 0; - return aId - bId; - }); - }); - } - }, [liveMessages, currentUser?.id, messages, isRunning]); - // Cancel ongoing request const cancelRun = useCallback(async () => { if (abortControllerRef.current) { diff --git a/surfsense_web/components/assistant-ui/assistant-message.tsx b/surfsense_web/components/assistant-ui/assistant-message.tsx index 513242d1b..681dc315a 100644 --- a/surfsense_web/components/assistant-ui/assistant-message.tsx +++ b/surfsense_web/components/assistant-ui/assistant-message.tsx @@ -25,7 +25,7 @@ import { TooltipIconButton } from "@/components/assistant-ui/tooltip-icon-button import { CommentPanelContainer } from "@/components/chat-comments/comment-panel-container/comment-panel-container"; import { CommentSheet } from "@/components/chat-comments/comment-sheet/comment-sheet"; import { CommentTrigger } from "@/components/chat-comments/comment-trigger/comment-trigger"; -import { useCommentsLive } from "@/hooks/use-comments-live"; +import { useComments } from "@/hooks/use-comments"; import { useMediaQuery } from "@/hooks/use-media-query"; import { cn } from "@/lib/utils"; @@ -115,8 +115,12 @@ export const AssistantMessage: FC = () => { const isLastMessage = useAssistantState(({ message }) => message?.isLast ?? false); const isMessageStreaming = isThreadRunning && isLastMessage; - // Live sync for real-time comment count - const { commentCount } = useCommentsLive(dbMessageId); + const { data: commentsData } = useComments({ + messageId: dbMessageId ?? 0, + enabled: !!dbMessageId, + }); + + const commentCount = commentsData?.total_count ?? 0; const hasComments = commentCount > 0; const isAddingComment = dbMessageId !== null && addingCommentToMessageId === dbMessageId; const showCommentPanel = hasComments || isAddingComment; diff --git a/surfsense_web/components/chat-comments/comment-panel-container/comment-panel-container.tsx b/surfsense_web/components/chat-comments/comment-panel-container/comment-panel-container.tsx index 8281ca8fd..197ac0798 100644 --- a/surfsense_web/components/chat-comments/comment-panel-container/comment-panel-container.tsx +++ b/surfsense_web/components/chat-comments/comment-panel-container/comment-panel-container.tsx @@ -10,7 +10,7 @@ import { } from "@/atoms/chat-comments/comments-mutation.atoms"; import { membersAtom } from "@/atoms/members/members-query.atoms"; import { currentUserAtom } from "@/atoms/user/user-query.atoms"; -import { useCommentsLive } from "@/hooks/use-comments-live"; +import { useComments } from "@/hooks/use-comments"; import { CommentPanel } from "../comment-panel/comment-panel"; import type { CommentPanelContainerProps } from "./types"; import { transformComment, transformMember } from "./utils"; @@ -21,10 +21,10 @@ export function CommentPanelContainer({ maxHeight, variant = "desktop", }: CommentPanelContainerProps) { - // Live sync for real-time comment updates - const { comments: liveComments, isLoading: isCommentsLoading } = useCommentsLive( - isOpen ? messageId : null - ); + const { data: commentsData, isLoading: isCommentsLoading } = useComments({ + messageId, + enabled: isOpen, + }); const [{ data: membersData, isLoading: isMembersLoading }] = useAtom(membersAtom); const [{ data: currentUser }] = useAtom(currentUserAtom); @@ -35,8 +35,9 @@ export function CommentPanelContainer({ const [{ mutate: deleteComment, isPending: isDeleting }] = useAtom(deleteCommentMutationAtom); const commentThreads = useMemo(() => { - return liveComments.map(transformComment); - }, [liveComments]); + if (!commentsData?.comments) return []; + return commentsData.comments.map(transformComment); + }, [commentsData]); const members = useMemo(() => { if (!membersData) return []; diff --git a/surfsense_web/contracts/types/chat-messages.types.ts b/surfsense_web/contracts/types/chat-messages.types.ts index 01edef3f2..faba71bff 100644 --- a/surfsense_web/contracts/types/chat-messages.types.ts +++ b/surfsense_web/contracts/types/chat-messages.types.ts @@ -6,7 +6,7 @@ import { z } from "zod"; export const rawMessage = z.object({ id: z.number(), thread_id: z.number(), - role: z.enum(["user", "assistant", "system"]), + role: z.string(), content: z.unknown(), author_id: z.string().nullable(), created_at: z.string(), diff --git a/surfsense_web/hooks/use-chat-messages-live.ts b/surfsense_web/hooks/use-chat-messages-live.ts index 39341d479..4a8ae97e6 100644 --- a/surfsense_web/hooks/use-chat-messages-live.ts +++ b/surfsense_web/hooks/use-chat-messages-live.ts @@ -55,10 +55,15 @@ export function useChatMessagesLive(threadId: number | null) { // Transform raw messages to MessageRecord with author info return [...messagesData].map((msg): MessageRecord => { const author = msg.author_id ? memberMap.get(msg.author_id) : null; + + const role = (typeof msg.role === "string" ? msg.role.toLowerCase() : msg.role) as + | "user" + | "assistant" + | "system"; return { id: msg.id, thread_id: msg.thread_id, - role: msg.role, + role, content: msg.content, created_at: msg.created_at, author_id: msg.author_id, From ac7d84571dbbe834d01ecf653d1c95df288946a5 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 22 Jan 2026 17:27:42 +0200 Subject: [PATCH 20/44] Add thread_id to chat_comments for Electric sync --- .../75_add_thread_id_to_chat_comments.py | 68 +++++++++++++++++++ surfsense_backend/app/db.py | 8 +++ .../app/services/chat_comments_service.py | 6 +- 3 files changed, 80 insertions(+), 2 deletions(-) create mode 100644 surfsense_backend/alembic/versions/75_add_thread_id_to_chat_comments.py diff --git a/surfsense_backend/alembic/versions/75_add_thread_id_to_chat_comments.py b/surfsense_backend/alembic/versions/75_add_thread_id_to_chat_comments.py new file mode 100644 index 000000000..ceeb589f4 --- /dev/null +++ b/surfsense_backend/alembic/versions/75_add_thread_id_to_chat_comments.py @@ -0,0 +1,68 @@ +"""Add thread_id to chat_comments for denormalized Electric subscriptions + +This denormalization allows a single Electric SQL subscription per thread +instead of one per message, significantly reducing connection overhead. + +Revision ID: 75 +Revises: 74 +""" + +from collections.abc import Sequence + +from alembic import op + +revision: str = "75" +down_revision: str | None = "74" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Add thread_id column to chat_comments and backfill from messages.""" + # Add the column (nullable initially for backfill) + op.execute( + """ + ALTER TABLE chat_comments + ADD COLUMN IF NOT EXISTS thread_id INTEGER; + """ + ) + + # Backfill thread_id from the related message + op.execute( + """ + UPDATE chat_comments c + SET thread_id = m.thread_id + FROM new_chat_messages m + WHERE c.message_id = m.id + AND c.thread_id IS NULL; + """ + ) + + # Make it NOT NULL after backfill + op.execute( + """ + ALTER TABLE chat_comments + ALTER COLUMN thread_id SET NOT NULL; + """ + ) + + # Add FK constraint + op.execute( + """ + ALTER TABLE chat_comments + ADD CONSTRAINT fk_chat_comments_thread_id + FOREIGN KEY (thread_id) REFERENCES new_chat_threads(id) ON DELETE CASCADE; + """ + ) + + # Add index for efficient Electric subscriptions by thread + op.execute( + "CREATE INDEX IF NOT EXISTS idx_chat_comments_thread_id ON chat_comments(thread_id)" + ) + + +def downgrade() -> None: + """Remove thread_id column from chat_comments.""" + op.execute("DROP INDEX IF EXISTS idx_chat_comments_thread_id") + op.execute("ALTER TABLE chat_comments DROP CONSTRAINT IF EXISTS fk_chat_comments_thread_id") + op.execute("ALTER TABLE chat_comments DROP COLUMN IF EXISTS thread_id") diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py index 35b512c5e..181f78101 100644 --- a/surfsense_backend/app/db.py +++ b/surfsense_backend/app/db.py @@ -413,6 +413,13 @@ class ChatComment(BaseModel, TimestampMixin): nullable=False, index=True, ) + # Denormalized thread_id for efficient Electric SQL subscriptions (one per thread) + thread_id = Column( + Integer, + ForeignKey("new_chat_threads.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) parent_id = Column( Integer, ForeignKey("chat_comments.id", ondelete="CASCADE"), @@ -436,6 +443,7 @@ class ChatComment(BaseModel, TimestampMixin): # Relationships message = relationship("NewChatMessage", back_populates="comments") + thread = relationship("NewChatThread") author = relationship("User") parent = relationship( "ChatComment", remote_side="ChatComment.id", backref="replies" diff --git a/surfsense_backend/app/services/chat_comments_service.py b/surfsense_backend/app/services/chat_comments_service.py index fa26bf6d5..7e07a56e6 100644 --- a/surfsense_backend/app/services/chat_comments_service.py +++ b/surfsense_backend/app/services/chat_comments_service.py @@ -281,8 +281,10 @@ async def create_comment( detail="You don't have permission to create comments in this search space", ) + thread = message.thread comment = ChatComment( message_id=message_id, + thread_id=thread.id, # Denormalized for efficient Electric subscriptions author_id=user.id, content=content, ) @@ -299,7 +301,6 @@ async def create_comment( user_names = await get_user_names_for_mentions(session, set(mentions_map.keys())) # Create notifications for mentioned users (excluding author) - thread = message.thread author_name = user.display_name or user.email content_preview = render_mentions(content, user_names) for mentioned_user_id, mention_id in mentions_map.items(): @@ -391,8 +392,10 @@ async def create_reply( detail="You don't have permission to create comments in this search space", ) + thread = parent_comment.message.thread reply = ChatComment( message_id=parent_comment.message_id, + thread_id=thread.id, # Denormalized for efficient Electric subscriptions parent_id=comment_id, author_id=user.id, content=content, @@ -410,7 +413,6 @@ async def create_reply( user_names = await get_user_names_for_mentions(session, set(mentions_map.keys())) # Create notifications for mentioned users (excluding author) - thread = parent_comment.message.thread author_name = user.display_name or user.email content_preview = render_mentions(content, user_names) for mentioned_user_id, mention_id in mentions_map.items(): From 4b57ba9d2bac1883d409bc337ea7a77f94f4f803 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 22 Jan 2026 17:56:45 +0200 Subject: [PATCH 21/44] Add chat_comments table to PGlite schema and thread_id to types --- .../contracts/types/chat-comments.types.ts | 1 + surfsense_web/lib/electric/client.ts | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/surfsense_web/contracts/types/chat-comments.types.ts b/surfsense_web/contracts/types/chat-comments.types.ts index c3b32be9d..46e064a4e 100644 --- a/surfsense_web/contracts/types/chat-comments.types.ts +++ b/surfsense_web/contracts/types/chat-comments.types.ts @@ -6,6 +6,7 @@ import { z } from "zod"; export const rawComment = z.object({ id: z.number(), message_id: z.number(), + thread_id: z.number(), // Denormalized for efficient Electric subscriptions parent_id: z.number().nullable(), author_id: z.string().nullable(), content: z.string(), diff --git a/surfsense_web/lib/electric/client.ts b/surfsense_web/lib/electric/client.ts index d900ddb0a..84df6c905 100644 --- a/surfsense_web/lib/electric/client.ts +++ b/surfsense_web/lib/electric/client.ts @@ -240,6 +240,24 @@ export async function initElectric(userId: string): Promise { CREATE INDEX IF NOT EXISTS idx_chat_comment_mentions_comment_id ON chat_comment_mentions(comment_id); `); + // Create chat_comments table for live comment sync + await db.exec(` + CREATE TABLE IF NOT EXISTS chat_comments ( + id INTEGER PRIMARY KEY, + message_id INTEGER NOT NULL, + thread_id INTEGER NOT NULL, + parent_id INTEGER, + author_id TEXT, + content TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + ); + + CREATE INDEX IF NOT EXISTS idx_chat_comments_thread_id ON chat_comments(thread_id); + CREATE INDEX IF NOT EXISTS idx_chat_comments_message_id ON chat_comments(message_id); + CREATE INDEX IF NOT EXISTS idx_chat_comments_parent_id ON chat_comments(parent_id); + `); + const electricUrl = getElectricUrl(); // STEP 4: Create the client wrapper From 0b8fed7304aee0275f8e249bcd81e80cbeca6087 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 22 Jan 2026 17:57:20 +0200 Subject: [PATCH 22/44] Add thread-level Electric sync for comments, remove per-message hooks --- .../components/assistant-ui/thread.tsx | 4 + surfsense_web/hooks/use-comments-electric.ts | 361 ++++++++++++++++++ surfsense_web/hooks/use-comments-live.ts | 154 -------- 3 files changed, 365 insertions(+), 154 deletions(-) create mode 100644 surfsense_web/hooks/use-comments-electric.ts delete mode 100644 surfsense_web/hooks/use-comments-live.ts diff --git a/surfsense_web/components/assistant-ui/thread.tsx b/surfsense_web/components/assistant-ui/thread.tsx index d65372c24..e1169867c 100644 --- a/surfsense_web/components/assistant-ui/thread.tsx +++ b/surfsense_web/components/assistant-ui/thread.tsx @@ -62,6 +62,7 @@ import type { ThinkingStep } from "@/components/tool-ui/deepagent-thinking"; import { Button } from "@/components/ui/button"; import type { Document } from "@/contracts/types/document.types"; import { useChatSessionState } from "@/hooks/use-chat-session-state"; +import { useCommentsElectric } from "@/hooks/use-comments-electric"; import { cn } from "@/lib/utils"; interface ThreadProps { @@ -238,6 +239,9 @@ const Composer: FC = () => { const { isAiResponding, respondingToUserId } = useChatSessionState(threadId); const isBlockedByOtherUser = isAiResponding && respondingToUserId !== currentUser?.id; + // Sync comments for the entire thread via Electric SQL (one subscription per thread) + useCommentsElectric(threadId); + // Auto-focus editor on new chat page after mount useEffect(() => { if (isThreadEmpty && !hasAutoFocusedRef.current && editorRef.current) { diff --git a/surfsense_web/hooks/use-comments-electric.ts b/surfsense_web/hooks/use-comments-electric.ts new file mode 100644 index 000000000..83a019ef3 --- /dev/null +++ b/surfsense_web/hooks/use-comments-electric.ts @@ -0,0 +1,361 @@ +"use client"; + +import { useQueryClient } from "@tanstack/react-query"; +import { useAtomValue } from "jotai"; +import { useCallback, useEffect, useMemo, useRef } from "react"; +import { membersAtom, myAccessAtom } from "@/atoms/members/members-query.atoms"; +import { currentUserAtom } from "@/atoms/user/user-query.atoms"; +import type { + Comment, + CommentReply, + Author, +} from "@/contracts/types/chat-comments.types"; +import type { Membership } from "@/contracts/types/members.types"; +import type { SyncHandle } from "@/lib/electric/client"; +import { useElectricClient } from "@/lib/electric/context"; +import { cacheKeys } from "@/lib/query-client/cache-keys"; + +// Raw comment from PGlite local database +interface RawCommentRow { + id: number; + message_id: number; + thread_id: number; + parent_id: number | null; + author_id: string | null; + content: string; + created_at: string; + updated_at: string; +} + +// Regex pattern to match @[uuid] mentions (matches backend MENTION_PATTERN) +const MENTION_PATTERN = /@\[([0-9a-fA-F-]{36})\]/g; + +type MemberInfo = Pick; + +/** + * Render mentions in content by replacing @[uuid] with @{DisplayName} + */ +function renderMentions(content: string, memberMap: Map): string { + return content.replace(MENTION_PATTERN, (match, uuid) => { + const member = memberMap.get(uuid); + if (member?.user_display_name) { + return `@{${member.user_display_name}}`; + } + return match; + }); +} + +/** + * Build member lookup map from membersData + */ +function buildMemberMap(membersData: Membership[] | undefined): Map { + const map = new Map(); + if (membersData) { + for (const m of membersData) { + map.set(m.user_id, { + user_display_name: m.user_display_name, + user_avatar_url: m.user_avatar_url, + user_email: m.user_email, + }); + } + } + return map; +} + +/** + * Build author object from member data + */ +function buildAuthor(authorId: string | null, memberMap: Map): Author | null { + if (!authorId) return null; + const m = memberMap.get(authorId); + if (!m) return null; + return { + id: authorId, + display_name: m.user_display_name ?? null, + avatar_url: m.user_avatar_url ?? null, + email: m.user_email ?? "", + }; +} + +/** + * Check if a comment has been edited by comparing timestamps. + * Uses a small threshold to handle precision differences. + */ +function isEdited(createdAt: string, updatedAt: string): boolean { + const created = new Date(createdAt).getTime(); + const updated = new Date(updatedAt).getTime(); + // Consider edited if updated_at is more than 1 second after created_at + return updated - created > 1000; +} + +/** + * Transform raw comment to CommentReply + */ +function transformReply( + raw: RawCommentRow, + memberMap: Map, + currentUserId: string | undefined, + isOwner: boolean +): CommentReply { + return { + id: raw.id, + content: raw.content, + content_rendered: renderMentions(raw.content, memberMap), + author: buildAuthor(raw.author_id, memberMap), + created_at: raw.created_at, + updated_at: raw.updated_at, + is_edited: isEdited(raw.created_at, raw.updated_at), + can_edit: currentUserId === raw.author_id, + can_delete: currentUserId === raw.author_id || isOwner, + }; +} + +/** + * Transform raw comments to Comment with replies + */ +function transformComments( + rawComments: RawCommentRow[], + memberMap: Map, + currentUserId: string | undefined, + isOwner: boolean +): Map { + // Group comments by message_id + const byMessage = new Map }>(); + + for (const raw of rawComments) { + if (!byMessage.has(raw.message_id)) { + byMessage.set(raw.message_id, { topLevel: [], replies: new Map() }); + } + const group = byMessage.get(raw.message_id)!; + + if (raw.parent_id === null) { + group.topLevel.push(raw); + } else { + if (!group.replies.has(raw.parent_id)) { + group.replies.set(raw.parent_id, []); + } + group.replies.get(raw.parent_id)!.push(raw); + } + } + + // Transform to Comment objects grouped by message_id + const result = new Map(); + + for (const [messageId, group] of byMessage) { + const comments: Comment[] = group.topLevel.map((raw) => { + const replies = (group.replies.get(raw.id) || []) + .sort((a, b) => new Date(a.created_at).getTime() - new Date(b.created_at).getTime()) + .map((r) => transformReply(r, memberMap, currentUserId, isOwner)); + + return { + id: raw.id, + message_id: raw.message_id, + content: raw.content, + content_rendered: renderMentions(raw.content, memberMap), + author: buildAuthor(raw.author_id, memberMap), + created_at: raw.created_at, + updated_at: raw.updated_at, + is_edited: isEdited(raw.created_at, raw.updated_at), + can_edit: currentUserId === raw.author_id, + can_delete: currentUserId === raw.author_id || isOwner, + reply_count: replies.length, + replies, + }; + }); + + // Sort by created_at + comments.sort((a, b) => new Date(a.created_at).getTime() - new Date(b.created_at).getTime()); + result.set(messageId, comments); + } + + return result; +} + +/** + * Hook for syncing comments with Electric SQL real-time sync. + * + * Syncs ALL comments for a thread in ONE subscription, then updates + * React Query cache for each message. This avoids N subscriptions for N messages. + * + * @param threadId - The thread ID to sync comments for + */ +export function useCommentsElectric(threadId: number | null) { + const electricClient = useElectricClient(); + const queryClient = useQueryClient(); + + const { data: membersData } = useAtomValue(membersAtom); + const { data: currentUser } = useAtomValue(currentUserAtom); + const { data: myAccess } = useAtomValue(myAccessAtom); + + const memberMap = useMemo(() => buildMemberMap(membersData), [membersData]); + const currentUserId = currentUser?.id; + const isOwner = myAccess?.is_owner ?? false; + + // Use refs for values needed in live query callback to avoid stale closures + const memberMapRef = useRef(memberMap); + const currentUserIdRef = useRef(currentUserId); + const isOwnerRef = useRef(isOwner); + const queryClientRef = useRef(queryClient); + + // Keep refs updated + useEffect(() => { + memberMapRef.current = memberMap; + currentUserIdRef.current = currentUserId; + isOwnerRef.current = isOwner; + queryClientRef.current = queryClient; + }, [memberMap, currentUserId, isOwner, queryClient]); + + const syncHandleRef = useRef(null); + const liveQueryRef = useRef<{ unsubscribe: () => void } | null>(null); + const syncKeyRef = useRef(null); + + // Stable callback that uses refs for fresh values + const updateReactQueryCache = useCallback((rows: RawCommentRow[]) => { + const commentsByMessage = transformComments( + rows, + memberMapRef.current, + currentUserIdRef.current, + isOwnerRef.current + ); + + for (const [messageId, comments] of commentsByMessage) { + const cacheKey = cacheKeys.comments.byMessage(messageId); + queryClientRef.current.setQueryData(cacheKey, { + comments, + total_count: comments.length, + }); + } + }, []); + + useEffect(() => { + if (!threadId || !electricClient) { + return; + } + + const syncKey = `comments_${threadId}`; + if (syncKeyRef.current === syncKey) { + return; + } + + // Capture in local variable for use in async functions + const client = electricClient; + + let mounted = true; + syncKeyRef.current = syncKey; + + async function startSync() { + try { + const handle = await client.syncShape({ + table: "chat_comments", + where: `thread_id = ${threadId}`, + columns: ["id", "message_id", "thread_id", "parent_id", "author_id", "content", "created_at", "updated_at"], + primaryKey: ["id"], + }); + + if (!handle.isUpToDate && handle.initialSyncPromise) { + try { + await Promise.race([ + handle.initialSyncPromise, + new Promise((resolve) => setTimeout(resolve, 3000)), + ]); + } catch { + // Initial sync timeout - continue anyway + } + } + + if (!mounted) { + handle.unsubscribe(); + return; + } + + syncHandleRef.current = handle; + + // Fetch initial comments and update cache + await fetchAndUpdateCache(); + + // Set up live query for real-time updates + await setupLiveQuery(); + } catch { + // Sync failed - will retry on next mount + } + } + + async function fetchAndUpdateCache() { + try { + const result = await client.db.query( + `SELECT id, message_id, thread_id, parent_id, author_id, content, created_at, updated_at + FROM chat_comments + WHERE thread_id = $1 + ORDER BY created_at ASC`, + [threadId] + ); + + if (mounted && result.rows) { + updateReactQueryCache(result.rows); + } + } catch { + // Query failed - data will be fetched from API + } + } + + async function setupLiveQuery() { + try { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const db = client.db as any; + + if (db.live?.query && typeof db.live.query === "function") { + const liveQuery = await db.live.query( + `SELECT id, message_id, thread_id, parent_id, author_id, content, created_at, updated_at + FROM chat_comments + WHERE thread_id = $1 + ORDER BY created_at ASC`, + [threadId] + ); + + if (!mounted) { + liveQuery.unsubscribe?.(); + return; + } + + // Set initial results + if (liveQuery.initialResults?.rows) { + updateReactQueryCache(liveQuery.initialResults.rows); + } else if (liveQuery.rows) { + updateReactQueryCache(liveQuery.rows); + } + + // Subscribe to changes + if (typeof liveQuery.subscribe === "function") { + liveQuery.subscribe((result: { rows: RawCommentRow[] }) => { + if (mounted && result.rows) { + updateReactQueryCache(result.rows); + } + }); + } + + if (typeof liveQuery.unsubscribe === "function") { + liveQueryRef.current = liveQuery; + } + } + } catch { + // Live query setup failed - will use initial fetch only + } + } + + startSync(); + + return () => { + mounted = false; + syncKeyRef.current = null; + + if (syncHandleRef.current) { + syncHandleRef.current.unsubscribe(); + syncHandleRef.current = null; + } + if (liveQueryRef.current) { + liveQueryRef.current.unsubscribe(); + liveQueryRef.current = null; + } + }; + }, [threadId, electricClient, updateReactQueryCache]); +} diff --git a/surfsense_web/hooks/use-comments-live.ts b/surfsense_web/hooks/use-comments-live.ts deleted file mode 100644 index f4d922888..000000000 --- a/surfsense_web/hooks/use-comments-live.ts +++ /dev/null @@ -1,154 +0,0 @@ -"use client"; - -import { useShape } from "@electric-sql/react"; -import { useAtomValue } from "jotai"; -import { useMemo } from "react"; -import { membersAtom, myAccessAtom } from "@/atoms/members/members-query.atoms"; -import { currentUserAtom } from "@/atoms/user/user-query.atoms"; -import type { Comment, CommentReply, Author } from "@/contracts/types/chat-comments.types"; -import type { Membership } from "@/contracts/types/members.types"; -import type { RawComment } from "@/contracts/types/chat-comments.types"; - -const ELECTRIC_URL = process.env.NEXT_PUBLIC_ELECTRIC_URL || "http://localhost:5133"; - -// Regex pattern to match @[uuid] mentions (matches backend MENTION_PATTERN) -const MENTION_PATTERN = /@\[([0-9a-fA-F-]{36})\]/g; - -/** - * Member info for building author objects - derived from Membership - */ -type MemberInfo = Pick; - -/** - * Render mentions in content by replacing @[uuid] with @{DisplayName} - */ -function renderMentions(content: string, memberMap: Map): string { - return content.replace(MENTION_PATTERN, (match, uuid) => { - const member = memberMap.get(uuid); - if (member?.user_display_name) { - return `@{${member.user_display_name}}`; - } - return match; - }); -} - -/** - * Hook to get live comments for a specific message. - * Uses Electric SQL for comments + membersAtom (API) for author info. - * Returns data matching the existing Comment type. - */ -export function useCommentsLive(messageId: number | null) { - const { - data: commentsData, - isLoading: commentsLoading, - isError: commentsError, - error: commentsErrorDetails, - } = useShape({ - url: `${ELECTRIC_URL}/v1/shape`, - params: { - table: "chat_comments", - where: `message_id = ${messageId}`, - }, - }); - - const { data: membersData, isLoading: membersLoading } = useAtomValue(membersAtom); - const { data: currentUser } = useAtomValue(currentUserAtom); - const { data: myAccess } = useAtomValue(myAccessAtom); - - const comments = useMemo(() => { - if (!commentsData) return []; - - // Build member lookup map - const memberMap = new Map(); - if (membersData) { - for (const member of membersData) { - memberMap.set(member.user_id, { - user_display_name: member.user_display_name, - user_avatar_url: member.user_avatar_url, - user_email: member.user_email, - }); - } - } - - const currentUserId = currentUser?.id; - const isOwnerOrAdmin = myAccess?.is_owner ?? false; - - // Build author object from member data - const buildAuthor = (authorId: string | null): Author | null => { - if (!authorId) return null; - const member = memberMap.get(authorId); - if (!member) return null; - return { - id: authorId, - display_name: member.user_display_name ?? null, - avatar_url: member.user_avatar_url ?? null, - email: member.user_email ?? "", - }; - }; - - // Transform raw comment to CommentReply - const transformToReply = (raw: RawComment): CommentReply => { - const isEdited = raw.created_at !== raw.updated_at; - const isAuthor = currentUserId === raw.author_id; - - return { - id: raw.id, - content: raw.content, - content_rendered: renderMentions(raw.content, memberMap), - author: buildAuthor(raw.author_id), - created_at: raw.created_at, - updated_at: raw.updated_at, - is_edited: isEdited, - can_edit: isAuthor, - can_delete: isAuthor || isOwnerOrAdmin, - }; - }; - - // Separate top-level comments and replies - const topLevelRaw: RawComment[] = []; - const repliesMap = new Map(); - - for (const raw of commentsData) { - if (raw.parent_id === null) { - topLevelRaw.push(raw); - } else { - const replies = repliesMap.get(raw.parent_id) || []; - replies.push(raw); - repliesMap.set(raw.parent_id, replies); - } - } - - // Transform top-level comments to Comment type - const transformToComment = (raw: RawComment): Comment => { - const isEdited = raw.created_at !== raw.updated_at; - const isAuthor = currentUserId === raw.author_id; - const rawReplies = repliesMap.get(raw.id) || []; - const replies = rawReplies.map(transformToReply); - - return { - id: raw.id, - message_id: raw.message_id, - content: raw.content, - content_rendered: renderMentions(raw.content, memberMap), - author: buildAuthor(raw.author_id), - created_at: raw.created_at, - updated_at: raw.updated_at, - is_edited: isEdited, - can_edit: isAuthor, - can_delete: isAuthor || isOwnerOrAdmin, - reply_count: replies.length, - replies, - }; - }; - - return topLevelRaw.map(transformToComment); - }, [commentsData, membersData, currentUser?.id, myAccess?.is_owner]); - - return { - comments, - commentCount: commentsData?.length ?? 0, - isLoading: commentsLoading || membersLoading, - isError: commentsError, - error: commentsError ? commentsErrorDetails : null, - }; -} From 12437f840a656a65a358bce7c1f6be270aebd99d Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 22 Jan 2026 18:43:20 +0200 Subject: [PATCH 23/44] Add new_chat_messages table to PGlite and create useMessagesElectric hook --- .../contracts/types/chat-messages.types.ts | 1 - surfsense_web/hooks/use-chat-messages-live.ts | 82 ---------- surfsense_web/hooks/use-messages-electric.ts | 154 ++++++++++++++++++ surfsense_web/lib/electric/client.ts | 15 ++ 4 files changed, 169 insertions(+), 83 deletions(-) delete mode 100644 surfsense_web/hooks/use-chat-messages-live.ts create mode 100644 surfsense_web/hooks/use-messages-electric.ts diff --git a/surfsense_web/contracts/types/chat-messages.types.ts b/surfsense_web/contracts/types/chat-messages.types.ts index faba71bff..78bf7b043 100644 --- a/surfsense_web/contracts/types/chat-messages.types.ts +++ b/surfsense_web/contracts/types/chat-messages.types.ts @@ -10,7 +10,6 @@ export const rawMessage = z.object({ content: z.unknown(), author_id: z.string().nullable(), created_at: z.string(), - updated_at: z.string(), }); export type RawMessage = z.infer; diff --git a/surfsense_web/hooks/use-chat-messages-live.ts b/surfsense_web/hooks/use-chat-messages-live.ts deleted file mode 100644 index 4a8ae97e6..000000000 --- a/surfsense_web/hooks/use-chat-messages-live.ts +++ /dev/null @@ -1,82 +0,0 @@ -"use client"; - -import { useShape } from "@electric-sql/react"; -import { useAtomValue } from "jotai"; -import { useMemo } from "react"; -import { membersAtom } from "@/atoms/members/members-query.atoms"; -import type { RawMessage } from "@/contracts/types/chat-messages.types"; -import type { Membership } from "@/contracts/types/members.types"; -import type { MessageRecord } from "@/lib/chat/thread-persistence"; - -const ELECTRIC_URL = process.env.NEXT_PUBLIC_ELECTRIC_URL || "http://localhost:5133"; - -/** - * Member info for building author data - derived from Membership - */ -type MemberInfo = Pick; - -/** - * Hook to get live chat messages for real-time sync. - * Uses Electric SQL for messages + membersAtom (API) for author info. - */ -export function useChatMessagesLive(threadId: number | null) { - - const { - data: messagesData, - isLoading: messagesLoading, - isError: messagesError, - error: messagesErrorDetails, - } = useShape({ - url: `${ELECTRIC_URL}/v1/shape`, - params: { - table: "new_chat_messages", - where: `thread_id = ${threadId}`, - }, - }); - - - const { data: membersData, isLoading: membersLoading } = useAtomValue(membersAtom); - - - const messages = useMemo(() => { - if (!messagesData) return []; - - // Build member lookup map - const memberMap = new Map(); - if (membersData) { - for (const member of membersData) { - memberMap.set(member.user_id, { - user_display_name: member.user_display_name, - user_avatar_url: member.user_avatar_url, - }); - } - } - - // Transform raw messages to MessageRecord with author info - return [...messagesData].map((msg): MessageRecord => { - const author = msg.author_id ? memberMap.get(msg.author_id) : null; - - const role = (typeof msg.role === "string" ? msg.role.toLowerCase() : msg.role) as - | "user" - | "assistant" - | "system"; - return { - id: msg.id, - thread_id: msg.thread_id, - role, - content: msg.content, - created_at: msg.created_at, - author_id: msg.author_id, - author_display_name: author?.user_display_name ?? null, - author_avatar_url: author?.user_avatar_url ?? null, - }; - }); - }, [messagesData, membersData]); - - return { - messages, - isLoading: messagesLoading || membersLoading, - isError: messagesError, - error: messagesError ? messagesErrorDetails : null, - }; -} diff --git a/surfsense_web/hooks/use-messages-electric.ts b/surfsense_web/hooks/use-messages-electric.ts new file mode 100644 index 000000000..e8c82e92b --- /dev/null +++ b/surfsense_web/hooks/use-messages-electric.ts @@ -0,0 +1,154 @@ +"use client"; + +import { useCallback, useEffect, useRef } from "react"; +import type { RawMessage } from "@/contracts/types/chat-messages.types"; +import type { SyncHandle } from "@/lib/electric/client"; +import { useElectricClient } from "@/lib/electric/context"; + +/** + * Syncs chat messages for a thread via Electric SQL. + * Calls onMessagesUpdate when messages change. + */ +export function useMessagesElectric( + threadId: number | null, + onMessagesUpdate: (messages: RawMessage[]) => void +) { + const electricClient = useElectricClient(); + + const syncHandleRef = useRef(null); + const liveQueryRef = useRef<{ unsubscribe: () => void } | null>(null); + const syncKeyRef = useRef(null); + const onMessagesUpdateRef = useRef(onMessagesUpdate); + + useEffect(() => { + onMessagesUpdateRef.current = onMessagesUpdate; + }, [onMessagesUpdate]); + + const handleMessagesUpdate = useCallback((rows: RawMessage[]) => { + onMessagesUpdateRef.current(rows); + }, []); + + useEffect(() => { + if (!threadId || !electricClient) { + return; + } + + const syncKey = `messages_${threadId}`; + if (syncKeyRef.current === syncKey) { + return; + } + + const client = electricClient; + let mounted = true; + syncKeyRef.current = syncKey; + + async function startSync() { + try { + const handle = await client.syncShape({ + table: "new_chat_messages", + where: `thread_id = ${threadId}`, + columns: ["id", "thread_id", "role", "content", "author_id", "created_at"], + primaryKey: ["id"], + }); + + if (!handle.isUpToDate && handle.initialSyncPromise) { + try { + await Promise.race([ + handle.initialSyncPromise, + new Promise((resolve) => setTimeout(resolve, 3000)), + ]); + } catch { + // Timeout + } + } + + if (!mounted) { + handle.unsubscribe(); + return; + } + + syncHandleRef.current = handle; + await fetchMessages(); + await setupLiveQuery(); + } catch { + // Sync failed + } + } + + async function fetchMessages() { + try { + const result = await client.db.query( + `SELECT id, thread_id, role, content, author_id, created_at + FROM new_chat_messages + WHERE thread_id = $1 + ORDER BY created_at ASC`, + [threadId] + ); + + if (mounted && result.rows) { + handleMessagesUpdate(result.rows); + } + } catch { + // Query failed + } + } + + async function setupLiveQuery() { + try { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const db = client.db as any; + + if (db.live?.query && typeof db.live.query === "function") { + const liveQuery = await db.live.query( + `SELECT id, thread_id, role, content, author_id, created_at + FROM new_chat_messages + WHERE thread_id = $1 + ORDER BY created_at ASC`, + [threadId] + ); + + if (!mounted) { + liveQuery.unsubscribe?.(); + return; + } + + if (liveQuery.initialResults?.rows) { + handleMessagesUpdate(liveQuery.initialResults.rows); + } else if (liveQuery.rows) { + handleMessagesUpdate(liveQuery.rows); + } + + if (typeof liveQuery.subscribe === "function") { + liveQuery.subscribe((result: { rows: RawMessage[] }) => { + if (mounted && result.rows) { + handleMessagesUpdate(result.rows); + } + }); + } + + if (typeof liveQuery.unsubscribe === "function") { + liveQueryRef.current = liveQuery; + } + } + } catch { + // Live query failed + } + } + + startSync(); + + return () => { + mounted = false; + syncKeyRef.current = null; + + if (syncHandleRef.current) { + syncHandleRef.current.unsubscribe(); + syncHandleRef.current = null; + } + if (liveQueryRef.current) { + liveQueryRef.current.unsubscribe(); + liveQueryRef.current = null; + } + }; + }, [threadId, electricClient, handleMessagesUpdate]); +} diff --git a/surfsense_web/lib/electric/client.ts b/surfsense_web/lib/electric/client.ts index 84df6c905..479026120 100644 --- a/surfsense_web/lib/electric/client.ts +++ b/surfsense_web/lib/electric/client.ts @@ -258,6 +258,21 @@ export async function initElectric(userId: string): Promise { CREATE INDEX IF NOT EXISTS idx_chat_comments_parent_id ON chat_comments(parent_id); `); + // Create new_chat_messages table for live message sync + await db.exec(` + CREATE TABLE IF NOT EXISTS new_chat_messages ( + id INTEGER PRIMARY KEY, + thread_id INTEGER NOT NULL, + role TEXT NOT NULL, + content JSONB NOT NULL, + author_id TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + ); + + CREATE INDEX IF NOT EXISTS idx_new_chat_messages_thread_id ON new_chat_messages(thread_id); + CREATE INDEX IF NOT EXISTS idx_new_chat_messages_created_at ON new_chat_messages(created_at); + `); + const electricUrl = getElectricUrl(); // STEP 4: Create the client wrapper From 4cb835f19fd4708f7a0ab8687de394d3809df6a6 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 22 Jan 2026 18:43:32 +0200 Subject: [PATCH 24/44] Integrate live message sync for shared chat collaboration --- .../new-chat/[[...chat_id]]/page.tsx | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx index 43c33ba5a..cb6e797bd 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx @@ -24,6 +24,7 @@ import { // extractWriteTodosFromContent, hydratePlanStateAtom, } from "@/atoms/chat/plan-state.atom"; +import { membersAtom } from "@/atoms/members/members-query.atoms"; import { currentUserAtom } from "@/atoms/user/user-query.atoms"; import { Thread } from "@/components/assistant-ui/thread"; import { ChatHeader } from "@/components/new-chat/chat-header"; @@ -49,6 +50,8 @@ import { type MessageRecord, type ThreadRecord, } from "@/lib/chat/thread-persistence"; +import { useChatSessionState } from "@/hooks/use-chat-session-state"; +import { useMessagesElectric } from "@/hooks/use-messages-electric"; import { trackChatCreated, trackChatError, @@ -257,6 +260,59 @@ export default function NewChatPage() { // Get current user for author info in shared chats const { data: currentUser } = useAtomValue(currentUserAtom); + // Live collaboration: sync messages from other users via Electric SQL + const { isAiResponding, respondingToUserId } = useChatSessionState(threadId); + const { data: membersData } = useAtomValue(membersAtom); + + const handleElectricMessagesUpdate = useCallback( + (electricMessages: { id: number; thread_id: number; role: string; content: unknown; author_id: string | null; created_at: string }[]) => { + // Skip sync while AI is responding to us or during local streaming + if ((isAiResponding && respondingToUserId === currentUser?.id) || isRunning) { + return; + } + + setMessages((prev) => { + const existingIds = new Set( + prev.map((m) => { + const match = String(m.id).match(/^msg-(\d+)$/); + return match ? Number.parseInt(match[1], 10) : null; + }).filter((id): id is number => id !== null) + ); + + const newConverted: ReturnType[] = []; + for (const msg of electricMessages) { + if (existingIds.has(msg.id)) continue; + + const member = msg.author_id + ? membersData?.find((m) => m.user_id === msg.author_id) + : null; + + newConverted.push( + convertToThreadMessage({ + id: msg.id, + thread_id: msg.thread_id, + role: msg.role.toLowerCase() as "user" | "assistant" | "system", + content: msg.content, + author_id: msg.author_id, + created_at: msg.created_at, + author_display_name: member?.user_display_name ?? null, + author_avatar_url: member?.user_avatar_url ?? null, + }) + ); + } + + if (newConverted.length === 0) { + return prev; + } + + return [...prev, ...newConverted]; + }); + }, + [isAiResponding, respondingToUserId, currentUser?.id, isRunning, membersData] + ); + + useMessagesElectric(threadId, handleElectricMessagesUpdate); + // Create the attachment adapter for file processing const attachmentAdapter = useMemo(() => createAttachmentAdapter(), []); From 39d434b00c32ef366a693926e40063cc503697ac Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 22 Jan 2026 19:04:23 +0200 Subject: [PATCH 25/44] Centralize chat session state sync with single subscription via atom --- .../new-chat/[[...chat_id]]/page.tsx | 10 ++++-- .../atoms/chat/chat-session-state.atom.ts | 15 ++++++++ .../components/assistant-ui/thread.tsx | 6 ++-- surfsense_web/hooks/use-chat-session-state.ts | 35 ++++++++++++------- 4 files changed, 48 insertions(+), 18 deletions(-) create mode 100644 surfsense_web/atoms/chat/chat-session-state.atom.ts diff --git a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx index cb6e797bd..e76b83a97 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx @@ -50,7 +50,8 @@ import { type MessageRecord, type ThreadRecord, } from "@/lib/chat/thread-persistence"; -import { useChatSessionState } from "@/hooks/use-chat-session-state"; +import { chatSessionStateAtom } from "@/atoms/chat/chat-session-state.atom"; +import { useChatSessionStateSync } from "@/hooks/use-chat-session-state"; import { useMessagesElectric } from "@/hooks/use-messages-electric"; import { trackChatCreated, @@ -260,8 +261,11 @@ export default function NewChatPage() { // Get current user for author info in shared chats const { data: currentUser } = useAtomValue(currentUserAtom); - // Live collaboration: sync messages from other users via Electric SQL - const { isAiResponding, respondingToUserId } = useChatSessionState(threadId); + // Live collaboration: sync session state and messages via Electric SQL + useChatSessionStateSync(threadId); + const sessionState = useAtomValue(chatSessionStateAtom); + const isAiResponding = sessionState?.isAiResponding ?? false; + const respondingToUserId = sessionState?.respondingToUserId ?? null; const { data: membersData } = useAtomValue(membersAtom); const handleElectricMessagesUpdate = useCallback( diff --git a/surfsense_web/atoms/chat/chat-session-state.atom.ts b/surfsense_web/atoms/chat/chat-session-state.atom.ts new file mode 100644 index 000000000..4d83a45d4 --- /dev/null +++ b/surfsense_web/atoms/chat/chat-session-state.atom.ts @@ -0,0 +1,15 @@ +"use client"; + +import { atom } from "jotai"; + +export interface ChatSessionStateData { + threadId: number; + isAiResponding: boolean; + respondingToUserId: string | null; +} + +/** + * Global chat session state atom. + * Updated by useChatSessionStateSync hook, read anywhere. + */ +export const chatSessionStateAtom = atom(null); diff --git a/surfsense_web/components/assistant-ui/thread.tsx b/surfsense_web/components/assistant-ui/thread.tsx index e1169867c..e419258f2 100644 --- a/surfsense_web/components/assistant-ui/thread.tsx +++ b/surfsense_web/components/assistant-ui/thread.tsx @@ -61,7 +61,7 @@ import { import type { ThinkingStep } from "@/components/tool-ui/deepagent-thinking"; import { Button } from "@/components/ui/button"; import type { Document } from "@/contracts/types/document.types"; -import { useChatSessionState } from "@/hooks/use-chat-session-state"; +import { chatSessionStateAtom } from "@/atoms/chat/chat-session-state.atom"; import { useCommentsElectric } from "@/hooks/use-comments-electric"; import { cn } from "@/lib/utils"; @@ -236,7 +236,9 @@ const Composer: FC = () => { } return typeof chat_id === "string" ? Number.parseInt(chat_id, 10) || null : null; }, [chat_id]); - const { isAiResponding, respondingToUserId } = useChatSessionState(threadId); + const sessionState = useAtomValue(chatSessionStateAtom); + const isAiResponding = sessionState?.isAiResponding ?? false; + const respondingToUserId = sessionState?.respondingToUserId ?? null; const isBlockedByOtherUser = isAiResponding && respondingToUserId !== currentUser?.id; // Sync comments for the entire thread via Electric SQL (one subscription per thread) diff --git a/surfsense_web/hooks/use-chat-session-state.ts b/surfsense_web/hooks/use-chat-session-state.ts index fb263826f..f3bdd7722 100644 --- a/surfsense_web/hooks/use-chat-session-state.ts +++ b/surfsense_web/hooks/use-chat-session-state.ts @@ -1,30 +1,39 @@ "use client"; import { useShape } from "@electric-sql/react"; +import { useSetAtom } from "jotai"; +import { useEffect } from "react"; +import { chatSessionStateAtom } from "@/atoms/chat/chat-session-state.atom"; import type { ChatSessionState } from "@/contracts/types/chat-session-state.types"; const ELECTRIC_URL = process.env.NEXT_PUBLIC_ELECTRIC_URL || "http://localhost:5133"; /** - * Hook to get live chat session state for collaboration. - * Tracks which user the AI is currently responding to. + * Syncs chat session state for a thread via Electric SQL. + * Call once per thread (in page.tsx). Updates global atom. */ -export function useChatSessionState(threadId: number | null) { - const { data, isLoading, isError, error } = useShape({ +export function useChatSessionStateSync(threadId: number | null) { + const setSessionState = useSetAtom(chatSessionStateAtom); + + const { data } = useShape({ url: `${ELECTRIC_URL}/v1/shape`, params: { table: "chat_session_state", - where: `thread_id = ${threadId}`, + where: `thread_id = ${threadId ?? -1}`, }, }); - const sessionState = data?.[0] ?? null; + useEffect(() => { + if (!threadId) { + setSessionState(null); + return; + } - return { - sessionState, - isAiResponding: !!sessionState?.ai_responding_to_user_id, - respondingToUserId: sessionState?.ai_responding_to_user_id ?? null, - loading: isLoading, - error: isError ? error : null, - }; + const row = data?.[0]; + setSessionState({ + threadId, + isAiResponding: !!row?.ai_responding_to_user_id, + respondingToUserId: row?.ai_responding_to_user_id ?? null, + }); + }, [threadId, data, setSessionState]); } From 6c5223e2621131b63b7f3282aaf93b3f98c204da Mon Sep 17 00:00:00 2001 From: Eric Lammertsma Date: Thu, 22 Jan 2026 12:26:37 -0500 Subject: [PATCH 26/44] Made numerous minor layout changes to main layout and sidebar --- .../dashboard/[search_space_id]/layout.tsx | 6 ---- .../components/dashboard-breadcrumb.tsx | 5 +-- .../layout/providers/LayoutDataProvider.tsx | 8 +---- .../components/layout/ui/sidebar/Sidebar.tsx | 10 +++--- .../layout/ui/sidebar/SidebarHeader.tsx | 8 ++++- .../components/new-chat/model-selector.tsx | 33 +++++++++++-------- surfsense_web/messages/en.json | 1 + surfsense_web/messages/zh.json | 1 + 8 files changed, 36 insertions(+), 36 deletions(-) diff --git a/surfsense_web/app/dashboard/[search_space_id]/layout.tsx b/surfsense_web/app/dashboard/[search_space_id]/layout.tsx index 1631f00b9..e434d85fb 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/layout.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/layout.tsx @@ -39,12 +39,6 @@ export default function DashboardLayout({ icon: "SquareLibrary", items: [], }, - { - title: "Logs", - url: `/dashboard/${search_space_id}/logs`, - icon: "Logs", - items: [], - }, ]; return ( diff --git a/surfsense_web/components/dashboard-breadcrumb.tsx b/surfsense_web/components/dashboard-breadcrumb.tsx index 0e9374fdd..223c4764a 100644 --- a/surfsense_web/components/dashboard-breadcrumb.tsx +++ b/surfsense_web/components/dashboard-breadcrumb.tsx @@ -76,9 +76,6 @@ export function DashboardBreadcrumb() { const segments = path.split("/").filter(Boolean); const breadcrumbs: BreadcrumbItemInterface[] = []; - // Always start with Dashboard - breadcrumbs.push({ label: t("dashboard"), href: "/dashboard" }); - // Handle search space if (segments[0] === "dashboard" && segments[1]) { // Use the actual search space name if available, otherwise fall back to the ID @@ -182,7 +179,7 @@ export function DashboardBreadcrumb() { const breadcrumbs = generateBreadcrumbs(pathname); - if (breadcrumbs.length <= 1) { + if (breadcrumbs.length === 0) { return null; // Don't show breadcrumbs for root dashboard } diff --git a/surfsense_web/components/layout/providers/LayoutDataProvider.tsx b/surfsense_web/components/layout/providers/LayoutDataProvider.tsx index 7f55e295f..f382aa395 100644 --- a/surfsense_web/components/layout/providers/LayoutDataProvider.tsx +++ b/surfsense_web/components/layout/providers/LayoutDataProvider.tsx @@ -2,7 +2,7 @@ import { useQuery, useQueryClient } from "@tanstack/react-query"; import { useAtomValue } from "jotai"; -import { LogOut, Logs, SquareLibrary, Trash2 } from "lucide-react"; +import { LogOut, SquareLibrary, Trash2 } from "lucide-react"; import { useParams, usePathname, useRouter } from "next/navigation"; import { useTranslations } from "next-intl"; import { useTheme } from "next-themes"; @@ -151,12 +151,6 @@ export function LayoutDataProvider({ icon: SquareLibrary, isActive: pathname?.includes("/documents"), }, - { - title: "Logs", - url: `/dashboard/${searchSpaceId}/logs`, - icon: Logs, - isActive: pathname?.includes("/logs"), - }, ], [searchSpaceId, pathname] ); diff --git a/surfsense_web/components/layout/ui/sidebar/Sidebar.tsx b/surfsense_web/components/layout/ui/sidebar/Sidebar.tsx index 9a632506c..d87bf61e6 100644 --- a/surfsense_web/components/layout/ui/sidebar/Sidebar.tsx +++ b/surfsense_web/components/layout/ui/sidebar/Sidebar.tsx @@ -95,6 +95,11 @@ export function Sidebar({ )} + {/* Platform navigation */} + {navItems.length > 0 && ( + + )} + {/* New chat button */}
{isCollapsed ? ( @@ -115,11 +120,6 @@ export function Sidebar({ )}
- {/* Platform navigation */} - {navItems.length > 0 && ( - - )} - {/* Scrollable content */} {isCollapsed ? ( diff --git a/surfsense_web/components/layout/ui/sidebar/SidebarHeader.tsx b/surfsense_web/components/layout/ui/sidebar/SidebarHeader.tsx index 6ca057819..4ed212b47 100644 --- a/surfsense_web/components/layout/ui/sidebar/SidebarHeader.tsx +++ b/surfsense_web/components/layout/ui/sidebar/SidebarHeader.tsx @@ -1,6 +1,7 @@ "use client"; -import { ChevronsUpDown, Settings, Users } from "lucide-react"; +import { ChevronsUpDown, ScrollText, Settings, Users } from "lucide-react"; +import { useRouter } from "next/navigation"; import { useTranslations } from "next-intl"; import { Button } from "@/components/ui/button"; import { @@ -29,6 +30,7 @@ export function SidebarHeader({ className, }: SidebarHeaderProps) { const t = useTranslations("sidebar"); + const router = useRouter(); return (
@@ -52,6 +54,10 @@ export function SidebarHeader({ {t("manage_members")} + router.push(`/dashboard/${searchSpace?.id}/logs`)}> + + {t("logs")} + diff --git a/surfsense_web/components/new-chat/model-selector.tsx b/surfsense_web/components/new-chat/model-selector.tsx index ead378e86..5693c0b15 100644 --- a/surfsense_web/components/new-chat/model-selector.tsx +++ b/surfsense_web/components/new-chat/model-selector.tsx @@ -124,6 +124,11 @@ export function ModelSelector({ onEdit, onAddNew, className }: ModelSelectorProp ); }, [userConfigs, searchQuery]); + // Total model count for conditional search display + const totalModels = useMemo(() => { + return (globalConfigs?.length ?? 0) + (userConfigs?.length ?? 0); + }, [globalConfigs, userConfigs]); + const handleSelectConfig = useCallback( async (config: NewLLMConfigPublic | GlobalNewLLMConfig) => { // If already selected, just close @@ -230,15 +235,17 @@ export function ModelSelector({ onEdit, onAddNew, className }: ModelSelectorProp
)} -
- -
+ {totalModels > 3 && ( +
+ +
+ )} @@ -264,7 +271,7 @@ export function ModelSelector({ onEdit, onAddNew, className }: ModelSelectorProp value={`global-${config.id}`} onSelect={() => handleSelectConfig(config)} className={cn( - "mx-2 rounded-lg mb-1 cursor-pointer", + "mx-2 rounded-lg mb-1 cursor-pointer group", "aria-selected:bg-accent/50", isSelected && "bg-accent/80" )} @@ -295,7 +302,7 @@ export function ModelSelector({ onEdit, onAddNew, className }: ModelSelectorProp