Merge remote-tracking branch 'upstream/dev' into feat/replace-logs

surfsense_backend/alembic/versions/60_add_surfsense_docs_tables.py

@@ -0,0 +1,163 @@
+"""Add Surfsense docs tables for global documentation storage
+
+Revision ID: 60
+Revises: 59
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+from app.config import config
+
+# revision identifiers, used by Alembic.
+revision: str = "60"
+down_revision: str | None = "59"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+# Get embedding dimension from config
+EMBEDDING_DIM = config.embedding_model_instance.dimension
+
+
+def upgrade() -> None:
+    """Create surfsense_docs_documents and surfsense_docs_chunks tables."""
+
+    # Create surfsense_docs_documents table
+    op.execute(
+        f"""
+        DO $$
+        BEGIN
+            IF NOT EXISTS (
+                SELECT FROM information_schema.tables
+                WHERE table_name = 'surfsense_docs_documents'
+            ) THEN
+                CREATE TABLE surfsense_docs_documents (
+                    id SERIAL PRIMARY KEY,
+                    created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
+                    source VARCHAR NOT NULL UNIQUE,
+                    title VARCHAR NOT NULL,
+                    content TEXT NOT NULL,
+                    content_hash VARCHAR NOT NULL,
+                    embedding vector({EMBEDDING_DIM}),
+                    updated_at TIMESTAMP WITH TIME ZONE
+                );
+            END IF;
+        END$$;
+        """
+    )
+
+    # Create indexes for surfsense_docs_documents
+    op.execute(
+        """
+        DO $$
+        BEGIN
+            IF NOT EXISTS (
+                SELECT 1 FROM pg_indexes
+                WHERE tablename = 'surfsense_docs_documents' AND indexname = 'ix_surfsense_docs_documents_source'
+            ) THEN
+                CREATE INDEX ix_surfsense_docs_documents_source ON surfsense_docs_documents(source);
+            END IF;
+
+            IF NOT EXISTS (
+                SELECT 1 FROM pg_indexes
+                WHERE tablename = 'surfsense_docs_documents' AND indexname = 'ix_surfsense_docs_documents_content_hash'
+            ) THEN
+                CREATE INDEX ix_surfsense_docs_documents_content_hash ON surfsense_docs_documents(content_hash);
+            END IF;
+
+            IF NOT EXISTS (
+                SELECT 1 FROM pg_indexes
+                WHERE tablename = 'surfsense_docs_documents' AND indexname = 'ix_surfsense_docs_documents_updated_at'
+            ) THEN
+                CREATE INDEX ix_surfsense_docs_documents_updated_at ON surfsense_docs_documents(updated_at);
+            END IF;
+        END$$;
+        """
+    )
+
+    # Create surfsense_docs_chunks table
+    op.execute(
+        f"""
+        DO $$
+        BEGIN
+            IF NOT EXISTS (
+                SELECT FROM information_schema.tables
+                WHERE table_name = 'surfsense_docs_chunks'
+            ) THEN
+                CREATE TABLE surfsense_docs_chunks (
+                    id SERIAL PRIMARY KEY,
+                    created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
+                    content TEXT NOT NULL,
+                    embedding vector({EMBEDDING_DIM}),
+                    document_id INTEGER NOT NULL REFERENCES surfsense_docs_documents(id) ON DELETE CASCADE
+                );
+            END IF;
+        END$$;
+        """
+    )
+
+    # Create indexes for surfsense_docs_chunks
+    op.execute(
+        """
+        DO $$
+        BEGIN
+            IF NOT EXISTS (
+                SELECT 1 FROM pg_indexes
+                WHERE tablename = 'surfsense_docs_chunks' AND indexname = 'ix_surfsense_docs_chunks_document_id'
+            ) THEN
+                CREATE INDEX ix_surfsense_docs_chunks_document_id ON surfsense_docs_chunks(document_id);
+            END IF;
+        END$$;
+        """
+    )
+
+    # Create vector indexes for similarity search
+    op.execute(
+        """
+        CREATE INDEX IF NOT EXISTS surfsense_docs_documents_vector_index 
+        ON surfsense_docs_documents USING hnsw (embedding public.vector_cosine_ops);
+        """
+    )
+
+    op.execute(
+        """
+        CREATE INDEX IF NOT EXISTS surfsense_docs_chunks_vector_index 
+        ON surfsense_docs_chunks USING hnsw (embedding public.vector_cosine_ops);
+        """
+    )
+
+    # Create full-text search indexes (same pattern as documents/chunks tables)
+    op.execute(
+        """
+        CREATE INDEX IF NOT EXISTS surfsense_docs_documents_search_index 
+        ON surfsense_docs_documents USING gin (to_tsvector('english', content));
+        """
+    )
+
+    op.execute(
+        """
+        CREATE INDEX IF NOT EXISTS surfsense_docs_chunks_search_index 
+        ON surfsense_docs_chunks USING gin (to_tsvector('english', content));
+        """
+    )
+
+
+def downgrade() -> None:
+    """Remove surfsense docs tables."""
+    # Drop full-text search indexes
+    op.execute("DROP INDEX IF EXISTS surfsense_docs_chunks_search_index")
+    op.execute("DROP INDEX IF EXISTS surfsense_docs_documents_search_index")
+
+    # Drop vector indexes
+    op.execute("DROP INDEX IF EXISTS surfsense_docs_chunks_vector_index")
+    op.execute("DROP INDEX IF EXISTS surfsense_docs_documents_vector_index")
+
+    # Drop regular indexes
+    op.execute("DROP INDEX IF EXISTS ix_surfsense_docs_chunks_document_id")
+    op.execute("DROP INDEX IF EXISTS ix_surfsense_docs_documents_updated_at")
+    op.execute("DROP INDEX IF EXISTS ix_surfsense_docs_documents_content_hash")
+    op.execute("DROP INDEX IF EXISTS ix_surfsense_docs_documents_source")
+
+    # Drop tables (chunks first due to FK)
+    op.execute("DROP TABLE IF EXISTS surfsense_docs_chunks")
+    op.execute("DROP TABLE IF EXISTS surfsense_docs_documents")

surfsense_backend/app/agents/new_chat/system_prompt.py

@@ -26,6 +26,13 @@ SURFSENSE_TOOLS_INSTRUCTIONS = """
 <tools>
 You have access to the following tools:
 
+0. search_surfsense_docs: Search the official SurfSense documentation.
+  - Use this tool when the user asks anything about SurfSense itself (the application they are using).
+  - Args:
+    - query: The search query about SurfSense
+    - top_k: Number of documentation chunks to retrieve (default: 10)
+  - Returns: Documentation content with chunk IDs for citations (prefixed with 'doc-', e.g., [citation:doc-123])
+
 1. search_knowledge_base: Search the user's personal knowledge base for relevant information.
   - Args:
     - query: The search query - be specific and include key terms
@@ -152,6 +159,18 @@ You have access to the following tools:
   - Airtable/Notion: Check field values, apply mapping above
 </tools>
 <tool_call_examples>
+- User: "How do I install SurfSense?"
+  - Call: `search_surfsense_docs(query="installation setup")`
+
+- User: "What connectors does SurfSense support?"
+  - Call: `search_surfsense_docs(query="available connectors integrations")`
+
+- User: "How do I set up the Notion connector?"
+  - Call: `search_surfsense_docs(query="Notion connector setup configuration")`
+
+- User: "How do I use Docker to run SurfSense?"
+  - Call: `search_surfsense_docs(query="Docker installation setup")`
+
 - User: "Fetch all my notes and what's in them?"
   - Call: `search_knowledge_base(query="*", top_k=50, connectors_to_search=["NOTE"])`
 
@@ -308,7 +327,7 @@ The documents you receive are structured like this:
 </document_content>
 </document>
 
-IMPORTANT: You MUST cite using the chunk ids (e.g. 123, 124). Do NOT cite document_id.
+IMPORTANT: You MUST cite using the chunk ids (e.g. 123, 124, doc-45). Do NOT cite document_id.
 </document_structure_example>
 
 <citation_format>
@@ -319,11 +338,13 @@ IMPORTANT: You MUST cite using the chunk ids (e.g. 123, 124). Do NOT cite docume
 - NEVER create your own citation format - use the exact chunk_id values from the documents in the [citation:chunk_id] format
 - NEVER format citations as clickable links or as markdown links like "([citation:5](https://example.com))". Always use plain square brackets only
 - NEVER make up chunk IDs if you are unsure about the chunk_id. It is better to omit the citation than to guess
+- Copy the EXACT chunk id from the XML - if it says `<chunk id='doc-123'>`, use [citation:doc-123]
 </citation_format>
 
 <citation_examples>
 CORRECT citation formats:
 - [citation:5]
+- [citation:doc-123] (for Surfsense documentation chunks)
 - [citation:chunk_id1], [citation:chunk_id2], [citation:chunk_id3]
 
 INCORRECT citation formats (DO NOT use):

surfsense_backend/app/agents/new_chat/tools/__init__.py

@@ -6,6 +6,7 @@ To add a new tool, see the documentation in registry.py.
 
 Available tools:
 - search_knowledge_base: Search the user's personal knowledge base
+- search_surfsense_docs: Search Surfsense documentation for usage help
 - generate_podcast: Generate audio podcasts from content
 - link_preview: Fetch rich previews for URLs
 - display_image: Display images in chat
@@ -31,6 +32,7 @@ from .registry import (
     get_tool_by_name,
 )
 from .scrape_webpage import create_scrape_webpage_tool
+from .search_surfsense_docs import create_search_surfsense_docs_tool
 
 __all__ = [
     # Registry
@@ -43,6 +45,7 @@ __all__ = [
     "create_link_preview_tool",
     "create_scrape_webpage_tool",
     "create_search_knowledge_base_tool",
+    "create_search_surfsense_docs_tool",
     # Knowledge base utilities
     "format_documents_for_context",
     "get_all_tool_names",

surfsense_backend/app/agents/new_chat/tools/registry.py

@@ -48,6 +48,7 @@ from .knowledge_base import create_search_knowledge_base_tool
 from .link_preview import create_link_preview_tool
 from .podcast import create_generate_podcast_tool
 from .scrape_webpage import create_scrape_webpage_tool
+from .search_surfsense_docs import create_search_surfsense_docs_tool
 
 # =============================================================================
 # Tool Definition
@@ -126,6 +127,15 @@ BUILTIN_TOOLS: list[ToolDefinition] = [
         requires=[],  # firecrawl_api_key is optional
     ),
     # Note: write_todos is now provided by TodoListMiddleware from deepagents
+    # Surfsense documentation search tool
+    ToolDefinition(
+        name="search_surfsense_docs",
+        description="Search Surfsense documentation for help with using the application",
+        factory=lambda deps: create_search_surfsense_docs_tool(
+            db_session=deps["db_session"],
+        ),
+        requires=["db_session"],
+    ),
     # =========================================================================
     # ADD YOUR CUSTOM TOOLS BELOW
     # =========================================================================

surfsense_backend/app/agents/new_chat/tools/search_surfsense_docs.py

@@ -0,0 +1,163 @@
+"""
+Surfsense documentation search tool.
+
+This tool allows the agent to search the pre-indexed Surfsense documentation
+to help users with questions about how to use the application.
+
+The documentation is indexed at deployment time from MDX files and stored
+in dedicated tables (surfsense_docs_documents, surfsense_docs_chunks).
+"""
+
+import json
+
+from langchain_core.tools import tool
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.config import config
+from app.db import SurfsenseDocsChunk, SurfsenseDocsDocument
+
+
+def format_surfsense_docs_results(results: list[tuple]) -> str:
+    """
+    Format search results into XML structure for the LLM context.
+
+    Uses the same XML structure as format_documents_for_context from knowledge_base.py
+    but with 'doc-' prefix on chunk IDs. This allows:
+    - LLM to use consistent [citation:doc-XXX] format
+    - Frontend to detect 'doc-' prefix and route to surfsense docs endpoint
+
+    Args:
+        results: List of (chunk, document) tuples from the database query
+
+    Returns:
+        Formatted XML string with documentation content and citation-ready chunks
+    """
+    if not results:
+        return "No relevant Surfsense documentation found for your query."
+
+    # Group chunks by document
+    grouped: dict[int, dict] = {}
+    for chunk, doc in results:
+        if doc.id not in grouped:
+            grouped[doc.id] = {
+                "document_id": f"doc-{doc.id}",
+                "document_type": "SURFSENSE_DOCS",
+                "title": doc.title,
+                "url": doc.source,
+                "metadata": {"source": doc.source},
+                "chunks": [],
+            }
+        grouped[doc.id]["chunks"].append(
+            {
+                "chunk_id": f"doc-{chunk.id}",
+                "content": chunk.content,
+            }
+        )
+
+    # Render XML matching format_documents_for_context structure
+    parts: list[str] = []
+    for g in grouped.values():
+        metadata_json = json.dumps(g["metadata"], ensure_ascii=False)
+
+        parts.append("<document>")
+        parts.append("<document_metadata>")
+        parts.append(f"  <document_id>{g['document_id']}</document_id>")
+        parts.append(f"  <document_type>{g['document_type']}</document_type>")
+        parts.append(f"  <title><![CDATA[{g['title']}]]></title>")
+        parts.append(f"  <url><![CDATA[{g['url']}]]></url>")
+        parts.append(f"  <metadata_json><![CDATA[{metadata_json}]]></metadata_json>")
+        parts.append("</document_metadata>")
+        parts.append("")
+        parts.append("<document_content>")
+
+        for ch in g["chunks"]:
+            parts.append(
+                f"  <chunk id='{ch['chunk_id']}'><![CDATA[{ch['content']}]]></chunk>"
+            )
+
+        parts.append("</document_content>")
+        parts.append("</document>")
+        parts.append("")
+
+    return "\n".join(parts).strip()
+
+
+async def search_surfsense_docs_async(
+    query: str,
+    db_session: AsyncSession,
+    top_k: int = 10,
+) -> str:
+    """
+    Search Surfsense documentation using vector similarity.
+
+    Args:
+        query: The search query about Surfsense usage
+        db_session: Database session for executing queries
+        top_k: Number of results to return
+
+    Returns:
+        Formatted string with relevant documentation content
+    """
+    # Get embedding for the query
+    query_embedding = config.embedding_model_instance.embed(query)
+
+    # Vector similarity search on chunks, joining with documents
+    stmt = (
+        select(SurfsenseDocsChunk, SurfsenseDocsDocument)
+        .join(
+            SurfsenseDocsDocument,
+            SurfsenseDocsChunk.document_id == SurfsenseDocsDocument.id,
+        )
+        .order_by(SurfsenseDocsChunk.embedding.op("<=>")(query_embedding))
+        .limit(top_k)
+    )
+
+    result = await db_session.execute(stmt)
+    rows = result.all()
+
+    return format_surfsense_docs_results(rows)
+
+
+def create_search_surfsense_docs_tool(db_session: AsyncSession):
+    """
+    Factory function to create the search_surfsense_docs tool.
+
+    Args:
+        db_session: Database session for executing queries
+
+    Returns:
+        A configured tool function for searching Surfsense documentation
+    """
+
+    @tool
+    async def search_surfsense_docs(query: str, top_k: int = 10) -> str:
+        """
+        Search Surfsense documentation for help with using the application.
+
+        Use this tool when the user asks questions about:
+        - How to use Surfsense features
+        - Installation and setup instructions
+        - Configuration options and settings
+        - Troubleshooting common issues
+        - Available connectors and integrations
+        - Browser extension usage
+        - API documentation
+
+        This searches the official Surfsense documentation that was indexed
+        at deployment time. It does NOT search the user's personal knowledge base.
+
+        Args:
+            query: The search query about Surfsense usage or features
+            top_k: Number of documentation chunks to retrieve (default: 10)
+
+        Returns:
+            Relevant documentation content formatted with chunk IDs for citations
+        """
+        return await search_surfsense_docs_async(
+            query=query,
+            db_session=db_session,
+            top_k=top_k,
+        )
+
+    return search_surfsense_docs

surfsense_backend/app/app.py

@@ -13,6 +13,7 @@ from app.config import config
 from app.db import User, create_db_and_tables, get_async_session
 from app.routes import router as crud_router
 from app.schemas import UserCreate, UserRead, UserUpdate
+from app.tasks.surfsense_docs_indexer import seed_surfsense_docs
 from app.users import SECRET, auth_backend, current_active_user, fastapi_users
 
 
@@ -22,6 +23,8 @@ async def lifespan(app: FastAPI):
     await create_db_and_tables()
     # Setup LangGraph checkpointer tables for conversation persistence
     await setup_checkpointer_tables()
+    # Seed Surfsense documentation
+    await seed_surfsense_docs()
     yield
     # Cleanup: close checkpointer connection on shutdown
     await close_checkpointer()

surfsense_backend/app/db.py

@@ -428,6 +428,46 @@ class Chunk(BaseModel, TimestampMixin):
     document = relationship("Document", back_populates="chunks")
 
 
+class SurfsenseDocsDocument(BaseModel, TimestampMixin):
+    """
+    Surfsense documentation storage.
+    Indexed at migration time from MDX files.
+    """
+
+    __tablename__ = "surfsense_docs_documents"
+
+    source = Column(
+        String, nullable=False, unique=True, index=True
+    )  # File path: "connectors/slack.mdx"
+    title = Column(String, nullable=False)
+    content = Column(Text, nullable=False)
+    content_hash = Column(String, nullable=False, index=True)  # For detecting changes
+    embedding = Column(Vector(config.embedding_model_instance.dimension))
+    updated_at = Column(TIMESTAMP(timezone=True), nullable=True, index=True)
+
+    chunks = relationship(
+        "SurfsenseDocsChunk",
+        back_populates="document",
+        cascade="all, delete-orphan",
+    )
+
+
+class SurfsenseDocsChunk(BaseModel, TimestampMixin):
+    """Chunk storage for Surfsense documentation."""
+
+    __tablename__ = "surfsense_docs_chunks"
+
+    content = Column(Text, nullable=False)
+    embedding = Column(Vector(config.embedding_model_instance.dimension))
+
+    document_id = Column(
+        Integer,
+        ForeignKey("surfsense_docs_documents.id", ondelete="CASCADE"),
+        nullable=False,
+    )
+    document = relationship("SurfsenseDocsDocument", back_populates="chunks")
+
+
 class Podcast(BaseModel, TimestampMixin):
     """Podcast model for storing generated podcasts."""
 

surfsense_backend/app/routes/__init__.py

@@ -31,6 +31,7 @@ from .rbac_routes import router as rbac_router
 from .search_source_connectors_routes import router as search_source_connectors_router
 from .search_spaces_routes import router as search_spaces_router
 from .slack_add_connector_route import router as slack_add_connector_router
+from .surfsense_docs_routes import router as surfsense_docs_router
 from .teams_add_connector_route import router as teams_add_connector_router
 
 router = APIRouter()
@@ -59,3 +60,4 @@ router.include_router(clickup_add_connector_router)
 router.include_router(new_llm_config_router)  # LLM configs with prompt configuration
 router.include_router(logs_router)
 router.include_router(circleback_webhook_router)  # Circleback meeting webhooks
+router.include_router(surfsense_docs_router)  # Surfsense documentation for citations

surfsense_backend/app/routes/search_source_connectors_routes.py

@@ -624,10 +624,7 @@ async def index_connector_content(
             SearchSourceConnectorType.LUMA_CONNECTOR,
         ]:
             # Default to today if no end_date provided (users can manually select future dates)
-            if end_date is None:
-                indexing_to = today_str
-            else:
-                indexing_to = end_date
+            indexing_to = today_str if end_date is None else end_date
         else:
             # For non-calendar connectors, cap at today
             indexing_to = end_date if end_date else today_str

surfsense_backend/app/routes/surfsense_docs_routes.py

@@ -0,0 +1,89 @@
+"""
+Routes for Surfsense documentation.
+
+These endpoints support the citation system for Surfsense docs,
+allowing the frontend to fetch document details when a user clicks
+on a [citation:doc-XXX] link.
+"""
+
+from fastapi import APIRouter, Depends, HTTPException
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.orm import selectinload
+
+from app.db import (
+    SurfsenseDocsChunk,
+    SurfsenseDocsDocument,
+    User,
+    get_async_session,
+)
+from app.schemas.surfsense_docs import (
+    SurfsenseDocsChunkRead,
+    SurfsenseDocsDocumentWithChunksRead,
+)
+from app.users import current_active_user
+
+router = APIRouter()
+
+
+@router.get(
+    "/surfsense-docs/by-chunk/{chunk_id}",
+    response_model=SurfsenseDocsDocumentWithChunksRead,
+)
+async def get_surfsense_doc_by_chunk_id(
+    chunk_id: int,
+    session: AsyncSession = Depends(get_async_session),
+    user: User = Depends(current_active_user),
+):
+    """
+    Retrieves a Surfsense documentation document based on a chunk ID.
+
+    This endpoint is used by the frontend to resolve [citation:doc-XXX] links.
+    """
+    try:
+        # Get the chunk
+        chunk_result = await session.execute(
+            select(SurfsenseDocsChunk).filter(SurfsenseDocsChunk.id == chunk_id)
+        )
+        chunk = chunk_result.scalars().first()
+
+        if not chunk:
+            raise HTTPException(
+                status_code=404,
+                detail=f"Surfsense docs chunk with id {chunk_id} not found",
+            )
+
+        # Get the associated document with all its chunks
+        document_result = await session.execute(
+            select(SurfsenseDocsDocument)
+            .options(selectinload(SurfsenseDocsDocument.chunks))
+            .filter(SurfsenseDocsDocument.id == chunk.document_id)
+        )
+        document = document_result.scalars().first()
+
+        if not document:
+            raise HTTPException(
+                status_code=404,
+                detail="Surfsense docs document not found",
+            )
+
+        # Sort chunks by ID
+        sorted_chunks = sorted(document.chunks, key=lambda x: x.id)
+
+        return SurfsenseDocsDocumentWithChunksRead(
+            id=document.id,
+            title=document.title,
+            source=document.source,
+            content=document.content,
+            chunks=[
+                SurfsenseDocsChunkRead(id=c.id, content=c.content)
+                for c in sorted_chunks
+            ],
+        )
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(
+            status_code=500,
+            detail=f"Failed to retrieve Surfsense documentation: {e!s}",
+        ) from e

surfsense_backend/app/schemas/surfsense_docs.py

@@ -0,0 +1,26 @@
+"""
+Schemas for Surfsense documentation.
+"""
+
+from pydantic import BaseModel, ConfigDict
+
+
+class SurfsenseDocsChunkRead(BaseModel):
+    """Schema for a Surfsense docs chunk."""
+
+    id: int
+    content: str
+
+    model_config = ConfigDict(from_attributes=True)
+
+
+class SurfsenseDocsDocumentWithChunksRead(BaseModel):
+    """Schema for a Surfsense docs document with its chunks."""
+
+    id: int
+    title: str
+    source: str
+    content: str
+    chunks: list[SurfsenseDocsChunkRead]
+
+    model_config = ConfigDict(from_attributes=True)

surfsense_backend/app/tasks/surfsense_docs_indexer.py

@@ -0,0 +1,229 @@
+"""
+Surfsense documentation indexer.
+Indexes MDX documentation files at startup.
+"""
+
+import hashlib
+import logging
+import re
+from datetime import UTC, datetime
+from pathlib import Path
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.orm import selectinload
+
+from app.config import config
+from app.db import SurfsenseDocsChunk, SurfsenseDocsDocument, async_session_maker
+
+logger = logging.getLogger(__name__)
+
+# Path to docs relative to project root
+DOCS_DIR = (
+    Path(__file__).resolve().parent.parent.parent.parent
+    / "surfsense_web"
+    / "content"
+    / "docs"
+)
+
+
+def parse_mdx_frontmatter(content: str) -> tuple[str, str]:
+    """
+    Parse MDX file to extract frontmatter title and content.
+
+    Args:
+        content: Raw MDX file content
+
+    Returns:
+        Tuple of (title, content_without_frontmatter)
+    """
+    # Match frontmatter between --- markers
+    frontmatter_pattern = r"^---\s*\n(.*?)\n---\s*\n"
+    match = re.match(frontmatter_pattern, content, re.DOTALL)
+
+    if match:
+        frontmatter = match.group(1)
+        content_without_frontmatter = content[match.end() :]
+
+        # Extract title from frontmatter
+        title_match = re.search(r"^title:\s*(.+)$", frontmatter, re.MULTILINE)
+        title = title_match.group(1).strip() if title_match else "Untitled"
+
+        # Remove quotes if present
+        title = title.strip("\"'")
+
+        return title, content_without_frontmatter.strip()
+
+    return "Untitled", content.strip()
+
+
+def get_all_mdx_files() -> list[Path]:
+    """
+    Get all MDX files from the docs directory.
+
+    Returns:
+        List of Path objects for each MDX file
+    """
+    if not DOCS_DIR.exists():
+        logger.warning(f"Docs directory not found: {DOCS_DIR}")
+        return []
+
+    return list(DOCS_DIR.rglob("*.mdx"))
+
+
+def generate_surfsense_docs_content_hash(content: str) -> str:
+    """Generate SHA-256 hash for Surfsense docs content."""
+    return hashlib.sha256(content.encode("utf-8")).hexdigest()
+
+
+def create_surfsense_docs_chunks(content: str) -> list[SurfsenseDocsChunk]:
+    """
+    Create chunks from Surfsense documentation content.
+
+    Args:
+        content: Document content to chunk
+
+    Returns:
+        List of SurfsenseDocsChunk objects with embeddings
+    """
+    return [
+        SurfsenseDocsChunk(
+            content=chunk.text,
+            embedding=config.embedding_model_instance.embed(chunk.text),
+        )
+        for chunk in config.chunker_instance.chunk(content)
+    ]
+
+
+async def index_surfsense_docs(session: AsyncSession) -> tuple[int, int, int, int]:
+    """
+    Index all Surfsense documentation files.
+
+    Args:
+        session: SQLAlchemy async session
+
+    Returns:
+        Tuple of (created, updated, skipped, deleted) counts
+    """
+    created = 0
+    updated = 0
+    skipped = 0
+    deleted = 0
+
+    # Get all existing docs from database
+    existing_docs_result = await session.execute(
+        select(SurfsenseDocsDocument).options(
+            selectinload(SurfsenseDocsDocument.chunks)
+        )
+    )
+    existing_docs = {doc.source: doc for doc in existing_docs_result.scalars().all()}
+
+    # Track which sources we've processed
+    processed_sources = set()
+
+    # Get all MDX files
+    mdx_files = get_all_mdx_files()
+    logger.info(f"Found {len(mdx_files)} MDX files to index")
+
+    for mdx_file in mdx_files:
+        try:
+            source = str(mdx_file.relative_to(DOCS_DIR))
+            processed_sources.add(source)
+
+            # Read file content
+            raw_content = mdx_file.read_text(encoding="utf-8")
+            title, content = parse_mdx_frontmatter(raw_content)
+            content_hash = generate_surfsense_docs_content_hash(raw_content)
+
+            if source in existing_docs:
+                existing_doc = existing_docs[source]
+
+                # Check if content changed
+                if existing_doc.content_hash == content_hash:
+                    logger.debug(f"Skipping unchanged: {source}")
+                    skipped += 1
+                    continue
+
+                # Content changed - update document
+                logger.info(f"Updating changed document: {source}")
+
+                # Create new chunks
+                chunks = create_surfsense_docs_chunks(content)
+
+                # Update document fields
+                existing_doc.title = title
+                existing_doc.content = content
+                existing_doc.content_hash = content_hash
+                existing_doc.embedding = config.embedding_model_instance.embed(content)
+                existing_doc.chunks = chunks
+                existing_doc.updated_at = datetime.now(UTC)
+
+                updated += 1
+            else:
+                # New document - create it
+                logger.info(f"Creating new document: {source}")
+
+                chunks = create_surfsense_docs_chunks(content)
+
+                document = SurfsenseDocsDocument(
+                    source=source,
+                    title=title,
+                    content=content,
+                    content_hash=content_hash,
+                    embedding=config.embedding_model_instance.embed(content),
+                    chunks=chunks,
+                    updated_at=datetime.now(UTC),
+                )
+
+                session.add(document)
+                created += 1
+
+        except Exception as e:
+            logger.error(f"Error processing {mdx_file}: {e}", exc_info=True)
+            continue
+
+    # Delete documents for removed files
+    for source, doc in existing_docs.items():
+        if source not in processed_sources:
+            logger.info(f"Deleting removed document: {source}")
+            await session.delete(doc)
+            deleted += 1
+
+    # Commit all changes
+    await session.commit()
+
+    logger.info(
+        f"Indexing complete: {created} created, {updated} updated, "
+        f"{skipped} skipped, {deleted} deleted"
+    )
+
+    return created, updated, skipped, deleted
+
+
+async def seed_surfsense_docs() -> tuple[int, int, int, int]:
+    """
+    Seed Surfsense documentation into the database.
+
+    This function indexes all MDX files from the docs directory.
+    It handles creating, updating, and deleting docs based on content changes.
+
+    Returns:
+        Tuple of (created, updated, skipped, deleted) counts
+        Returns (0, 0, 0, 0) if an error occurs
+    """
+    logger.info("Starting Surfsense docs indexing...")
+
+    try:
+        async with async_session_maker() as session:
+            created, updated, skipped, deleted = await index_surfsense_docs(session)
+
+        logger.info(
+            f"Surfsense docs indexing complete: "
+            f"created={created}, updated={updated}, skipped={skipped}, deleted={deleted}"
+        )
+
+        return created, updated, skipped, deleted
+
+    except Exception as e:
+        logger.error(f"Failed to seed Surfsense docs: {e}", exc_info=True)
+        return 0, 0, 0, 0

surfsense_backend/scripts/seed_surfsense_docs.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+"""
+Seed Surfsense documentation into the database.
+
+CLI wrapper for the seed_surfsense_docs function.
+Can be run manually for debugging or re-indexing.
+
+Usage:
+    python scripts/seed_surfsense_docs.py
+"""
+
+import asyncio
+import sys
+from pathlib import Path
+
+# Add the parent directory to the path so we can import app modules
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from app.tasks.surfsense_docs_indexer import seed_surfsense_docs
+
+
+def main():
+    """CLI entry point for seeding Surfsense docs."""
+    print("=" * 50)
+    print("  Surfsense Documentation Seeding")
+    print("=" * 50)
+
+    created, updated, skipped, deleted = asyncio.run(seed_surfsense_docs())
+
+    print()
+    print("Results:")
+    print(f"  Created: {created}")
+    print(f"  Updated: {updated}")
+    print(f"  Skipped: {skipped}")
+    print(f"  Deleted: {deleted}")
+    print("=" * 50)
+
+
+if __name__ == "__main__":
+    main()