apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-06-22 21:28:12 +02:00
Code Issues Projects Releases Packages Wiki Activity

Merge remote-tracking branch 'upstream/dev' into feat/replace-logs

Browse source
This commit is contained in:
Anish Sarkar 2026-01-14 02:04:54 +05:30
commit 99bd2df463
59 changed files with 2788 additions and 1579 deletions
Download patch file Download diff file Expand all files Collapse all files

109
surfsense_backend/alembic/versions/61_add_chat_visibility_and_created_by.py Normal file
View file

@ -0,0 +1,109 @@
"""Add chat visibility and created_by_id columns to new_chat_threads
This migration adds:
- ChatVisibility enum (PRIVATE, SEARCH_SPACE)
- visibility column to new_chat_threads table (default: PRIVATE)
- created_by_id column to track who created the chat thread
Revision ID: 61
Revises: 60
"""
from collections.abc import Sequence
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "61"
down_revision: str | None = "60"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Add visibility and created_by_id columns to new_chat_threads."""
# Create the ChatVisibility enum type
op.execute(
"""
DO $$
BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'chatvisibility') THEN
CREATE TYPE chatvisibility AS ENUM ('PRIVATE', 'SEARCH_SPACE');
END IF;
END$$;
"""
)
# Add visibility column with default value PRIVATE
op.execute(
"""
DO $$
BEGIN
IF NOT EXISTS (
SELECT 1 FROM information_schema.columns
WHERE table_name = 'new_chat_threads' AND column_name = 'visibility'
) THEN
ALTER TABLE new_chat_threads
ADD COLUMN visibility chatvisibility NOT NULL DEFAULT 'PRIVATE';
END IF;
END$$;
"""
)
# Create index on visibility column for efficient filtering
op.execute(
"""
CREATE INDEX IF NOT EXISTS ix_new_chat_threads_visibility
ON new_chat_threads(visibility);
"""
)
# Add created_by_id column (nullable to handle existing records)
op.execute(
"""
DO $$
BEGIN
IF NOT EXISTS (
SELECT 1 FROM information_schema.columns
WHERE table_name = 'new_chat_threads' AND column_name = 'created_by_id'
) THEN
ALTER TABLE new_chat_threads
ADD COLUMN created_by_id UUID REFERENCES "user"(id) ON DELETE SET NULL;
END IF;
END$$;
"""
)
# Create index on created_by_id column for efficient filtering
op.execute(
"""
CREATE INDEX IF NOT EXISTS ix_new_chat_threads_created_by_id
ON new_chat_threads(created_by_id);
"""
)
def downgrade() -> None:
"""Remove visibility and created_by_id columns from new_chat_threads."""
# Drop indexes
op.execute("DROP INDEX IF EXISTS ix_new_chat_threads_created_by_id")
op.execute("DROP INDEX IF EXISTS ix_new_chat_threads_visibility")
# Drop columns
op.execute(
"""
ALTER TABLE new_chat_threads
DROP COLUMN IF EXISTS created_by_id;
"""
)
op.execute(
"""
ALTER TABLE new_chat_threads
DROP COLUMN IF EXISTS visibility;
"""
)
# Drop enum type (only if not used elsewhere)
op.execute("DROP TYPE IF EXISTS chatvisibility")

8
surfsense_backend/alembic/versions/61_add_notifications_table.py → surfsense_backend/alembic/versions/62_add_notifications_table.py
View file

@ -1,7 +1,7 @@
"""Add notifications table
Revision ID: 61
Revises: 60
Revision ID: 62
Revises: 61
Note: Electric SQL replication setup (REPLICA IDENTITY FULL and publication)
is handled in app/db.py setup_electric_replication() which runs on app startup.
@ -11,8 +11,8 @@ from collections.abc import Sequence
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "61"
down_revision: str | None = "60"
revision: str = "62"
down_revision: str | None = "61"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None

46
surfsense_backend/app/db.py
View file

@ -326,6 +326,20 @@ class NewChatMessageRole(str, Enum):
SYSTEM = "system"
class ChatVisibility(str, Enum):
"""
Visibility/sharing level for chat threads.
PRIVATE: Only the creator can see/access the chat (default)
SEARCH_SPACE: All members of the search space can see/access the chat
PUBLIC: (Future) Anyone with the link can access the chat
"""
PRIVATE = "PRIVATE"
SEARCH_SPACE = "SEARCH_SPACE"
# PUBLIC = "PUBLIC" # Reserved for future implementation
class NewChatThread(BaseModel, TimestampMixin):
"""
Thread model for the new chat feature using assistant-ui.
@ -345,13 +359,31 @@ class NewChatThread(BaseModel, TimestampMixin):
index=True,
)
# Visibility/sharing control
visibility = Column(
SQLAlchemyEnum(ChatVisibility),
nullable=False,
default=ChatVisibility.PRIVATE,
server_default="PRIVATE",
index=True,
)
# Foreign keys
search_space_id = Column(
Integer, ForeignKey("searchspaces.id", ondelete="CASCADE"), nullable=False
)
# Track who created this chat thread (for visibility filtering)
created_by_id = Column(
UUID(as_uuid=True),
ForeignKey("user.id", ondelete="SET NULL"),
nullable=True, # Nullable for existing records before migration
index=True,
)
# Relationships
search_space = relationship("SearchSpace", back_populates="new_chat_threads")
created_by = relationship("User", back_populates="new_chat_threads")
messages = relationship(
"NewChatMessage",
back_populates="thread",
@ -857,6 +889,13 @@ if config.AUTH_TYPE == "GOOGLE":
passive_deletes=True,
)
# Chat threads created by this user
new_chat_threads = relationship(
"NewChatThread",
back_populates="created_by",
passive_deletes=True,
)
# Page usage tracking for ETL services
pages_limit = Column(
Integer,
@ -889,6 +928,13 @@ else:
passive_deletes=True,
)
# Chat threads created by this user
new_chat_threads = relationship(
"NewChatThread",
back_populates="created_by",
passive_deletes=True,
)
# Page usage tracking for ETL services
pages_limit = Column(
Integer,

279
surfsense_backend/app/routes/new_chat_routes.py
View file

@ -19,12 +19,14 @@ from datetime import UTC, datetime
from fastapi import APIRouter, Depends, File, HTTPException, Request, UploadFile
from fastapi.responses import StreamingResponse
from sqlalchemy import or_
from sqlalchemy.exc import IntegrityError, OperationalError
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.future import select
from sqlalchemy.orm import selectinload
from app.db import (
ChatVisibility,
NewChatMessage,
NewChatMessageRole,
NewChatThread,
@ -40,6 +42,7 @@ from app.schemas.new_chat import (
NewChatThreadCreate,
NewChatThreadRead,
NewChatThreadUpdate,
NewChatThreadVisibilityUpdate,
NewChatThreadWithMessages,
ThreadHistoryLoadResponse,
ThreadListItem,
@ -52,6 +55,82 @@ from app.utils.rbac import check_permission
router = APIRouter()
async def check_thread_access(
session: AsyncSession,
thread: NewChatThread,
user: User,
require_ownership: bool = False,
) -> bool:
"""
Check if a user has access to a thread based on visibility rules.
Access is granted if:
- User is the creator of the thread
- Thread visibility is SEARCH_SPACE (any member can access)
- Thread is a legacy thread (created_by_id is NULL) - only if user is search space owner
Args:
session: Database session
thread: The thread to check access for
user: The user requesting access
require_ownership: If True, only the creator can access (for edit/delete operations)
For SEARCH_SPACE threads, any member with permission can access
Legacy threads (NULL creator) are accessible by search space owner
Returns:
True if access is granted
Raises:
HTTPException: If access is denied
"""
is_owner = thread.created_by_id == user.id
is_legacy = thread.created_by_id is None
# Shared threads (SEARCH_SPACE) are accessible by any member
# This check comes first so shared threads are always accessible
if thread.visibility == ChatVisibility.SEARCH_SPACE:
# For ownership-required operations on shared threads, any member can proceed
# (permission check is done at route level)
return True
# For legacy threads (created before visibility feature),
# only the search space owner can access
if is_legacy:
search_space_query = select(SearchSpace).filter(
SearchSpace.id == thread.search_space_id
)
search_space_result = await session.execute(search_space_query)
search_space = search_space_result.scalar_one_or_none()
is_search_space_owner = search_space and search_space.user_id == user.id
if is_search_space_owner:
return True
# Legacy threads are not accessible to non-owners
raise HTTPException(
status_code=403,
detail="You don't have access to this chat",
)
# If ownership is required, only the creator can access
if require_ownership:
if not is_owner:
raise HTTPException(
status_code=403,
detail="Only the creator of this chat can perform this action",
)
return True
# For read access: owner can access their own private threads
if is_owner:
return True
# Private thread and user is not the owner
raise HTTPException(
status_code=403,
detail="You don't have access to this private chat",
)
# =============================================================================
# Thread Endpoints
# =============================================================================
@ -65,9 +144,14 @@ async def list_threads(
user: User = Depends(current_active_user),
):
"""
List all threads for the current user in a search space.
List all accessible threads for the current user in a search space.
Returns threads and archived_threads for ThreadListPrimitive.
A user can see threads that are:
- Created by them (regardless of visibility)
- Shared with the search space (visibility = SEARCH_SPACE)
- Legacy threads with no creator (created_by_id is NULL) - only if user is search space owner
Args:
search_space_id: The search space to list threads for
limit: Optional limit on number of threads to return (applies to active threads only)
@ -83,10 +167,33 @@ async def list_threads(
"You don't have permission to read chats in this search space",
)
# Get all threads in this search space
# Check if user is the search space owner (for legacy thread visibility)
search_space_query = select(SearchSpace).filter(
SearchSpace.id == search_space_id
)
search_space_result = await session.execute(search_space_query)
search_space = search_space_result.scalar_one_or_none()
is_search_space_owner = search_space and search_space.user_id == user.id
# Build filter conditions:
# 1. Created by the current user (any visibility)
# 2. Shared with the search space (visibility = SEARCH_SPACE)
# 3. Legacy threads (created_by_id is NULL) - only visible to search space owner
filter_conditions = [
NewChatThread.created_by_id == user.id,
NewChatThread.visibility == ChatVisibility.SEARCH_SPACE,
]
# Only include legacy threads for the search space owner
if is_search_space_owner:
filter_conditions.append(NewChatThread.created_by_id.is_(None))
query = (
select(NewChatThread)
.filter(NewChatThread.search_space_id == search_space_id)
.filter(
NewChatThread.search_space_id == search_space_id,
or_(*filter_conditions),
)
.order_by(NewChatThread.updated_at.desc())
)
@ -98,10 +205,17 @@ async def list_threads(
archived_threads = []
for thread in all_threads:
# Legacy threads (no creator) are treated as own threads for owner
is_own_thread = thread.created_by_id == user.id or (
thread.created_by_id is None and is_search_space_owner
)
item = ThreadListItem(
id=thread.id,
title=thread.title,
archived=thread.archived,
visibility=thread.visibility,
created_by_id=thread.created_by_id,
is_own_thread=is_own_thread,
created_at=thread.created_at,
updated_at=thread.updated_at,
)
@ -137,7 +251,12 @@ async def search_threads(
user: User = Depends(current_active_user),
):
"""
Search threads by title in a search space.
Search accessible threads by title in a search space.
A user can search threads that are:
- Created by them (regardless of visibility)
- Shared with the search space (visibility = SEARCH_SPACE)
- Legacy threads with no creator (created_by_id is NULL) - only if user is search space owner
Args:
search_space_id: The search space to search in
@ -154,12 +273,31 @@ async def search_threads(
"You don't have permission to read chats in this search space",
)
# Search threads by title (case-insensitive)
# Check if user is the search space owner (for legacy thread visibility)
search_space_query = select(SearchSpace).filter(
SearchSpace.id == search_space_id
)
search_space_result = await session.execute(search_space_query)
search_space = search_space_result.scalar_one_or_none()
is_search_space_owner = search_space and search_space.user_id == user.id
# Build filter conditions
filter_conditions = [
NewChatThread.created_by_id == user.id,
NewChatThread.visibility == ChatVisibility.SEARCH_SPACE,
]
# Only include legacy threads for the search space owner
if is_search_space_owner:
filter_conditions.append(NewChatThread.created_by_id.is_(None))
# Search accessible threads by title (case-insensitive)
query = (
select(NewChatThread)
.filter(
NewChatThread.search_space_id == search_space_id,
NewChatThread.title.ilike(f"%{title}%"),
or_(*filter_conditions),
)
.order_by(NewChatThread.updated_at.desc())
)
@ -172,6 +310,13 @@ async def search_threads(
id=thread.id,
title=thread.title,
archived=thread.archived,
visibility=thread.visibility,
created_by_id=thread.created_by_id,
# Legacy threads (no creator) are treated as own threads for owner
is_own_thread=(
thread.created_by_id == user.id
or (thread.created_by_id is None and is_search_space_owner)
),
created_at=thread.created_at,
updated_at=thread.updated_at,
)
@ -200,6 +345,9 @@ async def create_thread(
"""
Create a new chat thread.
The thread is created with the specified visibility (defaults to PRIVATE).
The current user is recorded as the creator of the thread.
Requires CHATS_CREATE permission.
"""
try:
@ -215,7 +363,9 @@ async def create_thread(
db_thread = NewChatThread(
title=thread.title,
archived=thread.archived,
visibility=thread.visibility,
search_space_id=thread.search_space_id,
created_by_id=user.id,
updated_at=now,
)
session.add(db_thread)
@ -254,6 +404,10 @@ async def get_thread_messages(
Get a thread with all its messages.
This is used by ThreadHistoryAdapter.load() to restore conversation.
Access is granted if:
- User is the creator of the thread
- Thread visibility is SEARCH_SPACE
Requires CHATS_READ permission.
"""
try:
@ -268,7 +422,7 @@ async def get_thread_messages(
if not thread:
raise HTTPException(status_code=404, detail="Thread not found")
# Check permission and ownership
# Check permission to read chats in this search space
await check_permission(
session,
user,
@ -277,6 +431,9 @@ async def get_thread_messages(
"You don't have permission to read chats in this search space",
)
# Check thread-level access based on visibility
await check_thread_access(session, thread, user)
# Return messages in the format expected by assistant-ui
messages = [
NewChatMessageRead(
@ -313,6 +470,10 @@ async def get_thread_full(
"""
Get full thread details with all messages.
Access is granted if:
- User is the creator of the thread
- Thread visibility is SEARCH_SPACE
Requires CHATS_READ permission.
"""
try:
@ -334,6 +495,9 @@ async def get_thread_full(
"You don't have permission to read chats in this search space",
)
# Check thread-level access based on visibility
await check_thread_access(session, thread, user)
return thread
except HTTPException:
@ -360,6 +524,9 @@ async def update_thread(
Update a thread (title, archived status).
Used for renaming and archiving threads.
- PRIVATE threads: Only the creator can update
- SEARCH_SPACE threads: Any member with CHATS_UPDATE permission can update
Requires CHATS_UPDATE permission.
"""
try:
@ -379,6 +546,11 @@ async def update_thread(
"You don't have permission to update chats in this search space",
)
# For PRIVATE threads, only the creator can update
# For SEARCH_SPACE threads, any member with permission can update
if db_thread.visibility == ChatVisibility.PRIVATE:
await check_thread_access(session, db_thread, user, require_ownership=True)
# Update fields
update_data = thread_update.model_dump(exclude_unset=True)
for key, value in update_data.items():
@ -420,6 +592,9 @@ async def delete_thread(
"""
Delete a thread and all its messages.
- PRIVATE threads: Only the creator can delete
- SEARCH_SPACE threads: Any member with CHATS_DELETE permission can delete
Requires CHATS_DELETE permission.
"""
try:
@ -439,6 +614,11 @@ async def delete_thread(
"You don't have permission to delete chats in this search space",
)
# For PRIVATE threads, only the creator can delete
# For SEARCH_SPACE threads, any member with permission can delete
if db_thread.visibility == ChatVisibility.PRIVATE:
await check_thread_access(session, db_thread, user, require_ownership=True)
await session.delete(db_thread)
await session.commit()
return {"message": "Thread deleted successfully"}
@ -463,6 +643,71 @@ async def delete_thread(
) from None
@router.patch("/threads/{thread_id}/visibility", response_model=NewChatThreadRead)
async def update_thread_visibility(
thread_id: int,
visibility_update: NewChatThreadVisibilityUpdate,
session: AsyncSession = Depends(get_async_session),
user: User = Depends(current_active_user),
):
"""
Update the visibility/sharing settings of a thread.
Only the creator of the thread can change its visibility.
- PRIVATE: Only the creator can access the thread (default)
- SEARCH_SPACE: All members of the search space can access the thread
Requires CHATS_UPDATE permission.
"""
try:
result = await session.execute(
select(NewChatThread).filter(NewChatThread.id == thread_id)
)
db_thread = result.scalars().first()
if not db_thread:
raise HTTPException(status_code=404, detail="Thread not found")
await check_permission(
session,
user,
db_thread.search_space_id,
Permission.CHATS_UPDATE.value,
"You don't have permission to update chats in this search space",
)
# Only the creator can change visibility
await check_thread_access(session, db_thread, user, require_ownership=True)
# Update visibility
db_thread.visibility = visibility_update.visibility
db_thread.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(db_thread)
return db_thread
except HTTPException:
raise
except IntegrityError:
await session.rollback()
raise HTTPException(
status_code=400,
detail="Database constraint violation. Please check your input data.",
) from None
except OperationalError:
await session.rollback()
raise HTTPException(
status_code=503, detail="Database operation failed. Please try again later."
) from None
except Exception as e:
await session.rollback()
raise HTTPException(
status_code=500,
detail=f"An unexpected error occurred while updating thread visibility: {e!s}",
) from None
# =============================================================================
# Message Endpoints
# =============================================================================
@ -479,6 +724,10 @@ async def append_message(
Append a message to a thread.
This is used by ThreadHistoryAdapter.append() to persist messages.
Access is granted if:
- User is the creator of the thread
- Thread visibility is SEARCH_SPACE
Requires CHATS_UPDATE permission.
"""
try:
@ -513,6 +762,9 @@ async def append_message(
"You don't have permission to update chats in this search space",
)
# Check thread-level access based on visibility
await check_thread_access(session, thread, user)
# Convert string role to enum
role_str = (
message.role.lower() if isinstance(message.role, str) else message.role
@ -597,6 +849,10 @@ async def list_messages(
"""
List messages in a thread with pagination.
Access is granted if:
- User is the creator of the thread
- Thread visibility is SEARCH_SPACE
Requires CHATS_READ permission.
"""
try:
@ -617,6 +873,9 @@ async def list_messages(
"You don't have permission to read chats in this search space",
)
# Check thread-level access based on visibility
await check_thread_access(session, thread, user)
# Get messages
query = (
select(NewChatMessage)
@ -659,6 +918,10 @@ async def handle_new_chat(
This endpoint handles the new chat functionality with streaming responses
using Server-Sent Events (SSE) format compatible with Vercel AI SDK.
Access is granted if:
- User is the creator of the thread
- Thread visibility is SEARCH_SPACE
Requires CHATS_CREATE permission.
"""
try:
@ -679,6 +942,9 @@ async def handle_new_chat(
"You don't have permission to chat in this search space",
)
# Check thread-level access based on visibility
await check_thread_access(session, thread, user)
# Get search space to check LLM config preferences
search_space_result = await session.execute(
select(SearchSpace).filter(SearchSpace.id == request.search_space_id)
@ -706,6 +972,7 @@ async def handle_new_chat(
llm_config_id=llm_config_id,
attachments=request.attachments,
mentioned_document_ids=request.mentioned_document_ids,
mentioned_surfsense_doc_ids=request.mentioned_surfsense_doc_ids,
),
media_type="text/event-stream",
headers={

93
surfsense_backend/app/routes/rbac_routes.py
View file

@ -556,6 +556,54 @@ async def update_member_role(
) from e
# NOTE: /members/me must be defined BEFORE /members/{membership_id}
# because FastAPI matches routes in order, and "me" would otherwise
# be interpreted as a membership_id (causing a 422 validation error)
@router.delete("/searchspaces/{search_space_id}/members/me")
async def leave_search_space(
search_space_id: int,
session: AsyncSession = Depends(get_async_session),
user: User = Depends(current_active_user),
):
"""
Leave a search space (remove own membership).
Owners cannot leave their search space.
"""
try:
result = await session.execute(
select(SearchSpaceMembership).filter(
SearchSpaceMembership.user_id == user.id,
SearchSpaceMembership.search_space_id == search_space_id,
)
)
db_membership = result.scalars().first()
if not db_membership:
raise HTTPException(
status_code=404,
detail="You are not a member of this search space",
)
if db_membership.is_owner:
raise HTTPException(
status_code=400,
detail="Owners cannot leave their search space. Transfer ownership first or delete the search space.",
)
await session.delete(db_membership)
await session.commit()
return {"message": "Successfully left the search space"}
except HTTPException:
raise
except Exception as e:
await session.rollback()
logger.error(f"Failed to leave search space: {e!s}", exc_info=True)
raise HTTPException(
status_code=500, detail=f"Failed to leave search space: {e!s}"
) from e
@router.delete("/searchspaces/{search_space_id}/members/{membership_id}")
async def remove_member(
search_space_id: int,
@ -608,51 +656,6 @@ async def remove_member(
) from e
@router.delete("/searchspaces/{search_space_id}/members/me")
async def leave_search_space(
search_space_id: int,
session: AsyncSession = Depends(get_async_session),
user: User = Depends(current_active_user),
):
"""
Leave a search space (remove own membership).
Owners cannot leave their search space.
"""
try:
result = await session.execute(
select(SearchSpaceMembership).filter(
SearchSpaceMembership.user_id == user.id,
SearchSpaceMembership.search_space_id == search_space_id,
)
)
db_membership = result.scalars().first()
if not db_membership:
raise HTTPException(
status_code=404,
detail="You are not a member of this search space",
)
if db_membership.is_owner:
raise HTTPException(
status_code=400,
detail="Owners cannot leave their search space. Transfer ownership first or delete the search space.",
)
await session.delete(db_membership)
await session.commit()
return {"message": "Successfully left the search space"}
except HTTPException:
raise
except Exception as e:
await session.rollback()
logger.error(f"Failed to leave search space: {e!s}", exc_info=True)
raise HTTPException(
status_code=500, detail=f"Failed to leave search space: {e!s}"
) from e
# ============ Invite Endpoints ============

82
surfsense_backend/app/routes/surfsense_docs_routes.py
View file

@ -7,7 +7,7 @@ on a [citation:doc-XXX] link.
"""
from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy import select
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import selectinload
@ -17,8 +17,10 @@ from app.db import (
User,
get_async_session,
)
from app.schemas import PaginatedResponse
from app.schemas.surfsense_docs import (
SurfsenseDocsChunkRead,
SurfsenseDocsDocumentRead,
SurfsenseDocsDocumentWithChunksRead,
)
from app.users import current_active_user
@ -87,3 +89,81 @@ async def get_surfsense_doc_by_chunk_id(
status_code=500,
detail=f"Failed to retrieve Surfsense documentation: {e!s}",
) from e
@router.get(
"/surfsense-docs",
response_model=PaginatedResponse[SurfsenseDocsDocumentRead],
)
async def list_surfsense_docs(
page: int = 0,
page_size: int = 50,
title: str | None = None,
session: AsyncSession = Depends(get_async_session),
user: User = Depends(current_active_user),
):
"""
List all Surfsense documentation documents.
Args:
page: Zero-based page index.
page_size: Number of items per page (default: 50).
title: Optional title filter (case-insensitive substring match).
session: Database session (injected).
user: Current authenticated user (injected).
Returns:
PaginatedResponse[SurfsenseDocsDocumentRead]: Paginated list of Surfsense docs.
"""
try:
# Base query
query = select(SurfsenseDocsDocument)
count_query = select(func.count()).select_from(SurfsenseDocsDocument)
# Filter by title if provided
if title and title.strip():
query = query.filter(SurfsenseDocsDocument.title.ilike(f"%{title}%"))
count_query = count_query.filter(
SurfsenseDocsDocument.title.ilike(f"%{title}%")
)
# Get total count
total_result = await session.execute(count_query)
total = total_result.scalar() or 0
# Calculate offset
offset = page * page_size
# Get paginated results
result = await session.execute(
query.order_by(SurfsenseDocsDocument.title).offset(offset).limit(page_size)
)
docs = result.scalars().all()
# Convert to response format
items = [
SurfsenseDocsDocumentRead(
id=doc.id,
title=doc.title,
source=doc.source,
content=doc.content,
created_at=doc.created_at,
updated_at=doc.updated_at,
)
for doc in docs
]
has_more = (offset + len(items)) < total
return PaginatedResponse(
items=items,
total=total,
page=page,
page_size=page_size,
has_more=has_more,
)
except Exception as e:
raise HTTPException(
status_code=500,
detail=f"Failed to list Surfsense documentation: {e!s}",
) from e

19
surfsense_backend/app/schemas/new_chat.py
View file

@ -8,10 +8,11 @@ These schemas follow the assistant-ui ThreadHistoryAdapter pattern:
from datetime import datetime
from typing import Any
from uuid import UUID
from pydantic import BaseModel, ConfigDict, Field
from app.db import NewChatMessageRole
from app.db import ChatVisibility, NewChatMessageRole
from .base import IDModel, TimestampModel
@ -66,6 +67,8 @@ class NewChatThreadCreate(NewChatThreadBase):
"""Schema for creating a new thread."""
search_space_id: int
# Visibility defaults to PRIVATE, but can be set on creation
visibility: ChatVisibility = ChatVisibility.PRIVATE
class NewChatThreadUpdate(BaseModel):
@ -75,12 +78,20 @@ class NewChatThreadUpdate(BaseModel):
archived: bool | None = None
class NewChatThreadVisibilityUpdate(BaseModel):
"""Schema for updating thread visibility/sharing settings."""
visibility: ChatVisibility
class NewChatThreadRead(NewChatThreadBase, IDModel):
"""
Schema for reading a thread (matches assistant-ui ThreadRecord).
"""
search_space_id: int
visibility: ChatVisibility
created_by_id: UUID | None = None
created_at: datetime
updated_at: datetime
@ -116,6 +127,9 @@ class ThreadListItem(BaseModel):
id: int
title: str
archived: bool
visibility: ChatVisibility
created_by_id: UUID | None = None
is_own_thread: bool = False # True if the current user created this thread
created_at: datetime = Field(alias="createdAt")
updated_at: datetime = Field(alias="updatedAt")
@ -163,3 +177,6 @@ class NewChatRequest(BaseModel):
mentioned_document_ids: list[int] | None = (
None # Optional document IDs mentioned with @ in the chat
)
mentioned_surfsense_doc_ids: list[int] | None = (
None # Optional SurfSense documentation IDs mentioned with @ in the chat
)

15
surfsense_backend/app/schemas/surfsense_docs.py
View file

@ -2,6 +2,8 @@
Schemas for Surfsense documentation.
"""
from datetime import datetime
from pydantic import BaseModel, ConfigDict
@ -14,6 +16,19 @@ class SurfsenseDocsChunkRead(BaseModel):
model_config = ConfigDict(from_attributes=True)
class SurfsenseDocsDocumentRead(BaseModel):
"""Schema for a Surfsense docs document (without chunks)."""
id: int
title: str
source: str
content: str
created_at: datetime | None = None
updated_at: datetime | None = None
model_config = ConfigDict(from_attributes=True)
class SurfsenseDocsDocumentWithChunksRead(BaseModel):
"""Schema for a Surfsense docs document with its chunks."""

93
surfsense_backend/app/tasks/chat/stream_new_chat.py
View file

@ -25,7 +25,7 @@ from app.agents.new_chat.llm_config import (
load_agent_config,
load_llm_config_from_yaml,
)
from app.db import Document
from app.db import Document, SurfsenseDocsDocument
from app.schemas.new_chat import ChatAttachment
from app.services.connector_service import ConnectorService
from app.services.new_streaming_service import VercelStreamingService
@ -69,6 +69,57 @@ def format_mentioned_documents_as_context(documents: list[Document]) -> str:
return "\n".join(context_parts)
def format_mentioned_surfsense_docs_as_context(
documents: list[SurfsenseDocsDocument],
) -> str:
"""Format mentioned SurfSense documentation as context for the agent."""
if not documents:
return ""
import json
context_parts = ["<mentioned_surfsense_docs>"]
context_parts.append(
"The user has explicitly mentioned the following SurfSense documentation pages. "
"These are official documentation about how to use SurfSense and should be used to answer questions about the application. "
"Use [citation:CHUNK_ID] format for citations (e.g., [citation:doc-123])."
)
for doc in documents:
metadata_json = json.dumps({"source": doc.source}, ensure_ascii=False)
context_parts.append("<document>")
context_parts.append("<document_metadata>")
context_parts.append(f" <document_id>doc-{doc.id}</document_id>")
context_parts.append(" <document_type>SURFSENSE_DOCS</document_type>")
context_parts.append(f" <title><![CDATA[{doc.title}]]></title>")
context_parts.append(f" <url><![CDATA[{doc.source}]]></url>")
context_parts.append(
f" <metadata_json><![CDATA[{metadata_json}]]></metadata_json>"
)
context_parts.append("</document_metadata>")
context_parts.append("")
context_parts.append("<document_content>")
if hasattr(doc, "chunks") and doc.chunks:
for chunk in doc.chunks:
context_parts.append(
f" <chunk id='doc-{chunk.id}'><![CDATA[{chunk.content}]]></chunk>"
)
else:
context_parts.append(
f" <chunk id='doc-0'><![CDATA[{doc.content}]]></chunk>"
)
context_parts.append("</document_content>")
context_parts.append("</document>")
context_parts.append("")
context_parts.append("</mentioned_surfsense_docs>")
return "\n".join(context_parts)
def extract_todos_from_deepagents(command_output) -> dict:
"""
Extract todos from deepagents' TodoListMiddleware Command output.
@ -101,6 +152,7 @@ async def stream_new_chat(
llm_config_id: int = -1,
attachments: list[ChatAttachment] | None = None,
mentioned_document_ids: list[int] | None = None,
mentioned_surfsense_doc_ids: list[int] | None = None,
) -> AsyncGenerator[str, None]:
"""
Stream chat responses from the new SurfSense deep agent.
@ -118,6 +170,7 @@ async def stream_new_chat(
messages: Optional chat history from frontend (list of ChatMessage)
attachments: Optional attachments with extracted content
mentioned_document_ids: Optional list of document IDs mentioned with @ in the chat
mentioned_surfsense_doc_ids: Optional list of SurfSense doc IDs mentioned with @ in the chat
Yields:
str: SSE formatted response strings
@ -208,7 +261,21 @@ async def stream_new_chat(
)
mentioned_documents = list(result.scalars().all())
# Format the user query with context (attachments + mentioned documents)
# Fetch mentioned SurfSense docs if any
mentioned_surfsense_docs: list[SurfsenseDocsDocument] = []
if mentioned_surfsense_doc_ids:
from sqlalchemy.orm import selectinload
result = await session.execute(
select(SurfsenseDocsDocument)
.options(selectinload(SurfsenseDocsDocument.chunks))
.filter(
SurfsenseDocsDocument.id.in_(mentioned_surfsense_doc_ids),
)
)
mentioned_surfsense_docs = list(result.scalars().all())
# Format the user query with context (attachments + mentioned documents + surfsense docs)
final_query = user_query
context_parts = []
@ -220,6 +287,11 @@ async def stream_new_chat(
format_mentioned_documents_as_context(mentioned_documents)
)
if mentioned_surfsense_docs:
context_parts.append(
format_mentioned_surfsense_docs_as_context(mentioned_surfsense_docs)
)
if context_parts:
context = "\n\n".join(context_parts)
final_query = f"{context}\n\n<user_query>{user_query}</user_query>"
@ -296,13 +368,13 @@ async def stream_new_chat(
last_active_step_id = analyze_step_id
# Determine step title and action verb based on context
if attachments and mentioned_documents:
if attachments and (mentioned_documents or mentioned_surfsense_docs):
last_active_step_title = "Analyzing your content"
action_verb = "Reading"
elif attachments:
last_active_step_title = "Reading your content"
action_verb = "Reading"
elif mentioned_documents:
elif mentioned_documents or mentioned_surfsense_docs:
last_active_step_title = "Analyzing referenced content"
action_verb = "Analyzing"
else:
@ -342,6 +414,19 @@ async def stream_new_chat(
else:
processing_parts.append(f"[{len(doc_names)} documents]")
# Add mentioned SurfSense docs inline
if mentioned_surfsense_docs:
doc_names = []
for doc in mentioned_surfsense_docs:
title = doc.title
if len(title) > 30:
title = title[:27] + "..."
doc_names.append(title)
if len(doc_names) == 1:
processing_parts.append(f"[📖 {doc_names[0]}]")
else:
processing_parts.append(f"[📖 {len(doc_names)} docs]")
last_active_step_items = [f"{action_verb}: {' '.join(processing_parts)}"]
yield streaming_service.format_thinking_step(