diff --git a/docs/chinese-llm-setup.md b/docs/chinese-llm-setup.md index 2a184608f..37042aa2f 100644 --- a/docs/chinese-llm-setup.md +++ b/docs/chinese-llm-setup.md @@ -14,6 +14,7 @@ SurfSense 现已支持以下国产 LLM： - ✅ **阿里通义千问 (Alibaba Qwen)** - 阿里云通义千问大模型 - ✅ **月之暗面 Kimi (Moonshot)** - 月之暗面 Kimi 大模型 - ✅ **智谱 AI GLM (Zhipu)** - 智谱 AI GLM 系列模型 +- ✅ **MiniMax** - MiniMax 大模型 (M2.5 系列，204K 上下文) --- @@ -197,6 +198,52 @@ API Base URL: https://open.bigmodel.cn/api/paas/v4 --- +## 5️⃣ MiniMax 配置 | MiniMax Configuration + +### 获取 API Key + +1. 访问 [MiniMax 开放平台](https://platform.minimaxi.com/) +2. 注册并登录账号 +3. 进入 **API Keys** 页面 +4. 创建新的 API Key +5. 复制 API Key + +### 在 SurfSense 中配置 + +| 字段 | 值 | 说明 | +|------|-----|------| +| **Configuration Name** | `MiniMax M2.5` | 配置名称（自定义） | +| **Provider** | `MINIMAX` | 选择 MiniMax | +| **Model Name** | `MiniMax-M2.5` | 推荐模型
其他选项: `MiniMax-M2.5-highspeed` | +| **API Key** | `eyJ...` | 你的 MiniMax API Key | +| **API Base URL** | `https://api.minimax.io/v1` | MiniMax API 地址 | +| **Parameters** | `{"temperature": 1.0}` | 注意：temperature 必须在 (0.0, 1.0] 范围内，不能为 0 | + +### 示例配置 + +``` +Configuration Name: MiniMax M2.5 +Provider: MINIMAX +Model Name: MiniMax-M2.5 +API Key: eyJxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +API Base URL: https://api.minimax.io/v1 +``` + +### 可用模型 + +- **MiniMax-M2.5**: 高性能通用模型，204K 上下文窗口（推荐） +- **MiniMax-M2.5-highspeed**: 高速推理版本，204K 上下文窗口 + +### 注意事项 + +- **temperature 参数**: MiniMax 要求 temperature 必须在 (0.0, 1.0] 范围内，不能设置为 0。建议使用 1.0。 +- 两个模型都支持 204K 超长上下文窗口，适合处理长文本任务。 + +### 定价 +- 请访问 [MiniMax 定价页面](https://platform.minimaxi.com/document/Price) 查看最新价格 + +--- + ## ⚙️ 高级配置 | Advanced Configuration ### 自定义参数 | Custom Parameters @@ -268,8 +315,8 @@ docker compose logs backend | grep -i "error" |---------|---------|------| | **文档摘要** | Qwen-Plus, GLM-4 | 平衡性能和成本 | | **代码分析** | DeepSeek-Coder | 代码专用 | -| **长文本处理** | Kimi 128K | 超长上下文 | -| **快速响应** | Qwen-Turbo, GLM-4-Flash | 速度优先 | +| **长文本处理** | Kimi 128K, MiniMax-M2.5 (204K) | 超长上下文 | +| **快速响应** | Qwen-Turbo, GLM-4-Flash, MiniMax-M2.5-highspeed | 速度优先 | ### 2. 成本优化 @@ -294,6 +341,7 @@ docker compose logs backend | grep -i "error" - [阿里云百炼文档](https://help.aliyun.com/zh/model-studio/) - [Moonshot AI 文档](https://platform.moonshot.cn/docs) - [智谱 AI 文档](https://open.bigmodel.cn/dev/api) +- [MiniMax 文档](https://platform.minimaxi.com/document/Guides) ### SurfSense 文档 diff --git a/surfsense_backend/alembic/versions/106_add_minimax_to_litellmprovider_enum.py b/surfsense_backend/alembic/versions/106_add_minimax_to_litellmprovider_enum.py new file mode 100644 index 000000000..fed3bc7c3 --- /dev/null +++ b/surfsense_backend/alembic/versions/106_add_minimax_to_litellmprovider_enum.py @@ -0,0 +1,23 @@ +"""Add MINIMAX to LiteLLMProvider enum + +Revision ID: 106 +Revises: 105 +""" + +from collections.abc import Sequence + +from alembic import op + +revision: str = "106" +down_revision: str | None = "105" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.execute("COMMIT") + op.execute("ALTER TYPE litellmprovider ADD VALUE IF NOT EXISTS 'MINIMAX'") + + +def downgrade() -> None: + pass diff --git a/surfsense_backend/app/agents/new_chat/llm_config.py b/surfsense_backend/app/agents/new_chat/llm_config.py index 4ddb47330..60cd2a452 100644 --- a/surfsense_backend/app/agents/new_chat/llm_config.py +++ b/surfsense_backend/app/agents/new_chat/llm_config.py @@ -59,6 +59,7 @@ PROVIDER_MAP = { "DATABRICKS": "databricks", "COMETAPI": "cometapi", "HUGGINGFACE": "huggingface", + "MINIMAX": "openai", "CUSTOM": "custom", } diff --git a/surfsense_backend/app/config/global_llm_config.example.yaml b/surfsense_backend/app/config/global_llm_config.example.yaml index 0bb00c398..6ca3e95e3 100644 --- a/surfsense_backend/app/config/global_llm_config.example.yaml +++ b/surfsense_backend/app/config/global_llm_config.example.yaml @@ -183,6 +183,23 @@ global_llm_configs: use_default_system_instructions: true citations_enabled: true + # Example: MiniMax M2.5 - High-performance with 204K context window + - id: -8 + name: "Global MiniMax M2.5" + description: "MiniMax M2.5 with 204K context window and competitive pricing" + provider: "MINIMAX" + model_name: "MiniMax-M2.5" + api_key: "your-minimax-api-key-here" + api_base: "https://api.minimax.io/v1" + rpm: 60 + tpm: 100000 + litellm_params: + temperature: 1.0 # MiniMax requires temperature in (0.0, 1.0], cannot be 0 + max_tokens: 4000 + system_instructions: "" + use_default_system_instructions: true + citations_enabled: true + # ============================================================================= # Image Generation Configuration # ============================================================================= diff --git a/surfsense_backend/app/connectors/composio_gmail_connector.py b/surfsense_backend/app/connectors/composio_gmail_connector.py index 2a382f3b8..e675085db 100644 --- a/surfsense_backend/app/connectors/composio_gmail_connector.py +++ b/surfsense_backend/app/connectors/composio_gmail_connector.py @@ -463,7 +463,7 @@ async def _process_gmail_messages_phase2( "connector_id": connector_id, "source": "composio", } - safe_set_chunks(document, chunks) + await safe_set_chunks(session, document, chunks) document.updated_at = get_current_timestamp() document.status = DocumentStatus.ready() diff --git a/surfsense_backend/app/connectors/composio_google_calendar_connector.py b/surfsense_backend/app/connectors/composio_google_calendar_connector.py index 63bade873..6344f9f38 100644 --- a/surfsense_backend/app/connectors/composio_google_calendar_connector.py +++ b/surfsense_backend/app/connectors/composio_google_calendar_connector.py @@ -477,7 +477,7 @@ async def index_composio_google_calendar( "connector_id": connector_id, "source": "composio", } - safe_set_chunks(document, chunks) + await safe_set_chunks(session, document, chunks) document.updated_at = get_current_timestamp() document.status = DocumentStatus.ready() diff --git a/surfsense_backend/app/connectors/composio_google_drive_connector.py b/surfsense_backend/app/connectors/composio_google_drive_connector.py index c10edb7e9..30ce4a77b 100644 --- a/surfsense_backend/app/connectors/composio_google_drive_connector.py +++ b/surfsense_backend/app/connectors/composio_google_drive_connector.py @@ -1112,7 +1112,7 @@ async def _index_composio_drive_delta_sync( "connector_id": connector_id, "source": "composio", } - safe_set_chunks(document, chunks) + await safe_set_chunks(session, document, chunks) document.updated_at = get_current_timestamp() document.status = DocumentStatus.ready() @@ -1520,7 +1520,7 @@ async def _index_composio_drive_full_scan( "connector_id": connector_id, "source": "composio", } - safe_set_chunks(document, chunks) + await safe_set_chunks(session, document, chunks) document.updated_at = get_current_timestamp() document.status = DocumentStatus.ready() diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py index 062b11b3a..95ae8e728 100644 --- a/surfsense_backend/app/db.py +++ b/surfsense_backend/app/db.py @@ -215,6 +215,7 @@ class LiteLLMProvider(StrEnum): COMETAPI = "COMETAPI" HUGGINGFACE = "HUGGINGFACE" GITHUB_MODELS = "GITHUB_MODELS" + MINIMAX = "MINIMAX" CUSTOM = "CUSTOM" diff --git a/surfsense_backend/app/retriever/chunks_hybrid_search.py b/surfsense_backend/app/retriever/chunks_hybrid_search.py index 5ab2964ca..2a0bc1a4a 100644 --- a/surfsense_backend/app/retriever/chunks_hybrid_search.py +++ b/surfsense_backend/app/retriever/chunks_hybrid_search.py @@ -1,3 +1,4 @@ +import asyncio import time from datetime import datetime @@ -49,7 +50,7 @@ class ChucksHybridSearchRetriever: # Get embedding for the query embedding_model = config.embedding_model_instance t_embed = time.perf_counter() - query_embedding = embedding_model.embed(query_text) + query_embedding = await asyncio.to_thread(embedding_model.embed, query_text) perf.debug( "[chunk_search] vector_search embedding in %.3fs", time.perf_counter() - t_embed, @@ -195,7 +196,7 @@ class ChucksHybridSearchRetriever: if query_embedding is None: embedding_model = config.embedding_model_instance t_embed = time.perf_counter() - query_embedding = embedding_model.embed(query_text) + query_embedding = await asyncio.to_thread(embedding_model.embed, query_text) perf.debug( "[chunk_search] hybrid_search embedding in %.3fs", time.perf_counter() - t_embed, @@ -427,4 +428,4 @@ class ChucksHybridSearchRetriever: search_space_id, document_type, ) - return final_docs + return final_docs \ No newline at end of file diff --git a/surfsense_backend/app/services/linear/kb_sync_service.py b/surfsense_backend/app/services/linear/kb_sync_service.py index 8d1bc47c7..4f97a3dd0 100644 --- a/surfsense_backend/app/services/linear/kb_sync_service.py +++ b/surfsense_backend/app/services/linear/kb_sync_service.py @@ -1,11 +1,10 @@ import logging from datetime import datetime -from sqlalchemy import delete from sqlalchemy.ext.asyncio import AsyncSession from app.connectors.linear_connector import LinearConnector -from app.db import Chunk, Document +from app.db import Document from app.services.llm_service import get_user_long_context_llm from app.utils.document_converters import ( create_document_chunks, @@ -105,10 +104,6 @@ class LinearKBSyncService: ) summary_embedding = embed_text(summary_content) - await self.db_session.execute( - delete(Chunk).where(Chunk.document_id == document.id) - ) - chunks = await create_document_chunks(issue_content) document.title = f"{issue_identifier}: {issue_title}" @@ -131,7 +126,7 @@ class LinearKBSyncService: "connector_id": connector_id, } flag_modified(document, "document_metadata") - safe_set_chunks(document, chunks) + await safe_set_chunks(self.db_session, document, chunks) document.updated_at = get_current_timestamp() await self.db_session.commit() diff --git a/surfsense_backend/app/services/llm_router_service.py b/surfsense_backend/app/services/llm_router_service.py index 7a0b6e55b..63d8d10b9 100644 --- a/surfsense_backend/app/services/llm_router_service.py +++ b/surfsense_backend/app/services/llm_router_service.py @@ -85,6 +85,7 @@ PROVIDER_MAP = { "ZHIPU": "openai", "GITHUB_MODELS": "github", "HUGGINGFACE": "huggingface", + "MINIMAX": "openai", "CUSTOM": "custom", } diff --git a/surfsense_backend/app/services/llm_service.py b/surfsense_backend/app/services/llm_service.py index fc28f477f..e11abd886 100644 --- a/surfsense_backend/app/services/llm_service.py +++ b/surfsense_backend/app/services/llm_service.py @@ -127,6 +127,7 @@ async def validate_llm_config( "ALIBABA_QWEN": "openai", "MOONSHOT": "openai", "ZHIPU": "openai", # GLM needs special handling + "MINIMAX": "openai", "GITHUB_MODELS": "github", } provider_prefix = provider_map.get(provider, provider.lower()) @@ -277,6 +278,7 @@ async def get_search_space_llm_instance( "ALIBABA_QWEN": "openai", "MOONSHOT": "openai", "ZHIPU": "openai", + "MINIMAX": "openai", } provider_prefix = provider_map.get( global_config["provider"], global_config["provider"].lower() @@ -350,6 +352,7 @@ async def get_search_space_llm_instance( "ALIBABA_QWEN": "openai", "MOONSHOT": "openai", "ZHIPU": "openai", + "MINIMAX": "openai", "GITHUB_MODELS": "github", } provider_prefix = provider_map.get( diff --git a/surfsense_backend/app/services/notion/kb_sync_service.py b/surfsense_backend/app/services/notion/kb_sync_service.py index ce31e0d35..d6c64897f 100644 --- a/surfsense_backend/app/services/notion/kb_sync_service.py +++ b/surfsense_backend/app/services/notion/kb_sync_service.py @@ -1,10 +1,9 @@ import logging from datetime import datetime -from sqlalchemy import delete from sqlalchemy.ext.asyncio import AsyncSession -from app.db import Chunk, Document +from app.db import Document from app.services.llm_service import get_user_long_context_llm from app.utils.document_converters import ( create_document_chunks, @@ -130,11 +129,6 @@ class NotionKBSyncService: summary_content = f"Notion Page: {document.document_metadata.get('page_title')}\n\n{full_content}" summary_embedding = embed_text(summary_content) - logger.debug(f"Deleting old chunks for document {document_id}") - await self.db_session.execute( - delete(Chunk).where(Chunk.document_id == document.id) - ) - logger.debug("Creating new chunks") chunks = await create_document_chunks(full_content) logger.debug(f"Created {len(chunks)} chunks") @@ -147,7 +141,7 @@ class NotionKBSyncService: **document.document_metadata, "indexed_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), } - safe_set_chunks(document, chunks) + await safe_set_chunks(self.db_session, document, chunks) document.updated_at = get_current_timestamp() logger.debug("Committing changes to database") diff --git a/surfsense_backend/app/tasks/connector_indexers/airtable_indexer.py b/surfsense_backend/app/tasks/connector_indexers/airtable_indexer.py index 438a93815..6f020685a 100644 --- a/surfsense_backend/app/tasks/connector_indexers/airtable_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/airtable_indexer.py @@ -432,7 +432,7 @@ async def index_airtable_records( "table_name": item["table_name"], "connector_id": connector_id, } - safe_set_chunks(document, chunks) + await safe_set_chunks(session, document, chunks) document.updated_at = get_current_timestamp() document.status = DocumentStatus.ready() diff --git a/surfsense_backend/app/tasks/connector_indexers/base.py b/surfsense_backend/app/tasks/connector_indexers/base.py index 139aed1d3..b6ce2f2f9 100644 --- a/surfsense_backend/app/tasks/connector_indexers/base.py +++ b/surfsense_backend/app/tasks/connector_indexers/base.py @@ -28,45 +28,37 @@ def get_current_timestamp() -> datetime: return datetime.now(UTC) -def safe_set_chunks(document: Document, chunks: list) -> None: +async def safe_set_chunks( + session: "AsyncSession", document: Document, chunks: list +) -> None: """ - Safely assign chunks to a document without triggering lazy loading. + Delete old chunks and assign new ones to a document. - ALWAYS use this instead of `document.chunks = chunks` to avoid - SQLAlchemy async errors (MissingGreenlet / greenlet_spawn). - - Why this is needed: - - Direct assignment `document.chunks = chunks` triggers SQLAlchemy to - load the OLD chunks first (for comparison/orphan detection) - - This lazy loading fails in async context with asyncpg driver - - set_committed_value bypasses this by setting the value directly - - This function is safe regardless of how the document was loaded - (with or without selectinload). + This replaces direct ``document.chunks = chunks`` which triggers lazy + loading (and MissingGreenlet errors in async contexts). It also + explicitly deletes pre-existing chunks so they don't accumulate across + repeated re-indexes — ``set_committed_value`` bypasses SQLAlchemy's + delete-orphan cascade. Args: - document: The Document object to update - chunks: List of Chunk objects to assign - - Example: - # Instead of: document.chunks = chunks (DANGEROUS!) - safe_set_chunks(document, chunks) # Always safe + session: The current async database session. + document: The Document object to update. + chunks: List of Chunk objects to assign. """ - from sqlalchemy.orm import object_session + from sqlalchemy import delete from sqlalchemy.orm.attributes import set_committed_value - # Keep relationship assignment lazy-load-safe. - set_committed_value(document, "chunks", chunks) + from app.db import Chunk - # Ensure chunk rows are actually persisted. - # set_committed_value bypasses normal unit-of-work tracking, so we need to - # explicitly attach chunk objects to the current session. - session = object_session(document) - if session is not None: - if document.id is not None: - for chunk in chunks: - chunk.document_id = document.id - session.add_all(chunks) + if document.id is not None: + await session.execute( + delete(Chunk).where(Chunk.document_id == document.id) + ) + for chunk in chunks: + chunk.document_id = document.id + + set_committed_value(document, "chunks", chunks) + session.add_all(chunks) def parse_date_flexible(date_str: str) -> datetime: diff --git a/surfsense_backend/app/tasks/connector_indexers/bookstack_indexer.py b/surfsense_backend/app/tasks/connector_indexers/bookstack_indexer.py index bf3aaa35f..0660531b2 100644 --- a/surfsense_backend/app/tasks/connector_indexers/bookstack_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/bookstack_indexer.py @@ -430,7 +430,7 @@ async def index_bookstack_pages( document.content_hash = item["content_hash"] document.embedding = summary_embedding document.document_metadata = doc_metadata - safe_set_chunks(document, chunks) + await safe_set_chunks(session, document, chunks) document.updated_at = get_current_timestamp() document.status = DocumentStatus.ready() diff --git a/surfsense_backend/app/tasks/connector_indexers/clickup_indexer.py b/surfsense_backend/app/tasks/connector_indexers/clickup_indexer.py index fd0233e87..af796ba3c 100644 --- a/surfsense_backend/app/tasks/connector_indexers/clickup_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/clickup_indexer.py @@ -439,7 +439,7 @@ async def index_clickup_tasks( "connector_id": connector_id, "indexed_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), } - safe_set_chunks(document, chunks) + await safe_set_chunks(session, document, chunks) document.updated_at = get_current_timestamp() document.status = DocumentStatus.ready() diff --git a/surfsense_backend/app/tasks/connector_indexers/confluence_indexer.py b/surfsense_backend/app/tasks/connector_indexers/confluence_indexer.py index c28e82b8f..3495c59a4 100644 --- a/surfsense_backend/app/tasks/connector_indexers/confluence_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/confluence_indexer.py @@ -413,7 +413,7 @@ async def index_confluence_pages( "indexed_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), "connector_id": connector_id, } - safe_set_chunks(document, chunks) + await safe_set_chunks(session, document, chunks) document.updated_at = get_current_timestamp() document.status = DocumentStatus.ready() diff --git a/surfsense_backend/app/tasks/connector_indexers/discord_indexer.py b/surfsense_backend/app/tasks/connector_indexers/discord_indexer.py index 0421352ff..e8e80a646 100644 --- a/surfsense_backend/app/tasks/connector_indexers/discord_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/discord_indexer.py @@ -690,7 +690,7 @@ async def index_discord_messages( "indexed_at": datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"), "connector_id": connector_id, } - safe_set_chunks(document, chunks) + await safe_set_chunks(session, document, chunks) document.updated_at = get_current_timestamp() document.status = DocumentStatus.ready() diff --git a/surfsense_backend/app/tasks/connector_indexers/elasticsearch_indexer.py b/surfsense_backend/app/tasks/connector_indexers/elasticsearch_indexer.py index 212afff39..f07c6c580 100644 --- a/surfsense_backend/app/tasks/connector_indexers/elasticsearch_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/elasticsearch_indexer.py @@ -386,7 +386,7 @@ async def index_elasticsearch_documents( document.content_hash = item["content_hash"] document.unique_identifier_hash = item["unique_identifier_hash"] document.document_metadata = metadata - safe_set_chunks(document, chunks) + await safe_set_chunks(session, document, chunks) document.updated_at = get_current_timestamp() document.status = DocumentStatus.ready() diff --git a/surfsense_backend/app/tasks/connector_indexers/github_indexer.py b/surfsense_backend/app/tasks/connector_indexers/github_indexer.py index fc6634024..61607dda3 100644 --- a/surfsense_backend/app/tasks/connector_indexers/github_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/github_indexer.py @@ -415,7 +415,7 @@ async def index_github_repos( document.content_hash = item["content_hash"] document.embedding = summary_embedding document.document_metadata = doc_metadata - safe_set_chunks(document, chunks_data) + await safe_set_chunks(session, document, chunks_data) document.updated_at = get_current_timestamp() document.status = DocumentStatus.ready() diff --git a/surfsense_backend/app/tasks/connector_indexers/google_calendar_indexer.py b/surfsense_backend/app/tasks/connector_indexers/google_calendar_indexer.py index 1407d98dd..24e822060 100644 --- a/surfsense_backend/app/tasks/connector_indexers/google_calendar_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/google_calendar_indexer.py @@ -528,7 +528,7 @@ async def index_google_calendar_events( "indexed_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), "connector_id": connector_id, } - safe_set_chunks(document, chunks) + await safe_set_chunks(session, document, chunks) document.updated_at = get_current_timestamp() document.status = DocumentStatus.ready() diff --git a/surfsense_backend/app/tasks/connector_indexers/google_gmail_indexer.py b/surfsense_backend/app/tasks/connector_indexers/google_gmail_indexer.py index 1a8b2b176..6e2408cbd 100644 --- a/surfsense_backend/app/tasks/connector_indexers/google_gmail_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/google_gmail_indexer.py @@ -451,7 +451,7 @@ async def index_google_gmail_messages( "date": item["date_str"], "connector_id": connector_id, } - safe_set_chunks(document, chunks) + await safe_set_chunks(session, document, chunks) document.updated_at = get_current_timestamp() document.status = DocumentStatus.ready() diff --git a/surfsense_backend/app/tasks/connector_indexers/jira_indexer.py b/surfsense_backend/app/tasks/connector_indexers/jira_indexer.py index dec37428a..1765a592e 100644 --- a/surfsense_backend/app/tasks/connector_indexers/jira_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/jira_indexer.py @@ -393,7 +393,7 @@ async def index_jira_issues( "indexed_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), "connector_id": connector_id, } - safe_set_chunks(document, chunks) + await safe_set_chunks(session, document, chunks) document.updated_at = get_current_timestamp() document.status = DocumentStatus.ready() diff --git a/surfsense_backend/app/tasks/connector_indexers/linear_indexer.py b/surfsense_backend/app/tasks/connector_indexers/linear_indexer.py index 1a2254c5b..bacafccc7 100644 --- a/surfsense_backend/app/tasks/connector_indexers/linear_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/linear_indexer.py @@ -431,7 +431,7 @@ async def index_linear_issues( "indexed_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), "connector_id": connector_id, } - safe_set_chunks(document, chunks) + await safe_set_chunks(session, document, chunks) document.updated_at = get_current_timestamp() document.status = DocumentStatus.ready() diff --git a/surfsense_backend/app/tasks/connector_indexers/luma_indexer.py b/surfsense_backend/app/tasks/connector_indexers/luma_indexer.py index 2d86b09c1..83cf54f4e 100644 --- a/surfsense_backend/app/tasks/connector_indexers/luma_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/luma_indexer.py @@ -488,7 +488,7 @@ async def index_luma_events( "indexed_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), "connector_id": connector_id, } - safe_set_chunks(document, chunks) + await safe_set_chunks(session, document, chunks) document.updated_at = get_current_timestamp() document.status = DocumentStatus.ready() diff --git a/surfsense_backend/app/tasks/connector_indexers/notion_indexer.py b/surfsense_backend/app/tasks/connector_indexers/notion_indexer.py index b0c49dea5..85daff94c 100644 --- a/surfsense_backend/app/tasks/connector_indexers/notion_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/notion_indexer.py @@ -479,7 +479,7 @@ async def index_notion_pages( "indexed_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), "connector_id": connector_id, } - safe_set_chunks(document, chunks) + await safe_set_chunks(session, document, chunks) document.updated_at = get_current_timestamp() document.status = DocumentStatus.ready() diff --git a/surfsense_backend/app/tasks/connector_indexers/obsidian_indexer.py b/surfsense_backend/app/tasks/connector_indexers/obsidian_indexer.py index c0eef84d5..d53baa3b0 100644 --- a/surfsense_backend/app/tasks/connector_indexers/obsidian_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/obsidian_indexer.py @@ -571,7 +571,7 @@ async def index_obsidian_vault( document.content_hash = content_hash document.embedding = embedding document.document_metadata = document_metadata - safe_set_chunks(document, chunks) + await safe_set_chunks(session, document, chunks) document.updated_at = get_current_timestamp() document.status = DocumentStatus.ready() diff --git a/surfsense_backend/app/tasks/connector_indexers/slack_indexer.py b/surfsense_backend/app/tasks/connector_indexers/slack_indexer.py index f83b171bc..1f2693844 100644 --- a/surfsense_backend/app/tasks/connector_indexers/slack_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/slack_indexer.py @@ -564,7 +564,7 @@ async def index_slack_messages( "indexed_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), "connector_id": connector_id, } - safe_set_chunks(document, chunks) + await safe_set_chunks(session, document, chunks) document.updated_at = get_current_timestamp() document.status = DocumentStatus.ready() diff --git a/surfsense_backend/app/tasks/connector_indexers/teams_indexer.py b/surfsense_backend/app/tasks/connector_indexers/teams_indexer.py index ad34e8696..d04a98177 100644 --- a/surfsense_backend/app/tasks/connector_indexers/teams_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/teams_indexer.py @@ -603,7 +603,7 @@ async def index_teams_messages( "indexed_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), "connector_id": connector_id, } - safe_set_chunks(document, chunks) + await safe_set_chunks(session, document, chunks) document.updated_at = get_current_timestamp() document.status = DocumentStatus.ready() diff --git a/surfsense_backend/app/tasks/connector_indexers/webcrawler_indexer.py b/surfsense_backend/app/tasks/connector_indexers/webcrawler_indexer.py index 94361cc27..4d2644420 100644 --- a/surfsense_backend/app/tasks/connector_indexers/webcrawler_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/webcrawler_indexer.py @@ -410,7 +410,7 @@ async def index_crawled_urls( "indexed_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), "connector_id": connector_id, } - safe_set_chunks(document, chunks) + await safe_set_chunks(session, document, chunks) document.status = DocumentStatus.ready() # READY status document.updated_at = get_current_timestamp() diff --git a/surfsense_backend/app/tasks/document_processors/base.py b/surfsense_backend/app/tasks/document_processors/base.py index 2edc48e91..580126d48 100644 --- a/surfsense_backend/app/tasks/document_processors/base.py +++ b/surfsense_backend/app/tasks/document_processors/base.py @@ -14,45 +14,37 @@ from app.db import Document md = MarkdownifyTransformer() -def safe_set_chunks(document: Document, chunks: list) -> None: +async def safe_set_chunks( + session: "AsyncSession", document: Document, chunks: list +) -> None: """ - Safely assign chunks to a document without triggering lazy loading. + Delete old chunks and assign new ones to a document. - ALWAYS use this instead of `document.chunks = chunks` to avoid - SQLAlchemy async errors (MissingGreenlet / greenlet_spawn). - - Why this is needed: - - Direct assignment `document.chunks = chunks` triggers SQLAlchemy to - load the OLD chunks first (for comparison/orphan detection) - - This lazy loading fails in async context with asyncpg driver - - set_committed_value bypasses this by setting the value directly - - This function is safe regardless of how the document was loaded - (with or without selectinload). + This replaces direct ``document.chunks = chunks`` which triggers lazy + loading (and MissingGreenlet errors in async contexts). It also + explicitly deletes pre-existing chunks so they don't accumulate across + repeated re-indexes — ``set_committed_value`` bypasses SQLAlchemy's + delete-orphan cascade. Args: - document: The Document object to update - chunks: List of Chunk objects to assign - - Example: - # Instead of: document.chunks = chunks (DANGEROUS!) - safe_set_chunks(document, chunks) # Always safe + session: The current async database session. + document: The Document object to update. + chunks: List of Chunk objects to assign. """ - from sqlalchemy.orm import object_session + from sqlalchemy import delete from sqlalchemy.orm.attributes import set_committed_value - # Keep relationship assignment lazy-load-safe. - set_committed_value(document, "chunks", chunks) + from app.db import Chunk - # Ensure chunk rows are actually persisted. - # set_committed_value bypasses normal unit-of-work tracking, so we need to - # explicitly attach chunk objects to the current session. - session = object_session(document) - if session is not None: - if document.id is not None: - for chunk in chunks: - chunk.document_id = document.id - session.add_all(chunks) + if document.id is not None: + await session.execute( + delete(Chunk).where(Chunk.document_id == document.id) + ) + for chunk in chunks: + chunk.document_id = document.id + + set_committed_value(document, "chunks", chunks) + session.add_all(chunks) def get_current_timestamp() -> datetime: diff --git a/surfsense_backend/app/tasks/document_processors/circleback_processor.py b/surfsense_backend/app/tasks/document_processors/circleback_processor.py index a86b64499..a6b9568b9 100644 --- a/surfsense_backend/app/tasks/document_processors/circleback_processor.py +++ b/surfsense_backend/app/tasks/document_processors/circleback_processor.py @@ -227,7 +227,7 @@ async def add_circleback_meeting_document( if summary_embedding is not None: document.embedding = summary_embedding document.document_metadata = document_metadata - safe_set_chunks(document, chunks) + await safe_set_chunks(session, document, chunks) document.source_markdown = markdown_content document.content_needs_reindexing = False document.updated_at = get_current_timestamp() diff --git a/surfsense_backend/app/tasks/document_processors/extension_processor.py b/surfsense_backend/app/tasks/document_processors/extension_processor.py index a6e482e15..7320ec9fa 100644 --- a/surfsense_backend/app/tasks/document_processors/extension_processor.py +++ b/surfsense_backend/app/tasks/document_processors/extension_processor.py @@ -21,6 +21,7 @@ from app.utils.document_converters import ( from .base import ( check_document_by_unique_identifier, get_current_timestamp, + safe_set_chunks, ) @@ -154,7 +155,7 @@ async def add_extension_received_document( existing_document.content_hash = content_hash existing_document.embedding = summary_embedding existing_document.document_metadata = content.metadata.model_dump() - existing_document.chunks = chunks + await safe_set_chunks(session, existing_document, chunks) existing_document.source_markdown = combined_document_string existing_document.updated_at = get_current_timestamp() diff --git a/surfsense_backend/app/tasks/document_processors/file_processors.py b/surfsense_backend/app/tasks/document_processors/file_processors.py index 5e97951bd..647435213 100644 --- a/surfsense_backend/app/tasks/document_processors/file_processors.py +++ b/surfsense_backend/app/tasks/document_processors/file_processors.py @@ -35,6 +35,7 @@ from .base import ( check_document_by_unique_identifier, check_duplicate_document, get_current_timestamp, + safe_set_chunks, ) from .markdown_processor import add_received_markdown_file_document @@ -488,7 +489,7 @@ async def add_received_file_document_using_unstructured( "FILE_NAME": file_name, "ETL_SERVICE": "UNSTRUCTURED", } - existing_document.chunks = chunks + await safe_set_chunks(session, existing_document, chunks) existing_document.source_markdown = file_in_markdown existing_document.content_needs_reindexing = False existing_document.updated_at = get_current_timestamp() @@ -622,7 +623,7 @@ async def add_received_file_document_using_llamacloud( "FILE_NAME": file_name, "ETL_SERVICE": "LLAMACLOUD", } - existing_document.chunks = chunks + await safe_set_chunks(session, existing_document, chunks) existing_document.source_markdown = file_in_markdown existing_document.content_needs_reindexing = False existing_document.updated_at = get_current_timestamp() @@ -777,7 +778,7 @@ async def add_received_file_document_using_docling( "FILE_NAME": file_name, "ETL_SERVICE": "DOCLING", } - existing_document.chunks = chunks + await safe_set_chunks(session, existing_document, chunks) existing_document.source_markdown = file_in_markdown existing_document.content_needs_reindexing = False existing_document.updated_at = get_current_timestamp() diff --git a/surfsense_backend/app/tasks/document_processors/markdown_processor.py b/surfsense_backend/app/tasks/document_processors/markdown_processor.py index a8d20c062..d598bf9dd 100644 --- a/surfsense_backend/app/tasks/document_processors/markdown_processor.py +++ b/surfsense_backend/app/tasks/document_processors/markdown_processor.py @@ -21,6 +21,7 @@ from .base import ( check_document_by_unique_identifier, check_duplicate_document, get_current_timestamp, + safe_set_chunks, ) @@ -258,7 +259,7 @@ async def add_received_markdown_file_document( existing_document.document_metadata = { "FILE_NAME": file_name, } - existing_document.chunks = chunks + await safe_set_chunks(session, existing_document, chunks) existing_document.source_markdown = file_in_markdown existing_document.updated_at = get_current_timestamp() existing_document.status = DocumentStatus.ready() # Mark as ready diff --git a/surfsense_backend/app/tasks/document_processors/youtube_processor.py b/surfsense_backend/app/tasks/document_processors/youtube_processor.py index 13b969fb6..0ed2e57d2 100644 --- a/surfsense_backend/app/tasks/document_processors/youtube_processor.py +++ b/surfsense_backend/app/tasks/document_processors/youtube_processor.py @@ -419,7 +419,7 @@ async def add_youtube_video_document( "author": video_data.get("author_name", "Unknown"), "thumbnail": video_data.get("thumbnail_url", ""), } - safe_set_chunks(document, chunks) + await safe_set_chunks(session, document, chunks) document.source_markdown = combined_document_string document.status = DocumentStatus.ready() # READY status - fully processed document.updated_at = get_current_timestamp() diff --git a/surfsense_backend/app/tasks/surfsense_docs_indexer.py b/surfsense_backend/app/tasks/surfsense_docs_indexer.py index ca4a83de3..7d449c0ab 100644 --- a/surfsense_backend/app/tasks/surfsense_docs_indexer.py +++ b/surfsense_backend/app/tasks/surfsense_docs_indexer.py @@ -13,12 +13,32 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import selectinload +from sqlalchemy import delete as sa_delete +from sqlalchemy.orm.attributes import set_committed_value + from app.config import config from app.db import SurfsenseDocsChunk, SurfsenseDocsDocument, async_session_maker from app.utils.document_converters import embed_text logger = logging.getLogger(__name__) + +async def _safe_set_docs_chunks( + session: AsyncSession, document: SurfsenseDocsDocument, chunks: list +) -> None: + """safe_set_chunks variant for the SurfsenseDocsDocument/Chunk models.""" + if document.id is not None: + await session.execute( + sa_delete(SurfsenseDocsChunk).where( + SurfsenseDocsChunk.document_id == document.id + ) + ) + for chunk in chunks: + chunk.document_id = document.id + + set_committed_value(document, "chunks", chunks) + session.add_all(chunks) + # Path to docs relative to project root DOCS_DIR = ( Path(__file__).resolve().parent.parent.parent.parent @@ -156,7 +176,7 @@ async def index_surfsense_docs(session: AsyncSession) -> tuple[int, int, int, in existing_doc.content = content existing_doc.content_hash = content_hash existing_doc.embedding = embed_text(content) - existing_doc.chunks = chunks + await _safe_set_docs_chunks(session, existing_doc, chunks) existing_doc.updated_at = datetime.now(UTC) updated += 1 diff --git a/surfsense_web/components/assistant-ui/thread.tsx b/surfsense_web/components/assistant-ui/thread.tsx index d016efb24..b7a5bcf0e 100644 --- a/surfsense_web/components/assistant-ui/thread.tsx +++ b/surfsense_web/components/assistant-ui/thread.tsx @@ -19,10 +19,12 @@ import { ChevronRightIcon, CopyIcon, DownloadIcon, + Plus, RefreshCwIcon, + Settings2, SquareIcon, Unplug, - Wrench, + Upload, X, } from "lucide-react"; import { useParams } from "next/navigation"; @@ -53,6 +55,7 @@ import { currentUserAtom } from "@/atoms/user/user-query.atoms"; import { AssistantMessage } from "@/components/assistant-ui/assistant-message"; import { ChatSessionStatus } from "@/components/assistant-ui/chat-session-status"; import { ConnectorIndicator } from "@/components/assistant-ui/connector-popup"; +import { useDocumentUploadDialog } from "@/components/assistant-ui/document-upload-popup"; import { InlineMentionEditor, type InlineMentionEditorRef, @@ -73,6 +76,13 @@ import { import type { ThinkingStep } from "@/components/tool-ui/deepagent-thinking"; import { Avatar, AvatarFallback, AvatarGroup } from "@/components/ui/avatar"; import { Button } from "@/components/ui/button"; +import { Drawer, DrawerContent, DrawerHandle, DrawerTitle } from "@/components/ui/drawer"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; import { Switch } from "@/components/ui/switch"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; @@ -278,21 +288,14 @@ const ConnectToolsBanner: FC = () => { ))} - { - if (e.key === "Enter" || e.key === " ") { - e.preventDefault(); - handleDismiss(e as unknown as React.MouseEvent); - } - }} className="shrink-0 ml-0.5 p-0.5 text-muted-foreground/40 hover:text-foreground transition-colors" aria-label="Dismiss" > - + ); @@ -564,6 +567,7 @@ const ComposerAction: FC = ({ isBlockedByOtherUser = false const setConnectorDialogOpen = useSetAtom(connectorDialogOpenAtom); const [toolsPopoverOpen, setToolsPopoverOpen] = useState(false); const isDesktop = useMediaQuery("(min-width: 640px)"); + const { openDialog: openUploadDialog } = useDocumentUploadDialog(); const [toolsScrollPos, setToolsScrollPos] = useState<"top" | "middle" | "bottom">("top"); const handleToolsScroll = useCallback((e: React.UIEvent) => { const el = e.currentTarget; @@ -607,87 +611,144 @@ const ComposerAction: FC = ({ isBlockedByOtherUser = false return (
- - - + + + + + + setToolsPopoverOpen(true)}> + + Manage Tools + + openUploadDialog()}> + + Upload Files + + + + + + +
+ Agent Tools + + {enabledCount}/{agentTools?.length ?? 0} enabled + +
+
+ {agentTools?.map((tool) => { + const isDisabled = disabledTools.includes(tool.name); + return ( +
+ + {formatToolName(tool.name)} + + toggleTool(tool.name)} + className="shrink-0" + /> +
+ ); + })} + {!agentTools?.length && ( +
+ Loading tools... +
+ )} +
+ + + + + ) : ( + + + + + + + e.preventDefault()} > - {agentTools?.map((tool) => { - const isDisabled = disabledTools.includes(tool.name); - const row = ( - - ); - if (!isDesktop) { - return
{row}
; - } - return ( - - {row} - - {tool.description} - - - ); - })} - {!agentTools?.length && ( -
- Loading tools... -
- )} -
- - - {!isDesktop && ( - setConnectorDialogOpen(true)} - > - - +
+ Agent Tools + + {enabledCount}/{agentTools?.length ?? 0} enabled + +
+
+ {agentTools?.map((tool) => { + const isDisabled = disabledTools.includes(tool.name); + const row = ( +
+ + {formatToolName(tool.name)} + + toggleTool(tool.name)} + className="shrink-0 scale-[0.6] sm:scale-75" + /> +
+ ); + return ( + + {row} + + {tool.description} + + + ); + })} + {!agentTools?.length && ( +
+ Loading tools... +
+ )} +
+ + )} {sidebarDocs.length > 0 && (