From df286305494bfd47c5e0c82b9c13000f8be179e5 Mon Sep 17 00:00:00 2001 From: Ethan Clarke Date: Fri, 13 Mar 2026 07:24:15 +0800 Subject: [PATCH 1/7] docs: add MiniMax (M2.5) to Chinese LLM setup guide --- docs/chinese-llm-setup.md | 57 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 53 insertions(+), 4 deletions(-) diff --git a/docs/chinese-llm-setup.md b/docs/chinese-llm-setup.md index 2a184608f..eb479bcfa 100644 --- a/docs/chinese-llm-setup.md +++ b/docs/chinese-llm-setup.md @@ -14,6 +14,7 @@ SurfSense 现已支持以下国产 LLM： - ✅ **阿里通义千问 (Alibaba Qwen)** - 阿里云通义千问大模型 - ✅ **月之暗面 Kimi (Moonshot)** - 月之暗面 Kimi 大模型 - ✅ **智谱 AI GLM (Zhipu)** - 智谱 AI GLM 系列模型 +- ✅ **MiniMax** - MiniMax 大模型 (M2.5 系列，204K 上下文) --- @@ -197,6 +198,52 @@ API Base URL: https://open.bigmodel.cn/api/paas/v4 --- +## 5️⃣ MiniMax 配置 | MiniMax Configuration + +### 获取 API Key + +1. 访问 [MiniMax 开放平台](https://platform.minimaxi.com/) +2. 注册并登录账号 +3. 进入 **API Keys** 页面 +4. 创建新的 API Key +5. 复制 API Key + +### 在 SurfSense 中配置 + +| 字段 | 值 | 说明 | +|------|-----|------| +| **Configuration Name** | `MiniMax M2.5` | 配置名称（自定义） | +| **Provider** | `MINIMAX` | 选择 MiniMax | +| **Model Name** | `MiniMax-M2.5` | 推荐模型
其他选项: `MiniMax-M2.5-highspeed` | +| **API Key** | `eyJ...` | 你的 MiniMax API Key | +| **API Base URL** | `https://api.minimax.io/v1` | MiniMax API 地址 | +| **Parameters** | `{"temperature": 1.0}` | MiniMax 要求 temperature 在 (0.0, 1.0] 范围内 | + +### 示例配置 + +``` +Configuration Name: MiniMax M2.5 +Provider: MINIMAX +Model Name: MiniMax-M2.5 +API Key: eyJxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +API Base URL: https://api.minimax.io/v1 +``` + +### 可用模型 + +- **MiniMax-M2.5**: 旗舰模型，204K 上下文窗口（推荐） +- **MiniMax-M2.5-highspeed**: 高速版本，204K 上下文窗口，更快响应 + +### 注意事项 + +- ⚠️ MiniMax API 要求 `temperature` 参数在 `(0.0, 1.0]` 范围内，不能设为 `0` +- 💡 如果使用默认参数时报错，请在 Parameters 中设置 `{"temperature": 1.0}` + +### 定价 +- 请访问 [MiniMax 开放平台定价](https://platform.minimaxi.com/document/Price) 查看最新价格 + +--- + ## ⚙️ 高级配置 | Advanced Configuration ### 自定义参数 | Custom Parameters @@ -268,13 +315,13 @@ docker compose logs backend | grep -i "error" |---------|---------|------| | **文档摘要** | Qwen-Plus, GLM-4 | 平衡性能和成本 | | **代码分析** | DeepSeek-Coder | 代码专用 | -| **长文本处理** | Kimi 128K | 超长上下文 | -| **快速响应** | Qwen-Turbo, GLM-4-Flash | 速度优先 | +| **长文本处理** | Kimi 128K, MiniMax-M2.5 | 超长上下文（204K） | +| **快速响应** | Qwen-Turbo, GLM-4-Flash, MiniMax-M2.5-highspeed | 速度优先 | ### 2. 成本优化 -- 🎯 **Long Context LLM**: 使用 Qwen-Plus 或 GLM-4（处理文档摘要） -- ⚡ **Fast LLM**: 使用 Qwen-Turbo 或 GLM-4-Flash（快速对话） +- 🎯 **Long Context LLM**: 使用 Qwen-Plus、GLM-4 或 MiniMax-M2.5（处理文档摘要，支持 204K 上下文） +- ⚡ **Fast LLM**: 使用 Qwen-Turbo、GLM-4-Flash 或 MiniMax-M2.5-highspeed（快速对话） - 🧠 **Strategic LLM**: 使用 Qwen-Max 或 DeepSeek-Chat（复杂推理） ### 3. API Key 安全 @@ -294,6 +341,7 @@ docker compose logs backend | grep -i "error" - [阿里云百炼文档](https://help.aliyun.com/zh/model-studio/) - [Moonshot AI 文档](https://platform.moonshot.cn/docs) - [智谱 AI 文档](https://open.bigmodel.cn/dev/api) +- [MiniMax 文档](https://platform.minimaxi.com/document/introduction) ### SurfSense 文档 @@ -315,6 +363,7 @@ docker compose logs backend | grep -i "error" ## 🔄 更新日志 | Changelog +- **2026-03-13**: 添加 MiniMax (M2.5) 配置指南 - **2025-01-12**: 初始版本，添加 DeepSeek、Qwen、Kimi、GLM 支持 --- From 760aa38225fe75cecd96feac09498bacf40ad643 Mon Sep 17 00:00:00 2001 From: PR Bot Date: Fri, 13 Mar 2026 07:27:47 +0800 Subject: [PATCH 2/7] feat: complete MiniMax LLM provider integration Add full MiniMax provider support across the entire stack: Backend: - Add MINIMAX to LiteLLMProvider enum in db.py - Add MINIMAX mapping to all provider_map dicts in llm_service.py, llm_router_service.py, and llm_config.py - Add Alembic migration (rev 106) for PostgreSQL enum - Add MiniMax M2.5 example in global_llm_config.example.yaml Frontend: - Add MiniMax to LLM_PROVIDERS enum with apiBase - Add MiniMax-M2.5 and MiniMax-M2.5-highspeed to LLM_MODELS - Add MINIMAX to Zod validation schema - Add MiniMax SVG icon and wire up in provider-icons Docs: - Add MiniMax setup guide in chinese-llm-setup.md MiniMax uses an OpenAI-compatible API (https://api.minimax.io/v1) with models supporting up to 204K context window. Co-Authored-By: Claude Opus 4.6 --- docs/chinese-llm-setup.md | 52 ++++++++++++++++++- ...106_add_minimax_to_litellmprovider_enum.py | 23 ++++++++ .../app/agents/new_chat/llm_config.py | 1 + .../app/config/global_llm_config.example.yaml | 17 ++++++ surfsense_backend/app/db.py | 1 + .../app/services/llm_router_service.py | 1 + surfsense_backend/app/services/llm_service.py | 3 ++ .../components/icons/providers/index.ts | 1 + .../components/icons/providers/minimax.svg | 1 + surfsense_web/contracts/enums/llm-models.ts | 14 +++++ .../contracts/enums/llm-providers.ts | 7 +++ .../contracts/types/new-llm-config.types.ts | 1 + surfsense_web/lib/provider-icons.tsx | 3 ++ 13 files changed, 123 insertions(+), 2 deletions(-) create mode 100644 surfsense_backend/alembic/versions/106_add_minimax_to_litellmprovider_enum.py create mode 100644 surfsense_web/components/icons/providers/minimax.svg diff --git a/docs/chinese-llm-setup.md b/docs/chinese-llm-setup.md index 2a184608f..37042aa2f 100644 --- a/docs/chinese-llm-setup.md +++ b/docs/chinese-llm-setup.md @@ -14,6 +14,7 @@ SurfSense 现已支持以下国产 LLM： - ✅ **阿里通义千问 (Alibaba Qwen)** - 阿里云通义千问大模型 - ✅ **月之暗面 Kimi (Moonshot)** - 月之暗面 Kimi 大模型 - ✅ **智谱 AI GLM (Zhipu)** - 智谱 AI GLM 系列模型 +- ✅ **MiniMax** - MiniMax 大模型 (M2.5 系列，204K 上下文) --- @@ -197,6 +198,52 @@ API Base URL: https://open.bigmodel.cn/api/paas/v4 --- +## 5️⃣ MiniMax 配置 | MiniMax Configuration + +### 获取 API Key + +1. 访问 [MiniMax 开放平台](https://platform.minimaxi.com/) +2. 注册并登录账号 +3. 进入 **API Keys** 页面 +4. 创建新的 API Key +5. 复制 API Key + +### 在 SurfSense 中配置 + +| 字段 | 值 | 说明 | +|------|-----|------| +| **Configuration Name** | `MiniMax M2.5` | 配置名称（自定义） | +| **Provider** | `MINIMAX` | 选择 MiniMax | +| **Model Name** | `MiniMax-M2.5` | 推荐模型
其他选项: `MiniMax-M2.5-highspeed` | +| **API Key** | `eyJ...` | 你的 MiniMax API Key | +| **API Base URL** | `https://api.minimax.io/v1` | MiniMax API 地址 | +| **Parameters** | `{"temperature": 1.0}` | 注意：temperature 必须在 (0.0, 1.0] 范围内，不能为 0 | + +### 示例配置 + +``` +Configuration Name: MiniMax M2.5 +Provider: MINIMAX +Model Name: MiniMax-M2.5 +API Key: eyJxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +API Base URL: https://api.minimax.io/v1 +``` + +### 可用模型 + +- **MiniMax-M2.5**: 高性能通用模型，204K 上下文窗口（推荐） +- **MiniMax-M2.5-highspeed**: 高速推理版本，204K 上下文窗口 + +### 注意事项 + +- **temperature 参数**: MiniMax 要求 temperature 必须在 (0.0, 1.0] 范围内，不能设置为 0。建议使用 1.0。 +- 两个模型都支持 204K 超长上下文窗口，适合处理长文本任务。 + +### 定价 +- 请访问 [MiniMax 定价页面](https://platform.minimaxi.com/document/Price) 查看最新价格 + +--- + ## ⚙️ 高级配置 | Advanced Configuration ### 自定义参数 | Custom Parameters @@ -268,8 +315,8 @@ docker compose logs backend | grep -i "error" |---------|---------|------| | **文档摘要** | Qwen-Plus, GLM-4 | 平衡性能和成本 | | **代码分析** | DeepSeek-Coder | 代码专用 | -| **长文本处理** | Kimi 128K | 超长上下文 | -| **快速响应** | Qwen-Turbo, GLM-4-Flash | 速度优先 | +| **长文本处理** | Kimi 128K, MiniMax-M2.5 (204K) | 超长上下文 | +| **快速响应** | Qwen-Turbo, GLM-4-Flash, MiniMax-M2.5-highspeed | 速度优先 | ### 2. 成本优化 @@ -294,6 +341,7 @@ docker compose logs backend | grep -i "error" - [阿里云百炼文档](https://help.aliyun.com/zh/model-studio/) - [Moonshot AI 文档](https://platform.moonshot.cn/docs) - [智谱 AI 文档](https://open.bigmodel.cn/dev/api) +- [MiniMax 文档](https://platform.minimaxi.com/document/Guides) ### SurfSense 文档 diff --git a/surfsense_backend/alembic/versions/106_add_minimax_to_litellmprovider_enum.py b/surfsense_backend/alembic/versions/106_add_minimax_to_litellmprovider_enum.py new file mode 100644 index 000000000..fed3bc7c3 --- /dev/null +++ b/surfsense_backend/alembic/versions/106_add_minimax_to_litellmprovider_enum.py @@ -0,0 +1,23 @@ +"""Add MINIMAX to LiteLLMProvider enum + +Revision ID: 106 +Revises: 105 +""" + +from collections.abc import Sequence + +from alembic import op + +revision: str = "106" +down_revision: str | None = "105" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.execute("COMMIT") + op.execute("ALTER TYPE litellmprovider ADD VALUE IF NOT EXISTS 'MINIMAX'") + + +def downgrade() -> None: + pass diff --git a/surfsense_backend/app/agents/new_chat/llm_config.py b/surfsense_backend/app/agents/new_chat/llm_config.py index 4ddb47330..60cd2a452 100644 --- a/surfsense_backend/app/agents/new_chat/llm_config.py +++ b/surfsense_backend/app/agents/new_chat/llm_config.py @@ -59,6 +59,7 @@ PROVIDER_MAP = { "DATABRICKS": "databricks", "COMETAPI": "cometapi", "HUGGINGFACE": "huggingface", + "MINIMAX": "openai", "CUSTOM": "custom", } diff --git a/surfsense_backend/app/config/global_llm_config.example.yaml b/surfsense_backend/app/config/global_llm_config.example.yaml index 0bb00c398..6ca3e95e3 100644 --- a/surfsense_backend/app/config/global_llm_config.example.yaml +++ b/surfsense_backend/app/config/global_llm_config.example.yaml @@ -183,6 +183,23 @@ global_llm_configs: use_default_system_instructions: true citations_enabled: true + # Example: MiniMax M2.5 - High-performance with 204K context window + - id: -8 + name: "Global MiniMax M2.5" + description: "MiniMax M2.5 with 204K context window and competitive pricing" + provider: "MINIMAX" + model_name: "MiniMax-M2.5" + api_key: "your-minimax-api-key-here" + api_base: "https://api.minimax.io/v1" + rpm: 60 + tpm: 100000 + litellm_params: + temperature: 1.0 # MiniMax requires temperature in (0.0, 1.0], cannot be 0 + max_tokens: 4000 + system_instructions: "" + use_default_system_instructions: true + citations_enabled: true + # ============================================================================= # Image Generation Configuration # ============================================================================= diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py index 062b11b3a..95ae8e728 100644 --- a/surfsense_backend/app/db.py +++ b/surfsense_backend/app/db.py @@ -215,6 +215,7 @@ class LiteLLMProvider(StrEnum): COMETAPI = "COMETAPI" HUGGINGFACE = "HUGGINGFACE" GITHUB_MODELS = "GITHUB_MODELS" + MINIMAX = "MINIMAX" CUSTOM = "CUSTOM" diff --git a/surfsense_backend/app/services/llm_router_service.py b/surfsense_backend/app/services/llm_router_service.py index 7a0b6e55b..63d8d10b9 100644 --- a/surfsense_backend/app/services/llm_router_service.py +++ b/surfsense_backend/app/services/llm_router_service.py @@ -85,6 +85,7 @@ PROVIDER_MAP = { "ZHIPU": "openai", "GITHUB_MODELS": "github", "HUGGINGFACE": "huggingface", + "MINIMAX": "openai", "CUSTOM": "custom", } diff --git a/surfsense_backend/app/services/llm_service.py b/surfsense_backend/app/services/llm_service.py index fc28f477f..e11abd886 100644 --- a/surfsense_backend/app/services/llm_service.py +++ b/surfsense_backend/app/services/llm_service.py @@ -127,6 +127,7 @@ async def validate_llm_config( "ALIBABA_QWEN": "openai", "MOONSHOT": "openai", "ZHIPU": "openai", # GLM needs special handling + "MINIMAX": "openai", "GITHUB_MODELS": "github", } provider_prefix = provider_map.get(provider, provider.lower()) @@ -277,6 +278,7 @@ async def get_search_space_llm_instance( "ALIBABA_QWEN": "openai", "MOONSHOT": "openai", "ZHIPU": "openai", + "MINIMAX": "openai", } provider_prefix = provider_map.get( global_config["provider"], global_config["provider"].lower() @@ -350,6 +352,7 @@ async def get_search_space_llm_instance( "ALIBABA_QWEN": "openai", "MOONSHOT": "openai", "ZHIPU": "openai", + "MINIMAX": "openai", "GITHUB_MODELS": "github", } provider_prefix = provider_map.get( diff --git a/surfsense_web/components/icons/providers/index.ts b/surfsense_web/components/icons/providers/index.ts index 0a9bdcc66..2afed7fa5 100644 --- a/surfsense_web/components/icons/providers/index.ts +++ b/surfsense_web/components/icons/providers/index.ts @@ -12,6 +12,7 @@ export { default as FireworksAiIcon } from "./fireworksai.svg"; export { default as GeminiIcon } from "./gemini.svg"; export { default as GroqIcon } from "./groq.svg"; export { default as HuggingFaceIcon } from "./huggingface.svg"; +export { default as MiniMaxIcon } from "./minimax.svg"; export { default as MistralIcon } from "./mistral.svg"; export { default as MoonshotIcon } from "./moonshot.svg"; export { default as NscaleIcon } from "./nscale.svg"; diff --git a/surfsense_web/components/icons/providers/minimax.svg b/surfsense_web/components/icons/providers/minimax.svg new file mode 100644 index 000000000..85ad3962b --- /dev/null +++ b/surfsense_web/components/icons/providers/minimax.svg @@ -0,0 +1 @@ + diff --git a/surfsense_web/contracts/enums/llm-models.ts b/surfsense_web/contracts/enums/llm-models.ts index 91eb0cbd8..31097ca6e 100644 --- a/surfsense_web/contracts/enums/llm-models.ts +++ b/surfsense_web/contracts/enums/llm-models.ts @@ -1525,6 +1525,20 @@ export const LLM_MODELS: LLMModel[] = [ provider: "GITHUB_MODELS", contextWindow: "64K", }, + + // MiniMax + { + value: "MiniMax-M2.5", + label: "MiniMax M2.5", + provider: "MINIMAX", + contextWindow: "204K", + }, + { + value: "MiniMax-M2.5-highspeed", + label: "MiniMax M2.5 Highspeed", + provider: "MINIMAX", + contextWindow: "204K", + }, ]; // Helper function to get models by provider diff --git a/surfsense_web/contracts/enums/llm-providers.ts b/surfsense_web/contracts/enums/llm-providers.ts index ef03ca80c..ce2b6afe9 100644 --- a/surfsense_web/contracts/enums/llm-providers.ts +++ b/surfsense_web/contracts/enums/llm-providers.ts @@ -181,6 +181,13 @@ export const LLM_PROVIDERS: LLMProvider[] = [ description: "AI models from GitHub Marketplace", apiBase: "https://models.github.ai/inference", }, + { + value: "MINIMAX", + label: "MiniMax", + example: "MiniMax-M2.5, MiniMax-M2.5-highspeed", + description: "High-performance models with 204K context", + apiBase: "https://api.minimax.io/v1", + }, { value: "CUSTOM", label: "Custom Provider", diff --git a/surfsense_web/contracts/types/new-llm-config.types.ts b/surfsense_web/contracts/types/new-llm-config.types.ts index 2814f2f25..3bb43680a 100644 --- a/surfsense_web/contracts/types/new-llm-config.types.ts +++ b/surfsense_web/contracts/types/new-llm-config.types.ts @@ -34,6 +34,7 @@ export const liteLLMProviderEnum = z.enum([ "COMETAPI", "HUGGINGFACE", "GITHUB_MODELS", + "MINIMAX", "CUSTOM", ]); diff --git a/surfsense_web/lib/provider-icons.tsx b/surfsense_web/lib/provider-icons.tsx index e405b7766..d017d9aa2 100644 --- a/surfsense_web/lib/provider-icons.tsx +++ b/surfsense_web/lib/provider-icons.tsx @@ -15,6 +15,7 @@ import { GeminiIcon, GroqIcon, HuggingFaceIcon, + MiniMaxIcon, MistralIcon, MoonshotIcon, NscaleIcon, @@ -85,6 +86,8 @@ export function getProviderIcon( return ; case "HUGGINGFACE": return ; + case "MINIMAX": + return ; case "MISTRAL": return ; case "MOONSHOT": From 2b33dfe728e0742acb9b41b88af6ec004c583e65 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Sun, 15 Mar 2026 00:44:27 -0700 Subject: [PATCH 3/7] refactor: update safe_set_chunks function to be asynchronous and modify all connector and document processor files to use the new async implementation --- .../connectors/composio_gmail_connector.py | 2 +- .../composio_google_calendar_connector.py | 2 +- .../composio_google_drive_connector.py | 4 +- .../app/services/linear/kb_sync_service.py | 9 +--- .../app/services/notion/kb_sync_service.py | 10 +--- .../connector_indexers/airtable_indexer.py | 2 +- .../app/tasks/connector_indexers/base.py | 54 ++++++++----------- .../connector_indexers/bookstack_indexer.py | 2 +- .../connector_indexers/clickup_indexer.py | 2 +- .../connector_indexers/confluence_indexer.py | 2 +- .../connector_indexers/discord_indexer.py | 2 +- .../elasticsearch_indexer.py | 2 +- .../connector_indexers/github_indexer.py | 2 +- .../google_calendar_indexer.py | 2 +- .../google_gmail_indexer.py | 2 +- .../tasks/connector_indexers/jira_indexer.py | 2 +- .../connector_indexers/linear_indexer.py | 2 +- .../tasks/connector_indexers/luma_indexer.py | 2 +- .../connector_indexers/notion_indexer.py | 2 +- .../connector_indexers/obsidian_indexer.py | 2 +- .../tasks/connector_indexers/slack_indexer.py | 2 +- .../tasks/connector_indexers/teams_indexer.py | 2 +- .../connector_indexers/webcrawler_indexer.py | 2 +- .../app/tasks/document_processors/base.py | 54 ++++++++----------- .../circleback_processor.py | 2 +- .../extension_processor.py | 3 +- .../document_processors/file_processors.py | 7 +-- .../document_processors/markdown_processor.py | 3 +- .../document_processors/youtube_processor.py | 2 +- .../app/tasks/surfsense_docs_indexer.py | 22 +++++++- 30 files changed, 102 insertions(+), 106 deletions(-) diff --git a/surfsense_backend/app/connectors/composio_gmail_connector.py b/surfsense_backend/app/connectors/composio_gmail_connector.py index 2a382f3b8..e675085db 100644 --- a/surfsense_backend/app/connectors/composio_gmail_connector.py +++ b/surfsense_backend/app/connectors/composio_gmail_connector.py @@ -463,7 +463,7 @@ async def _process_gmail_messages_phase2( "connector_id": connector_id, "source": "composio", } - safe_set_chunks(document, chunks) + await safe_set_chunks(session, document, chunks) document.updated_at = get_current_timestamp() document.status = DocumentStatus.ready() diff --git a/surfsense_backend/app/connectors/composio_google_calendar_connector.py b/surfsense_backend/app/connectors/composio_google_calendar_connector.py index 63bade873..6344f9f38 100644 --- a/surfsense_backend/app/connectors/composio_google_calendar_connector.py +++ b/surfsense_backend/app/connectors/composio_google_calendar_connector.py @@ -477,7 +477,7 @@ async def index_composio_google_calendar( "connector_id": connector_id, "source": "composio", } - safe_set_chunks(document, chunks) + await safe_set_chunks(session, document, chunks) document.updated_at = get_current_timestamp() document.status = DocumentStatus.ready() diff --git a/surfsense_backend/app/connectors/composio_google_drive_connector.py b/surfsense_backend/app/connectors/composio_google_drive_connector.py index c10edb7e9..30ce4a77b 100644 --- a/surfsense_backend/app/connectors/composio_google_drive_connector.py +++ b/surfsense_backend/app/connectors/composio_google_drive_connector.py @@ -1112,7 +1112,7 @@ async def _index_composio_drive_delta_sync( "connector_id": connector_id, "source": "composio", } - safe_set_chunks(document, chunks) + await safe_set_chunks(session, document, chunks) document.updated_at = get_current_timestamp() document.status = DocumentStatus.ready() @@ -1520,7 +1520,7 @@ async def _index_composio_drive_full_scan( "connector_id": connector_id, "source": "composio", } - safe_set_chunks(document, chunks) + await safe_set_chunks(session, document, chunks) document.updated_at = get_current_timestamp() document.status = DocumentStatus.ready() diff --git a/surfsense_backend/app/services/linear/kb_sync_service.py b/surfsense_backend/app/services/linear/kb_sync_service.py index 8d1bc47c7..4f97a3dd0 100644 --- a/surfsense_backend/app/services/linear/kb_sync_service.py +++ b/surfsense_backend/app/services/linear/kb_sync_service.py @@ -1,11 +1,10 @@ import logging from datetime import datetime -from sqlalchemy import delete from sqlalchemy.ext.asyncio import AsyncSession from app.connectors.linear_connector import LinearConnector -from app.db import Chunk, Document +from app.db import Document from app.services.llm_service import get_user_long_context_llm from app.utils.document_converters import ( create_document_chunks, @@ -105,10 +104,6 @@ class LinearKBSyncService: ) summary_embedding = embed_text(summary_content) - await self.db_session.execute( - delete(Chunk).where(Chunk.document_id == document.id) - ) - chunks = await create_document_chunks(issue_content) document.title = f"{issue_identifier}: {issue_title}" @@ -131,7 +126,7 @@ class LinearKBSyncService: "connector_id": connector_id, } flag_modified(document, "document_metadata") - safe_set_chunks(document, chunks) + await safe_set_chunks(self.db_session, document, chunks) document.updated_at = get_current_timestamp() await self.db_session.commit() diff --git a/surfsense_backend/app/services/notion/kb_sync_service.py b/surfsense_backend/app/services/notion/kb_sync_service.py index ce31e0d35..d6c64897f 100644 --- a/surfsense_backend/app/services/notion/kb_sync_service.py +++ b/surfsense_backend/app/services/notion/kb_sync_service.py @@ -1,10 +1,9 @@ import logging from datetime import datetime -from sqlalchemy import delete from sqlalchemy.ext.asyncio import AsyncSession -from app.db import Chunk, Document +from app.db import Document from app.services.llm_service import get_user_long_context_llm from app.utils.document_converters import ( create_document_chunks, @@ -130,11 +129,6 @@ class NotionKBSyncService: summary_content = f"Notion Page: {document.document_metadata.get('page_title')}\n\n{full_content}" summary_embedding = embed_text(summary_content) - logger.debug(f"Deleting old chunks for document {document_id}") - await self.db_session.execute( - delete(Chunk).where(Chunk.document_id == document.id) - ) - logger.debug("Creating new chunks") chunks = await create_document_chunks(full_content) logger.debug(f"Created {len(chunks)} chunks") @@ -147,7 +141,7 @@ class NotionKBSyncService: **document.document_metadata, "indexed_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), } - safe_set_chunks(document, chunks) + await safe_set_chunks(self.db_session, document, chunks) document.updated_at = get_current_timestamp() logger.debug("Committing changes to database") diff --git a/surfsense_backend/app/tasks/connector_indexers/airtable_indexer.py b/surfsense_backend/app/tasks/connector_indexers/airtable_indexer.py index 438a93815..6f020685a 100644 --- a/surfsense_backend/app/tasks/connector_indexers/airtable_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/airtable_indexer.py @@ -432,7 +432,7 @@ async def index_airtable_records( "table_name": item["table_name"], "connector_id": connector_id, } - safe_set_chunks(document, chunks) + await safe_set_chunks(session, document, chunks) document.updated_at = get_current_timestamp() document.status = DocumentStatus.ready() diff --git a/surfsense_backend/app/tasks/connector_indexers/base.py b/surfsense_backend/app/tasks/connector_indexers/base.py index 139aed1d3..b6ce2f2f9 100644 --- a/surfsense_backend/app/tasks/connector_indexers/base.py +++ b/surfsense_backend/app/tasks/connector_indexers/base.py @@ -28,45 +28,37 @@ def get_current_timestamp() -> datetime: return datetime.now(UTC) -def safe_set_chunks(document: Document, chunks: list) -> None: +async def safe_set_chunks( + session: "AsyncSession", document: Document, chunks: list +) -> None: """ - Safely assign chunks to a document without triggering lazy loading. + Delete old chunks and assign new ones to a document. - ALWAYS use this instead of `document.chunks = chunks` to avoid - SQLAlchemy async errors (MissingGreenlet / greenlet_spawn). - - Why this is needed: - - Direct assignment `document.chunks = chunks` triggers SQLAlchemy to - load the OLD chunks first (for comparison/orphan detection) - - This lazy loading fails in async context with asyncpg driver - - set_committed_value bypasses this by setting the value directly - - This function is safe regardless of how the document was loaded - (with or without selectinload). + This replaces direct ``document.chunks = chunks`` which triggers lazy + loading (and MissingGreenlet errors in async contexts). It also + explicitly deletes pre-existing chunks so they don't accumulate across + repeated re-indexes — ``set_committed_value`` bypasses SQLAlchemy's + delete-orphan cascade. Args: - document: The Document object to update - chunks: List of Chunk objects to assign - - Example: - # Instead of: document.chunks = chunks (DANGEROUS!) - safe_set_chunks(document, chunks) # Always safe + session: The current async database session. + document: The Document object to update. + chunks: List of Chunk objects to assign. """ - from sqlalchemy.orm import object_session + from sqlalchemy import delete from sqlalchemy.orm.attributes import set_committed_value - # Keep relationship assignment lazy-load-safe. - set_committed_value(document, "chunks", chunks) + from app.db import Chunk - # Ensure chunk rows are actually persisted. - # set_committed_value bypasses normal unit-of-work tracking, so we need to - # explicitly attach chunk objects to the current session. - session = object_session(document) - if session is not None: - if document.id is not None: - for chunk in chunks: - chunk.document_id = document.id - session.add_all(chunks) + if document.id is not None: + await session.execute( + delete(Chunk).where(Chunk.document_id == document.id) + ) + for chunk in chunks: + chunk.document_id = document.id + + set_committed_value(document, "chunks", chunks) + session.add_all(chunks) def parse_date_flexible(date_str: str) -> datetime: diff --git a/surfsense_backend/app/tasks/connector_indexers/bookstack_indexer.py b/surfsense_backend/app/tasks/connector_indexers/bookstack_indexer.py index bf3aaa35f..0660531b2 100644 --- a/surfsense_backend/app/tasks/connector_indexers/bookstack_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/bookstack_indexer.py @@ -430,7 +430,7 @@ async def index_bookstack_pages( document.content_hash = item["content_hash"] document.embedding = summary_embedding document.document_metadata = doc_metadata - safe_set_chunks(document, chunks) + await safe_set_chunks(session, document, chunks) document.updated_at = get_current_timestamp() document.status = DocumentStatus.ready() diff --git a/surfsense_backend/app/tasks/connector_indexers/clickup_indexer.py b/surfsense_backend/app/tasks/connector_indexers/clickup_indexer.py index fd0233e87..af796ba3c 100644 --- a/surfsense_backend/app/tasks/connector_indexers/clickup_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/clickup_indexer.py @@ -439,7 +439,7 @@ async def index_clickup_tasks( "connector_id": connector_id, "indexed_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), } - safe_set_chunks(document, chunks) + await safe_set_chunks(session, document, chunks) document.updated_at = get_current_timestamp() document.status = DocumentStatus.ready() diff --git a/surfsense_backend/app/tasks/connector_indexers/confluence_indexer.py b/surfsense_backend/app/tasks/connector_indexers/confluence_indexer.py index c28e82b8f..3495c59a4 100644 --- a/surfsense_backend/app/tasks/connector_indexers/confluence_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/confluence_indexer.py @@ -413,7 +413,7 @@ async def index_confluence_pages( "indexed_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), "connector_id": connector_id, } - safe_set_chunks(document, chunks) + await safe_set_chunks(session, document, chunks) document.updated_at = get_current_timestamp() document.status = DocumentStatus.ready() diff --git a/surfsense_backend/app/tasks/connector_indexers/discord_indexer.py b/surfsense_backend/app/tasks/connector_indexers/discord_indexer.py index 0421352ff..e8e80a646 100644 --- a/surfsense_backend/app/tasks/connector_indexers/discord_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/discord_indexer.py @@ -690,7 +690,7 @@ async def index_discord_messages( "indexed_at": datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"), "connector_id": connector_id, } - safe_set_chunks(document, chunks) + await safe_set_chunks(session, document, chunks) document.updated_at = get_current_timestamp() document.status = DocumentStatus.ready() diff --git a/surfsense_backend/app/tasks/connector_indexers/elasticsearch_indexer.py b/surfsense_backend/app/tasks/connector_indexers/elasticsearch_indexer.py index 212afff39..f07c6c580 100644 --- a/surfsense_backend/app/tasks/connector_indexers/elasticsearch_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/elasticsearch_indexer.py @@ -386,7 +386,7 @@ async def index_elasticsearch_documents( document.content_hash = item["content_hash"] document.unique_identifier_hash = item["unique_identifier_hash"] document.document_metadata = metadata - safe_set_chunks(document, chunks) + await safe_set_chunks(session, document, chunks) document.updated_at = get_current_timestamp() document.status = DocumentStatus.ready() diff --git a/surfsense_backend/app/tasks/connector_indexers/github_indexer.py b/surfsense_backend/app/tasks/connector_indexers/github_indexer.py index fc6634024..61607dda3 100644 --- a/surfsense_backend/app/tasks/connector_indexers/github_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/github_indexer.py @@ -415,7 +415,7 @@ async def index_github_repos( document.content_hash = item["content_hash"] document.embedding = summary_embedding document.document_metadata = doc_metadata - safe_set_chunks(document, chunks_data) + await safe_set_chunks(session, document, chunks_data) document.updated_at = get_current_timestamp() document.status = DocumentStatus.ready() diff --git a/surfsense_backend/app/tasks/connector_indexers/google_calendar_indexer.py b/surfsense_backend/app/tasks/connector_indexers/google_calendar_indexer.py index 1407d98dd..24e822060 100644 --- a/surfsense_backend/app/tasks/connector_indexers/google_calendar_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/google_calendar_indexer.py @@ -528,7 +528,7 @@ async def index_google_calendar_events( "indexed_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), "connector_id": connector_id, } - safe_set_chunks(document, chunks) + await safe_set_chunks(session, document, chunks) document.updated_at = get_current_timestamp() document.status = DocumentStatus.ready() diff --git a/surfsense_backend/app/tasks/connector_indexers/google_gmail_indexer.py b/surfsense_backend/app/tasks/connector_indexers/google_gmail_indexer.py index 1a8b2b176..6e2408cbd 100644 --- a/surfsense_backend/app/tasks/connector_indexers/google_gmail_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/google_gmail_indexer.py @@ -451,7 +451,7 @@ async def index_google_gmail_messages( "date": item["date_str"], "connector_id": connector_id, } - safe_set_chunks(document, chunks) + await safe_set_chunks(session, document, chunks) document.updated_at = get_current_timestamp() document.status = DocumentStatus.ready() diff --git a/surfsense_backend/app/tasks/connector_indexers/jira_indexer.py b/surfsense_backend/app/tasks/connector_indexers/jira_indexer.py index dec37428a..1765a592e 100644 --- a/surfsense_backend/app/tasks/connector_indexers/jira_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/jira_indexer.py @@ -393,7 +393,7 @@ async def index_jira_issues( "indexed_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), "connector_id": connector_id, } - safe_set_chunks(document, chunks) + await safe_set_chunks(session, document, chunks) document.updated_at = get_current_timestamp() document.status = DocumentStatus.ready() diff --git a/surfsense_backend/app/tasks/connector_indexers/linear_indexer.py b/surfsense_backend/app/tasks/connector_indexers/linear_indexer.py index 1a2254c5b..bacafccc7 100644 --- a/surfsense_backend/app/tasks/connector_indexers/linear_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/linear_indexer.py @@ -431,7 +431,7 @@ async def index_linear_issues( "indexed_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), "connector_id": connector_id, } - safe_set_chunks(document, chunks) + await safe_set_chunks(session, document, chunks) document.updated_at = get_current_timestamp() document.status = DocumentStatus.ready() diff --git a/surfsense_backend/app/tasks/connector_indexers/luma_indexer.py b/surfsense_backend/app/tasks/connector_indexers/luma_indexer.py index 2d86b09c1..83cf54f4e 100644 --- a/surfsense_backend/app/tasks/connector_indexers/luma_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/luma_indexer.py @@ -488,7 +488,7 @@ async def index_luma_events( "indexed_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), "connector_id": connector_id, } - safe_set_chunks(document, chunks) + await safe_set_chunks(session, document, chunks) document.updated_at = get_current_timestamp() document.status = DocumentStatus.ready() diff --git a/surfsense_backend/app/tasks/connector_indexers/notion_indexer.py b/surfsense_backend/app/tasks/connector_indexers/notion_indexer.py index b0c49dea5..85daff94c 100644 --- a/surfsense_backend/app/tasks/connector_indexers/notion_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/notion_indexer.py @@ -479,7 +479,7 @@ async def index_notion_pages( "indexed_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), "connector_id": connector_id, } - safe_set_chunks(document, chunks) + await safe_set_chunks(session, document, chunks) document.updated_at = get_current_timestamp() document.status = DocumentStatus.ready() diff --git a/surfsense_backend/app/tasks/connector_indexers/obsidian_indexer.py b/surfsense_backend/app/tasks/connector_indexers/obsidian_indexer.py index c0eef84d5..d53baa3b0 100644 --- a/surfsense_backend/app/tasks/connector_indexers/obsidian_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/obsidian_indexer.py @@ -571,7 +571,7 @@ async def index_obsidian_vault( document.content_hash = content_hash document.embedding = embedding document.document_metadata = document_metadata - safe_set_chunks(document, chunks) + await safe_set_chunks(session, document, chunks) document.updated_at = get_current_timestamp() document.status = DocumentStatus.ready() diff --git a/surfsense_backend/app/tasks/connector_indexers/slack_indexer.py b/surfsense_backend/app/tasks/connector_indexers/slack_indexer.py index f83b171bc..1f2693844 100644 --- a/surfsense_backend/app/tasks/connector_indexers/slack_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/slack_indexer.py @@ -564,7 +564,7 @@ async def index_slack_messages( "indexed_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), "connector_id": connector_id, } - safe_set_chunks(document, chunks) + await safe_set_chunks(session, document, chunks) document.updated_at = get_current_timestamp() document.status = DocumentStatus.ready() diff --git a/surfsense_backend/app/tasks/connector_indexers/teams_indexer.py b/surfsense_backend/app/tasks/connector_indexers/teams_indexer.py index ad34e8696..d04a98177 100644 --- a/surfsense_backend/app/tasks/connector_indexers/teams_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/teams_indexer.py @@ -603,7 +603,7 @@ async def index_teams_messages( "indexed_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), "connector_id": connector_id, } - safe_set_chunks(document, chunks) + await safe_set_chunks(session, document, chunks) document.updated_at = get_current_timestamp() document.status = DocumentStatus.ready() diff --git a/surfsense_backend/app/tasks/connector_indexers/webcrawler_indexer.py b/surfsense_backend/app/tasks/connector_indexers/webcrawler_indexer.py index 94361cc27..4d2644420 100644 --- a/surfsense_backend/app/tasks/connector_indexers/webcrawler_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/webcrawler_indexer.py @@ -410,7 +410,7 @@ async def index_crawled_urls( "indexed_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), "connector_id": connector_id, } - safe_set_chunks(document, chunks) + await safe_set_chunks(session, document, chunks) document.status = DocumentStatus.ready() # READY status document.updated_at = get_current_timestamp() diff --git a/surfsense_backend/app/tasks/document_processors/base.py b/surfsense_backend/app/tasks/document_processors/base.py index 2edc48e91..580126d48 100644 --- a/surfsense_backend/app/tasks/document_processors/base.py +++ b/surfsense_backend/app/tasks/document_processors/base.py @@ -14,45 +14,37 @@ from app.db import Document md = MarkdownifyTransformer() -def safe_set_chunks(document: Document, chunks: list) -> None: +async def safe_set_chunks( + session: "AsyncSession", document: Document, chunks: list +) -> None: """ - Safely assign chunks to a document without triggering lazy loading. + Delete old chunks and assign new ones to a document. - ALWAYS use this instead of `document.chunks = chunks` to avoid - SQLAlchemy async errors (MissingGreenlet / greenlet_spawn). - - Why this is needed: - - Direct assignment `document.chunks = chunks` triggers SQLAlchemy to - load the OLD chunks first (for comparison/orphan detection) - - This lazy loading fails in async context with asyncpg driver - - set_committed_value bypasses this by setting the value directly - - This function is safe regardless of how the document was loaded - (with or without selectinload). + This replaces direct ``document.chunks = chunks`` which triggers lazy + loading (and MissingGreenlet errors in async contexts). It also + explicitly deletes pre-existing chunks so they don't accumulate across + repeated re-indexes — ``set_committed_value`` bypasses SQLAlchemy's + delete-orphan cascade. Args: - document: The Document object to update - chunks: List of Chunk objects to assign - - Example: - # Instead of: document.chunks = chunks (DANGEROUS!) - safe_set_chunks(document, chunks) # Always safe + session: The current async database session. + document: The Document object to update. + chunks: List of Chunk objects to assign. """ - from sqlalchemy.orm import object_session + from sqlalchemy import delete from sqlalchemy.orm.attributes import set_committed_value - # Keep relationship assignment lazy-load-safe. - set_committed_value(document, "chunks", chunks) + from app.db import Chunk - # Ensure chunk rows are actually persisted. - # set_committed_value bypasses normal unit-of-work tracking, so we need to - # explicitly attach chunk objects to the current session. - session = object_session(document) - if session is not None: - if document.id is not None: - for chunk in chunks: - chunk.document_id = document.id - session.add_all(chunks) + if document.id is not None: + await session.execute( + delete(Chunk).where(Chunk.document_id == document.id) + ) + for chunk in chunks: + chunk.document_id = document.id + + set_committed_value(document, "chunks", chunks) + session.add_all(chunks) def get_current_timestamp() -> datetime: diff --git a/surfsense_backend/app/tasks/document_processors/circleback_processor.py b/surfsense_backend/app/tasks/document_processors/circleback_processor.py index a86b64499..a6b9568b9 100644 --- a/surfsense_backend/app/tasks/document_processors/circleback_processor.py +++ b/surfsense_backend/app/tasks/document_processors/circleback_processor.py @@ -227,7 +227,7 @@ async def add_circleback_meeting_document( if summary_embedding is not None: document.embedding = summary_embedding document.document_metadata = document_metadata - safe_set_chunks(document, chunks) + await safe_set_chunks(session, document, chunks) document.source_markdown = markdown_content document.content_needs_reindexing = False document.updated_at = get_current_timestamp() diff --git a/surfsense_backend/app/tasks/document_processors/extension_processor.py b/surfsense_backend/app/tasks/document_processors/extension_processor.py index a6e482e15..7320ec9fa 100644 --- a/surfsense_backend/app/tasks/document_processors/extension_processor.py +++ b/surfsense_backend/app/tasks/document_processors/extension_processor.py @@ -21,6 +21,7 @@ from app.utils.document_converters import ( from .base import ( check_document_by_unique_identifier, get_current_timestamp, + safe_set_chunks, ) @@ -154,7 +155,7 @@ async def add_extension_received_document( existing_document.content_hash = content_hash existing_document.embedding = summary_embedding existing_document.document_metadata = content.metadata.model_dump() - existing_document.chunks = chunks + await safe_set_chunks(session, existing_document, chunks) existing_document.source_markdown = combined_document_string existing_document.updated_at = get_current_timestamp() diff --git a/surfsense_backend/app/tasks/document_processors/file_processors.py b/surfsense_backend/app/tasks/document_processors/file_processors.py index 5e97951bd..647435213 100644 --- a/surfsense_backend/app/tasks/document_processors/file_processors.py +++ b/surfsense_backend/app/tasks/document_processors/file_processors.py @@ -35,6 +35,7 @@ from .base import ( check_document_by_unique_identifier, check_duplicate_document, get_current_timestamp, + safe_set_chunks, ) from .markdown_processor import add_received_markdown_file_document @@ -488,7 +489,7 @@ async def add_received_file_document_using_unstructured( "FILE_NAME": file_name, "ETL_SERVICE": "UNSTRUCTURED", } - existing_document.chunks = chunks + await safe_set_chunks(session, existing_document, chunks) existing_document.source_markdown = file_in_markdown existing_document.content_needs_reindexing = False existing_document.updated_at = get_current_timestamp() @@ -622,7 +623,7 @@ async def add_received_file_document_using_llamacloud( "FILE_NAME": file_name, "ETL_SERVICE": "LLAMACLOUD", } - existing_document.chunks = chunks + await safe_set_chunks(session, existing_document, chunks) existing_document.source_markdown = file_in_markdown existing_document.content_needs_reindexing = False existing_document.updated_at = get_current_timestamp() @@ -777,7 +778,7 @@ async def add_received_file_document_using_docling( "FILE_NAME": file_name, "ETL_SERVICE": "DOCLING", } - existing_document.chunks = chunks + await safe_set_chunks(session, existing_document, chunks) existing_document.source_markdown = file_in_markdown existing_document.content_needs_reindexing = False existing_document.updated_at = get_current_timestamp() diff --git a/surfsense_backend/app/tasks/document_processors/markdown_processor.py b/surfsense_backend/app/tasks/document_processors/markdown_processor.py index a8d20c062..d598bf9dd 100644 --- a/surfsense_backend/app/tasks/document_processors/markdown_processor.py +++ b/surfsense_backend/app/tasks/document_processors/markdown_processor.py @@ -21,6 +21,7 @@ from .base import ( check_document_by_unique_identifier, check_duplicate_document, get_current_timestamp, + safe_set_chunks, ) @@ -258,7 +259,7 @@ async def add_received_markdown_file_document( existing_document.document_metadata = { "FILE_NAME": file_name, } - existing_document.chunks = chunks + await safe_set_chunks(session, existing_document, chunks) existing_document.source_markdown = file_in_markdown existing_document.updated_at = get_current_timestamp() existing_document.status = DocumentStatus.ready() # Mark as ready diff --git a/surfsense_backend/app/tasks/document_processors/youtube_processor.py b/surfsense_backend/app/tasks/document_processors/youtube_processor.py index 13b969fb6..0ed2e57d2 100644 --- a/surfsense_backend/app/tasks/document_processors/youtube_processor.py +++ b/surfsense_backend/app/tasks/document_processors/youtube_processor.py @@ -419,7 +419,7 @@ async def add_youtube_video_document( "author": video_data.get("author_name", "Unknown"), "thumbnail": video_data.get("thumbnail_url", ""), } - safe_set_chunks(document, chunks) + await safe_set_chunks(session, document, chunks) document.source_markdown = combined_document_string document.status = DocumentStatus.ready() # READY status - fully processed document.updated_at = get_current_timestamp() diff --git a/surfsense_backend/app/tasks/surfsense_docs_indexer.py b/surfsense_backend/app/tasks/surfsense_docs_indexer.py index ca4a83de3..7d449c0ab 100644 --- a/surfsense_backend/app/tasks/surfsense_docs_indexer.py +++ b/surfsense_backend/app/tasks/surfsense_docs_indexer.py @@ -13,12 +13,32 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import selectinload +from sqlalchemy import delete as sa_delete +from sqlalchemy.orm.attributes import set_committed_value + from app.config import config from app.db import SurfsenseDocsChunk, SurfsenseDocsDocument, async_session_maker from app.utils.document_converters import embed_text logger = logging.getLogger(__name__) + +async def _safe_set_docs_chunks( + session: AsyncSession, document: SurfsenseDocsDocument, chunks: list +) -> None: + """safe_set_chunks variant for the SurfsenseDocsDocument/Chunk models.""" + if document.id is not None: + await session.execute( + sa_delete(SurfsenseDocsChunk).where( + SurfsenseDocsChunk.document_id == document.id + ) + ) + for chunk in chunks: + chunk.document_id = document.id + + set_committed_value(document, "chunks", chunks) + session.add_all(chunks) + # Path to docs relative to project root DOCS_DIR = ( Path(__file__).resolve().parent.parent.parent.parent @@ -156,7 +176,7 @@ async def index_surfsense_docs(session: AsyncSession) -> tuple[int, int, int, in existing_doc.content = content existing_doc.content_hash = content_hash existing_doc.embedding = embed_text(content) - existing_doc.chunks = chunks + await _safe_set_docs_chunks(session, existing_doc, chunks) existing_doc.updated_at = datetime.now(UTC) updated += 1 From 5189224f28164a3e37c91a1eaeb75f010ad01f5b Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Sun, 15 Mar 2026 00:44:49 -0700 Subject: [PATCH 4/7] Revert "docs: add MiniMax (M2.5) to Chinese LLM setup guide" This reverts commit df286305494bfd47c5e0c82b9c13000f8be179e5. --- docs/chinese-llm-setup.md | 57 +++------------------------------------ 1 file changed, 4 insertions(+), 53 deletions(-) diff --git a/docs/chinese-llm-setup.md b/docs/chinese-llm-setup.md index eb479bcfa..2a184608f 100644 --- a/docs/chinese-llm-setup.md +++ b/docs/chinese-llm-setup.md @@ -14,7 +14,6 @@ SurfSense 现已支持以下国产 LLM： - ✅ **阿里通义千问 (Alibaba Qwen)** - 阿里云通义千问大模型 - ✅ **月之暗面 Kimi (Moonshot)** - 月之暗面 Kimi 大模型 - ✅ **智谱 AI GLM (Zhipu)** - 智谱 AI GLM 系列模型 -- ✅ **MiniMax** - MiniMax 大模型 (M2.5 系列，204K 上下文) --- @@ -198,52 +197,6 @@ API Base URL: https://open.bigmodel.cn/api/paas/v4 --- -## 5️⃣ MiniMax 配置 | MiniMax Configuration - -### 获取 API Key - -1. 访问 [MiniMax 开放平台](https://platform.minimaxi.com/) -2. 注册并登录账号 -3. 进入 **API Keys** 页面 -4. 创建新的 API Key -5. 复制 API Key - -### 在 SurfSense 中配置 - -| 字段 | 值 | 说明 | -|------|-----|------| -| **Configuration Name** | `MiniMax M2.5` | 配置名称（自定义） | -| **Provider** | `MINIMAX` | 选择 MiniMax | -| **Model Name** | `MiniMax-M2.5` | 推荐模型
其他选项: `MiniMax-M2.5-highspeed` | -| **API Key** | `eyJ...` | 你的 MiniMax API Key | -| **API Base URL** | `https://api.minimax.io/v1` | MiniMax API 地址 | -| **Parameters** | `{"temperature": 1.0}` | MiniMax 要求 temperature 在 (0.0, 1.0] 范围内 | - -### 示例配置 - -``` -Configuration Name: MiniMax M2.5 -Provider: MINIMAX -Model Name: MiniMax-M2.5 -API Key: eyJxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx -API Base URL: https://api.minimax.io/v1 -``` - -### 可用模型 - -- **MiniMax-M2.5**: 旗舰模型，204K 上下文窗口（推荐） -- **MiniMax-M2.5-highspeed**: 高速版本，204K 上下文窗口，更快响应 - -### 注意事项 - -- ⚠️ MiniMax API 要求 `temperature` 参数在 `(0.0, 1.0]` 范围内，不能设为 `0` -- 💡 如果使用默认参数时报错，请在 Parameters 中设置 `{"temperature": 1.0}` - -### 定价 -- 请访问 [MiniMax 开放平台定价](https://platform.minimaxi.com/document/Price) 查看最新价格 - ---- - ## ⚙️ 高级配置 | Advanced Configuration ### 自定义参数 | Custom Parameters @@ -315,13 +268,13 @@ docker compose logs backend | grep -i "error" |---------|---------|------| | **文档摘要** | Qwen-Plus, GLM-4 | 平衡性能和成本 | | **代码分析** | DeepSeek-Coder | 代码专用 | -| **长文本处理** | Kimi 128K, MiniMax-M2.5 | 超长上下文（204K） | -| **快速响应** | Qwen-Turbo, GLM-4-Flash, MiniMax-M2.5-highspeed | 速度优先 | +| **长文本处理** | Kimi 128K | 超长上下文 | +| **快速响应** | Qwen-Turbo, GLM-4-Flash | 速度优先 | ### 2. 成本优化 -- 🎯 **Long Context LLM**: 使用 Qwen-Plus、GLM-4 或 MiniMax-M2.5（处理文档摘要，支持 204K 上下文） -- ⚡ **Fast LLM**: 使用 Qwen-Turbo、GLM-4-Flash 或 MiniMax-M2.5-highspeed（快速对话） +- 🎯 **Long Context LLM**: 使用 Qwen-Plus 或 GLM-4（处理文档摘要） +- ⚡ **Fast LLM**: 使用 Qwen-Turbo 或 GLM-4-Flash（快速对话） - 🧠 **Strategic LLM**: 使用 Qwen-Max 或 DeepSeek-Chat（复杂推理） ### 3. API Key 安全 @@ -341,7 +294,6 @@ docker compose logs backend | grep -i "error" - [阿里云百炼文档](https://help.aliyun.com/zh/model-studio/) - [Moonshot AI 文档](https://platform.moonshot.cn/docs) - [智谱 AI 文档](https://open.bigmodel.cn/dev/api) -- [MiniMax 文档](https://platform.minimaxi.com/document/introduction) ### SurfSense 文档 @@ -363,7 +315,6 @@ docker compose logs backend | grep -i "error" ## 🔄 更新日志 | Changelog -- **2026-03-13**: 添加 MiniMax (M2.5) 配置指南 - **2025-01-12**: 初始版本，添加 DeepSeek、Qwen、Kimi、GLM 支持 --- From 0269900c60de86b18dd6872651bf9ca771bc6835 Mon Sep 17 00:00:00 2001 From: Tim Ren <137012659+xr843@users.noreply.github.com> Date: Sun, 15 Mar 2026 16:21:19 +0800 Subject: [PATCH 5/7] fix: use asyncio.to_thread for embedding calls in search endpoints Wrap synchronous embedding_model.embed() calls with asyncio.to_thread() in both vector_search and hybrid_search methods. This prevents blocking the asyncio event loop during embedding computation, improving server responsiveness under concurrent load. Fixes #794 Signed-off-by: Tim Ren <137012659+xr843@users.noreply.github.com> --- surfsense_backend/app/retriever/chunks_hybrid_search.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/surfsense_backend/app/retriever/chunks_hybrid_search.py b/surfsense_backend/app/retriever/chunks_hybrid_search.py index 5ab2964ca..2a0bc1a4a 100644 --- a/surfsense_backend/app/retriever/chunks_hybrid_search.py +++ b/surfsense_backend/app/retriever/chunks_hybrid_search.py @@ -1,3 +1,4 @@ +import asyncio import time from datetime import datetime @@ -49,7 +50,7 @@ class ChucksHybridSearchRetriever: # Get embedding for the query embedding_model = config.embedding_model_instance t_embed = time.perf_counter() - query_embedding = embedding_model.embed(query_text) + query_embedding = await asyncio.to_thread(embedding_model.embed, query_text) perf.debug( "[chunk_search] vector_search embedding in %.3fs", time.perf_counter() - t_embed, @@ -195,7 +196,7 @@ class ChucksHybridSearchRetriever: if query_embedding is None: embedding_model = config.embedding_model_instance t_embed = time.perf_counter() - query_embedding = embedding_model.embed(query_text) + query_embedding = await asyncio.to_thread(embedding_model.embed, query_text) perf.debug( "[chunk_search] hybrid_search embedding in %.3fs", time.perf_counter() - t_embed, @@ -427,4 +428,4 @@ class ChucksHybridSearchRetriever: search_space_id, document_type, ) - return final_docs + return final_docs \ No newline at end of file From 1e6424ef423a0a29f7f67cb2de532cfe3769d674 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Sun, 15 Mar 2026 16:27:33 +0530 Subject: [PATCH 6/7] feat: enhance ComposerAction with dropdown menu for managing tools and file uploads --- .../components/assistant-ui/thread.tsx | 242 ++++++++++++------ 1 file changed, 158 insertions(+), 84 deletions(-) diff --git a/surfsense_web/components/assistant-ui/thread.tsx b/surfsense_web/components/assistant-ui/thread.tsx index d016efb24..243b3f7cb 100644 --- a/surfsense_web/components/assistant-ui/thread.tsx +++ b/surfsense_web/components/assistant-ui/thread.tsx @@ -19,9 +19,11 @@ import { ChevronRightIcon, CopyIcon, DownloadIcon, + Plus, RefreshCwIcon, SquareIcon, Unplug, + Upload, Wrench, X, } from "lucide-react"; @@ -71,9 +73,16 @@ import { type DocumentMentionPickerRef, } from "@/components/new-chat/document-mention-picker"; import type { ThinkingStep } from "@/components/tool-ui/deepagent-thinking"; +import { useDocumentUploadDialog } from "@/components/assistant-ui/document-upload-popup"; import { Avatar, AvatarFallback, AvatarGroup } from "@/components/ui/avatar"; import { Button } from "@/components/ui/button"; -import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; +import { Popover, PopoverAnchor, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; import { Switch } from "@/components/ui/switch"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; @@ -278,21 +287,14 @@ const ConnectToolsBanner: FC = () => { ))} - { - if (e.key === "Enter" || e.key === " ") { - e.preventDefault(); - handleDismiss(e as unknown as React.MouseEvent); - } - }} className="shrink-0 ml-0.5 p-0.5 text-muted-foreground/40 hover:text-foreground transition-colors" aria-label="Dismiss" > - + ); @@ -564,6 +566,7 @@ const ComposerAction: FC = ({ isBlockedByOtherUser = false const setConnectorDialogOpen = useSetAtom(connectorDialogOpenAtom); const [toolsPopoverOpen, setToolsPopoverOpen] = useState(false); const isDesktop = useMediaQuery("(min-width: 640px)"); + const { openDialog: openUploadDialog } = useDocumentUploadDialog(); const [toolsScrollPos, setToolsScrollPos] = useState<"top" | "middle" | "bottom">("top"); const handleToolsScroll = useCallback((e: React.UIEvent) => { const el = e.currentTarget; @@ -607,79 +610,82 @@ const ComposerAction: FC = ({ isBlockedByOtherUser = false return (
- - - + + + + + + + + + setToolsPopoverOpen(true)}> + + Manage Tools + + openUploadDialog()}> + + Upload Files + + + + e.preventDefault()} > - - - - e.preventDefault()} - > -
- Agent Tools - - {enabledCount}/{agentTools?.length ?? 0} enabled - -
-
- {agentTools?.map((tool) => { - const isDisabled = disabledTools.includes(tool.name); - const row = ( - - ); - if (!isDesktop) { - return
{row}
; - } - return ( - - {row} - - {tool.description} - - - ); - })} - {!agentTools?.length && ( -
- Loading tools... -
- )} -
- - - {!isDesktop && ( - + Agent Tools + + {enabledCount}/{agentTools?.length ?? 0} enabled + +
+
+ {agentTools?.map((tool) => { + const isDisabled = disabledTools.includes(tool.name); + return ( +
+ + {formatToolName(tool.name)} + + toggleTool(tool.name)} + className="shrink-0 scale-[0.6]" + /> +
+ ); + })} + {!agentTools?.length && ( +
+ Loading tools... +
+ )} +
+ + + + + ) : ( + + + + + + + e.preventDefault()} + > +
+ Agent Tools + + {enabledCount}/{agentTools?.length ?? 0} enabled + +
+
+ {agentTools?.map((tool) => { + const isDisabled = disabledTools.includes(tool.name); + const row = ( +
+ + {formatToolName(tool.name)} + + toggleTool(tool.name)} + className="shrink-0 scale-[0.6] sm:scale-75" + /> +
+ ); + return ( + + {row} + + {tool.description} + + + ); + })} + {!agentTools?.length && ( +
+ Loading tools... +
+ )} +
+ + )} {sidebarDocs.length > 0 && ( - - + + + setToolsPopoverOpen(true)}> - + Manage Tools openUploadDialog()}> @@ -638,62 +636,51 @@ const ComposerAction: FC = ({ isBlockedByOtherUser = false - e.preventDefault()} - > -
- Agent Tools - - {enabledCount}/{agentTools?.length ?? 0} enabled - -
-
- {agentTools?.map((tool) => { - const isDisabled = disabledTools.includes(tool.name); - return ( -
- - {formatToolName(tool.name)} - - toggleTool(tool.name)} - className="shrink-0 scale-[0.6]" - /> + + + +
+ Agent Tools + + {enabledCount}/{agentTools?.length ?? 0} enabled + +
+
+ {agentTools?.map((tool) => { + const isDisabled = disabledTools.includes(tool.name); + return ( +
+ + {formatToolName(tool.name)} + + toggleTool(tool.name)} + className="shrink-0" + /> +
+ ); + })} + {!agentTools?.length && ( +
+ Loading tools...
- ); - })} - {!agentTools?.length && ( -
- Loading tools... -
- )} -
- - - + )} +
+ + + ) : ( @@ -707,7 +694,7 @@ const ComposerAction: FC = ({ isBlockedByOtherUser = false aria-label="Manage tools" data-joyride="connector-icon" > - +