diff --git a/README.md b/README.md index cf0424fe2..dbd969f25 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ # SurfSense -While tools like NotebookLM and Perplexity are impressive and highly effective for conducting research on any topic/query, SurfSense elevates this capability by integrating with your personal knowledge base. It is a highly customizable AI research agent, connected to external sources such as Search Engines (Tavily, LinkUp), Slack, Linear, Jira, ClickUp, Confluence, Gmail, Notion, YouTube, GitHub, Discord, Airtable, Google Calendar, Luma and more to come. +While tools like NotebookLM and Perplexity are impressive and highly effective for conducting research on any topic/query, SurfSense elevates this capability by integrating with your personal knowledge base. It is a highly customizable AI research agent, connected to external sources such as Search Engines (SearxNG, Tavily, LinkUp), Slack, Linear, Jira, ClickUp, Confluence, Gmail, Notion, YouTube, GitHub, Discord, Airtable, Google Calendar, Luma and more to come.
MODSetter%2FSurfSense | Trendshift @@ -62,6 +62,7 @@ Open source and easy to deploy locally. ### ℹ️ **External Sources** - Search Engines (Tavily, LinkUp) +- SearxNG (self-hosted instances) - Slack - Linear - Jira @@ -303,4 +304,3 @@ For detailed contribution guidelines, please see our [CONTRIBUTING.md](CONTRIBUT --- --- - diff --git a/docs/chinese-llm-setup.md b/docs/chinese-llm-setup.md new file mode 100644 index 000000000..2a184608f --- /dev/null +++ b/docs/chinese-llm-setup.md @@ -0,0 +1,323 @@ +# 国产 LLM 配置指南 | Chinese LLM Setup Guide + +本指南将帮助你在 SurfSense 中配置和使用国产大语言模型。 + +This guide helps you configure and use Chinese LLM providers in SurfSense. + +--- + +## 📋 支持的提供商 | Supported Providers + +SurfSense 现已支持以下国产 LLM： + +- ✅ **DeepSeek** - 国产高性能 AI 模型 +- ✅ **阿里通义千问 (Alibaba Qwen)** - 阿里云通义千问大模型 +- ✅ **月之暗面 Kimi (Moonshot)** - 月之暗面 Kimi 大模型 +- ✅ **智谱 AI GLM (Zhipu)** - 智谱 AI GLM 系列模型 + +--- + +## 🚀 快速开始 | Quick Start + +### 通用配置步骤 | General Configuration Steps + +1. 登录 SurfSense Dashboard +2. 进入 **Settings** → **API Keys** (或 **LLM Configurations**) +3. 点击 **Add New Configuration** +4. 从 **Provider** 下拉菜单中选择你的国产 LLM 提供商 +5. 填写必填字段（见下方各提供商详细配置） +6. 点击 **Save** + +--- + +## 1️⃣ DeepSeek 配置 | DeepSeek Configuration + +### 获取 API Key + +1. 访问 [DeepSeek 开放平台](https://platform.deepseek.com/) +2. 注册并登录账号 +3. 进入 **API Keys** 页面 +4. 点击 **Create New API Key** +5. 复制生成的 API Key (格式: `sk-xxx`) + +### 在 SurfSense 中配置 + +| 字段 | 值 | 说明 | +|------|-----|------| +| **Configuration Name** | `DeepSeek Chat` | 配置名称（自定义） | +| **Provider** | `DEEPSEEK` | 选择 DeepSeek | +| **Model Name** | `deepseek-chat` | 推荐模型
其他选项: `deepseek-coder` | +| **API Key** | `sk-xxx...` | 你的 DeepSeek API Key | +| **API Base URL** | `https://api.deepseek.com` | DeepSeek API 地址 | +| **Parameters** | _(留空)_ | 使用默认参数 | + +### 示例配置 + +``` +Configuration Name: DeepSeek Chat +Provider: DEEPSEEK +Model Name: deepseek-chat +API Key: sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +API Base URL: https://api.deepseek.com +``` + +### 可用模型 + +- **deepseek-chat**: 通用对话模型（推荐） +- **deepseek-coder**: 代码专用模型 + +### 定价 +- 请访问 [DeepSeek 定价页面](https://platform.deepseek.com/pricing) 查看最新价格 + +--- + +## 2️⃣ 阿里通义千问 (Alibaba Qwen) 配置 + +### 获取 API Key + +1. 访问 [阿里云百炼平台](https://dashscope.aliyun.com/) +2. 登录阿里云账号 +3. 开通 DashScope 服务 +4. 进入 **API-KEY 管理** +5. 创建并复制 API Key + +### 在 SurfSense 中配置 + +| 字段 | 值 | 说明 | +|------|-----|------| +| **Configuration Name** | `通义千问 Max` | 配置名称（自定义） | +| **Provider** | `ALIBABA_QWEN` | 选择阿里通义千问 | +| **Model Name** | `qwen-max` | 推荐模型
其他选项: `qwen-plus`, `qwen-turbo` | +| **API Key** | `sk-xxx...` | 你的 DashScope API Key | +| **API Base URL** | `https://dashscope.aliyuncs.com/compatible-mode/v1` | 阿里云 API 地址 | +| **Parameters** | _(留空)_ | 使用默认参数 | + +### 示例配置 + +``` +Configuration Name: 通义千问 Max +Provider: ALIBABA_QWEN +Model Name: qwen-max +API Key: sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +API Base URL: https://dashscope.aliyuncs.com/compatible-mode/v1 +``` + +### 可用模型 + +- **qwen-max**: 最强性能，适合复杂任务 +- **qwen-plus**: 性价比高，适合日常使用（推荐） +- **qwen-turbo**: 速度快，适合简单任务 + +### 定价 +- 请访问 [阿里云百炼定价](https://help.aliyun.com/zh/model-studio/getting-started/billing) 查看最新价格 + +--- + +## 3️⃣ 月之暗面 Kimi (Moonshot) 配置 + +### 获取 API Key + +1. 访问 [Moonshot AI 开放平台](https://platform.moonshot.cn/) +2. 注册并登录账号 +3. 进入 **API Key 管理** +4. 创建新的 API Key +5. 复制 API Key + +### 在 SurfSense 中配置 + +| 字段 | 值 | 说明 | +|------|-----|------| +| **Configuration Name** | `Kimi` | 配置名称（自定义） | +| **Provider** | `MOONSHOT` | 选择月之暗面 Kimi | +| **Model Name** | `moonshot-v1-32k` | 推荐模型
其他选项: `moonshot-v1-8k`, `moonshot-v1-128k` | +| **API Key** | `sk-xxx...` | 你的 Moonshot API Key | +| **API Base URL** | `https://api.moonshot.cn/v1` | Moonshot API 地址 | +| **Parameters** | _(留空)_ | 使用默认参数 | + +### 示例配置 + +``` +Configuration Name: Kimi 32K +Provider: MOONSHOT +Model Name: moonshot-v1-32k +API Key: sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +API Base URL: https://api.moonshot.cn/v1 +``` + +### 可用模型 + +- **moonshot-v1-8k**: 8K 上下文（基础版） +- **moonshot-v1-32k**: 32K 上下文（推荐） +- **moonshot-v1-128k**: 128K 上下文（长文本专用） + +### 定价 +- 请访问 [Moonshot AI 定价](https://platform.moonshot.cn/pricing) 查看最新价格 + +--- + +## 4️⃣ 智谱 AI GLM (Zhipu) 配置 + +### 获取 API Key + +1. 访问 [智谱 AI 开放平台](https://open.bigmodel.cn/) +2. 注册并登录账号 +3. 进入 **API 管理** +4. 创建新的 API Key +5. 复制 API Key + +### 在 SurfSense 中配置 + +| 字段 | 值 | 说明 | +|------|-----|------| +| **Configuration Name** | `GLM-4` | 配置名称（自定义） | +| **Provider** | `ZHIPU` | 选择智谱 AI | +| **Model Name** | `glm-4` | 推荐模型
其他选项: `glm-4-flash`, `glm-3-turbo` | +| **API Key** | `xxx.yyy...` | 你的智谱 API Key | +| **API Base URL** | `https://open.bigmodel.cn/api/paas/v4` | 智谱 API 地址 | +| **Parameters** | _(留空)_ | 使用默认参数 | + +### 示例配置 + +``` +Configuration Name: GLM-4 +Provider: ZHIPU +Model Name: glm-4 +API Key: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.xxxxxxxxxxxxxxxx +API Base URL: https://open.bigmodel.cn/api/paas/v4 +``` + +### 可用模型 + +- **glm-4**: GLM-4 旗舰模型（推荐） +- **glm-4-flash**: 快速推理版本 +- **glm-3-turbo**: 高性价比版本 + +### 定价 +- 请访问 [智谱 AI 定价](https://open.bigmodel.cn/pricing) 查看最新价格 + +--- + +## ⚙️ 高级配置 | Advanced Configuration + +### 自定义参数 | Custom Parameters + +你可以在 **Parameters** 字段中添加自定义参数（JSON 格式）： + +```json +{ + "temperature": 0.7, + "max_tokens": 2000, + "top_p": 0.9 +} +``` + +### 常用参数说明 + +| 参数 | 说明 | 默认值 | 范围 | +|------|------|--------|------| +| `temperature` | 控制输出随机性，越高越随机 | 0.7 | 0.0 - 1.0 | +| `max_tokens` | 最大输出 Token 数 | 模型默认 | 1 - 模型上限 | +| `top_p` | 核采样参数 | 1.0 | 0.0 - 1.0 | + +--- + +## 🔧 故障排除 | Troubleshooting + +### 常见问题 + +#### 1. **错误: "Invalid API Key"** +- ✅ 检查 API Key 是否正确复制（无多余空格） +- ✅ 确认 API Key 是否已激活 +- ✅ 检查账户余额是否充足 + +#### 2. **错误: "Connection timeout"** +- ✅ 确认 API Base URL 是否正确 +- ✅ 检查网络连接 +- ✅ 确认防火墙是否允许访问 + +#### 3. **错误: "Model not found"** +- ✅ 确认模型名称是否拼写正确 +- ✅ 检查该模型是否已开通 +- ✅ 参照上方文档确认可用模型名称 + +#### 4. **文档处理卡住 (IN_PROGRESS)** +- ✅ 检查模型名称中是否有多余空格 +- ✅ 确认 API Key 有效且有额度 +- ✅ 查看后端日志: `docker compose logs backend` + +### 查看日志 + +```bash +# 查看后端日志 +docker compose logs backend --tail 100 + +# 实时查看日志 +docker compose logs -f backend + +# 搜索错误 +docker compose logs backend | grep -i "error" +``` + +--- + +## 💡 最佳实践 | Best Practices + +### 1. 模型选择建议 + +| 任务类型 | 推荐模型 | 说明 | +|---------|---------|------| +| **文档摘要** | Qwen-Plus, GLM-4 | 平衡性能和成本 | +| **代码分析** | DeepSeek-Coder | 代码专用 | +| **长文本处理** | Kimi 128K | 超长上下文 | +| **快速响应** | Qwen-Turbo, GLM-4-Flash | 速度优先 | + +### 2. 成本优化 + +- 🎯 **Long Context LLM**: 使用 Qwen-Plus 或 GLM-4（处理文档摘要） +- ⚡ **Fast LLM**: 使用 Qwen-Turbo 或 GLM-4-Flash（快速对话） +- 🧠 **Strategic LLM**: 使用 Qwen-Max 或 DeepSeek-Chat（复杂推理） + +### 3. API Key 安全 + +- ❌ 不要在公开代码中硬编码 API Key +- ✅ 定期轮换 API Key +- ✅ 为不同用途创建不同的 Key +- ✅ 设置合理的额度限制 + +--- + +## 📚 相关资源 | Resources + +### 官方文档 + +- [DeepSeek 文档](https://platform.deepseek.com/docs) +- [阿里云百炼文档](https://help.aliyun.com/zh/model-studio/) +- [Moonshot AI 文档](https://platform.moonshot.cn/docs) +- [智谱 AI 文档](https://open.bigmodel.cn/dev/api) + +### SurfSense 文档 + +- [安装指南](../README.md) +- [贡献指南](../CONTRIBUTING.md) +- [部署指南](../DEPLOYMENT_GUIDE.md) + +--- + +## 🆘 需要帮助？ | Need Help? + +如果遇到问题，可以通过以下方式获取帮助： + +- 💬 [GitHub Issues](https://github.com/MODSetter/SurfSense/issues) +- 💬 [Discord Community](https://discord.gg/ejRNvftDp9) +- 📧 Email: [项目维护者邮箱] + +--- + +## 🔄 更新日志 | Changelog + +- **2025-01-12**: 初始版本，添加 DeepSeek、Qwen、Kimi、GLM 支持 + +--- + +**祝你使用愉快！Happy coding with Chinese LLMs! 🚀** + diff --git a/surfsense_backend/alembic/env.py b/surfsense_backend/alembic/env.py index fd9740ee2..bd8c20356 100644 --- a/surfsense_backend/alembic/env.py +++ b/surfsense_backend/alembic/env.py @@ -20,6 +20,11 @@ from app.db import Base # Assuming your Base is defined in app.db # access to the values within the .ini file in use. config = context.config +# Override SQLAlchemy URL from environment variables when available +database_url = os.getenv("DATABASE_URL") +if database_url: + config.set_main_option("sqlalchemy.url", database_url) + # Interpret the config file for Python logging. # This line sets up loggers basically. if config.config_file_name is not None: diff --git a/surfsense_backend/alembic/versions/23_associate_connectors_with_search_spaces.py b/surfsense_backend/alembic/versions/23_associate_connectors_with_search_spaces.py new file mode 100644 index 000000000..a693b9ec5 --- /dev/null +++ b/surfsense_backend/alembic/versions/23_associate_connectors_with_search_spaces.py @@ -0,0 +1,158 @@ +"""Associate SearchSourceConnector with SearchSpace instead of User + +Revision ID: '23' +Revises: '22' + +""" + +from collections.abc import Sequence + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "23" +down_revision: str | None = "22" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """ + Add search_space_id to SearchSourceConnector and update unique constraint. + + Changes: + 1. Add search_space_id column (nullable initially) + 2. Populate search_space_id with user's first search space + 3. Make search_space_id NOT NULL + 4. Add foreign key constraint + 5. Drop old unique constraint (user_id, connector_type) + 6. Add new unique constraint (search_space_id, user_id, connector_type) + """ + + from sqlalchemy import inspect + + conn = op.get_bind() + inspector = inspect(conn) + + # Get existing columns + columns = [col["name"] for col in inspector.get_columns("search_source_connectors")] + + # Step 1: Add search_space_id column as nullable first (if it doesn't exist) + if "search_space_id" not in columns: + op.add_column( + "search_source_connectors", + sa.Column("search_space_id", sa.Integer(), nullable=True), + ) + + # Step 2: Populate search_space_id with each user's first search space + # This ensures existing connectors are assigned to a valid search space + op.execute( + """ + UPDATE search_source_connectors ssc + SET search_space_id = ( + SELECT id + FROM searchspaces ss + WHERE ss.user_id = ssc.user_id + ORDER BY ss.created_at ASC + LIMIT 1 + ) + WHERE search_space_id IS NULL + """ + ) + + # Step 3: Make search_space_id NOT NULL + op.alter_column( + "search_source_connectors", + "search_space_id", + nullable=False, + ) + + # Step 4: Add foreign key constraint (if it doesn't exist) + foreign_keys = [ + fk["name"] for fk in inspector.get_foreign_keys("search_source_connectors") + ] + if "fk_search_source_connectors_search_space_id" not in foreign_keys: + op.create_foreign_key( + "fk_search_source_connectors_search_space_id", + "search_source_connectors", + "searchspaces", + ["search_space_id"], + ["id"], + ondelete="CASCADE", + ) + + # Step 5: Drop the old unique constraint (user_id, connector_type) if it exists + unique_constraints = [ + uc["name"] + for uc in inspector.get_unique_constraints("search_source_connectors") + ] + if "uq_user_connector_type" in unique_constraints: + op.drop_constraint( + "uq_user_connector_type", + "search_source_connectors", + type_="unique", + ) + + # Step 6: Create new unique constraint (search_space_id, user_id, connector_type) if it doesn't exist + if "uq_searchspace_user_connector_type" not in unique_constraints: + op.create_unique_constraint( + "uq_searchspace_user_connector_type", + "search_source_connectors", + ["search_space_id", "user_id", "connector_type"], + ) + + +def downgrade() -> None: + """ + Revert SearchSourceConnector association back to User only. + + WARNING: This downgrade may result in data loss if multiple connectors + of the same type exist for a user across different search spaces. + """ + + from sqlalchemy import inspect + + conn = op.get_bind() + inspector = inspect(conn) + + # Get existing constraints and columns + unique_constraints = [ + uc["name"] + for uc in inspector.get_unique_constraints("search_source_connectors") + ] + foreign_keys = [ + fk["name"] for fk in inspector.get_foreign_keys("search_source_connectors") + ] + columns = [col["name"] for col in inspector.get_columns("search_source_connectors")] + + # Step 1: Drop the new unique constraint if it exists + if "uq_searchspace_user_connector_type" in unique_constraints: + op.drop_constraint( + "uq_searchspace_user_connector_type", + "search_source_connectors", + type_="unique", + ) + + # Step 2: Recreate the old unique constraint (user_id, connector_type) if it doesn't exist + # NOTE: This will fail if there are duplicate (user_id, connector_type) combinations + # Manual cleanup may be required before downgrading + if "uq_user_connector_type" not in unique_constraints: + op.create_unique_constraint( + "uq_user_connector_type", + "search_source_connectors", + ["user_id", "connector_type"], + ) + + # Step 3: Drop the foreign key constraint if it exists + if "fk_search_source_connectors_search_space_id" in foreign_keys: + op.drop_constraint( + "fk_search_source_connectors_search_space_id", + "search_source_connectors", + type_="foreignkey", + ) + + # Step 4: Drop the search_space_id column if it exists + if "search_space_id" in columns: + op.drop_column("search_source_connectors", "search_space_id") diff --git a/surfsense_backend/alembic/versions/24_fix_null_chat_types.py b/surfsense_backend/alembic/versions/24_fix_null_chat_types.py new file mode 100644 index 000000000..e0d371f1e --- /dev/null +++ b/surfsense_backend/alembic/versions/24_fix_null_chat_types.py @@ -0,0 +1,38 @@ +"""Fix NULL chat types by setting them to QNA + +Revision ID: 24 +Revises: 23 + +""" + +from collections.abc import Sequence + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "24" +down_revision: str | None = "23" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """ + Fix any chats with NULL type values by setting them to QNA. + This handles edge cases from previous migrations where type values were not properly migrated. + """ + # Update any NULL type values to QNA (the default chat type) + op.execute( + """ + UPDATE chats + SET type = 'QNA' + WHERE type IS NULL + """ + ) + + +def downgrade() -> None: + """ + No downgrade necessary - we can't restore NULL values as we don't know which ones were NULL. + """ + pass diff --git a/surfsense_backend/alembic/versions/25_migrate_llm_configs_to_search_spaces.py b/surfsense_backend/alembic/versions/25_migrate_llm_configs_to_search_spaces.py new file mode 100644 index 000000000..c9966599c --- /dev/null +++ b/surfsense_backend/alembic/versions/25_migrate_llm_configs_to_search_spaces.py @@ -0,0 +1,351 @@ +"""Migrate LLM configs to search spaces and add user preferences + +Revision ID: 25 +Revises: 24 + +Changes: +1. Migrate llm_configs from user association to search_space association +2. Create user_search_space_preferences table for per-user LLM preferences +3. Migrate existing user LLM preferences to user_search_space_preferences +4. Remove LLM preference columns from user table +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "25" +down_revision: str | None = "24" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """ + Upgrade schema to support collaborative search spaces with per-user preferences. + + Migration steps: + 1. Add search_space_id to llm_configs + 2. Migrate existing llm_configs to first search space of their user + 3. Replace user_id with search_space_id in llm_configs + 4. Create user_search_space_preferences table + 5. Migrate user LLM preferences to user_search_space_preferences + 6. Remove LLM preference columns from user table + """ + + from sqlalchemy import inspect + + conn = op.get_bind() + inspector = inspect(conn) + + # Get existing columns + llm_config_columns = [col["name"] for col in inspector.get_columns("llm_configs")] + user_columns = [col["name"] for col in inspector.get_columns("user")] + + # ===== STEP 1: Add search_space_id to llm_configs ===== + if "search_space_id" not in llm_config_columns: + op.add_column( + "llm_configs", + sa.Column("search_space_id", sa.Integer(), nullable=True), + ) + + # ===== STEP 2: Populate search_space_id with user's first search space ===== + # This ensures existing LLM configs are assigned to a valid search space + op.execute( + """ + UPDATE llm_configs lc + SET search_space_id = ( + SELECT id + FROM searchspaces ss + WHERE ss.user_id = lc.user_id + ORDER BY ss.created_at ASC + LIMIT 1 + ) + WHERE search_space_id IS NULL AND user_id IS NOT NULL + """ + ) + + # ===== STEP 3: Make search_space_id NOT NULL and add FK constraint ===== + op.alter_column( + "llm_configs", + "search_space_id", + nullable=False, + ) + + # Add foreign key constraint + foreign_keys = [fk["name"] for fk in inspector.get_foreign_keys("llm_configs")] + if "fk_llm_configs_search_space_id" not in foreign_keys: + op.create_foreign_key( + "fk_llm_configs_search_space_id", + "llm_configs", + "searchspaces", + ["search_space_id"], + ["id"], + ondelete="CASCADE", + ) + + # Drop old user_id foreign key if it exists + if "fk_llm_configs_user_id_user" in foreign_keys: + op.drop_constraint( + "fk_llm_configs_user_id_user", + "llm_configs", + type_="foreignkey", + ) + + # Remove user_id column + if "user_id" in llm_config_columns: + op.drop_column("llm_configs", "user_id") + + # ===== STEP 4: Create user_search_space_preferences table ===== + op.execute( + """ + DO $$ + BEGIN + IF NOT EXISTS ( + SELECT FROM information_schema.tables + WHERE table_name = 'user_search_space_preferences' + ) THEN + CREATE TABLE user_search_space_preferences ( + id SERIAL PRIMARY KEY, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), + user_id UUID NOT NULL REFERENCES "user"(id) ON DELETE CASCADE, + search_space_id INTEGER NOT NULL REFERENCES searchspaces(id) ON DELETE CASCADE, + long_context_llm_id INTEGER REFERENCES llm_configs(id) ON DELETE SET NULL, + fast_llm_id INTEGER REFERENCES llm_configs(id) ON DELETE SET NULL, + strategic_llm_id INTEGER REFERENCES llm_configs(id) ON DELETE SET NULL, + CONSTRAINT uq_user_searchspace UNIQUE (user_id, search_space_id) + ); + END IF; + END$$; + """ + ) + + # Create indexes + op.execute( + """ + DO $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_indexes + WHERE tablename = 'user_search_space_preferences' + AND indexname = 'ix_user_search_space_preferences_id' + ) THEN + CREATE INDEX ix_user_search_space_preferences_id + ON user_search_space_preferences(id); + END IF; + + IF NOT EXISTS ( + SELECT 1 FROM pg_indexes + WHERE tablename = 'user_search_space_preferences' + AND indexname = 'ix_user_search_space_preferences_created_at' + ) THEN + CREATE INDEX ix_user_search_space_preferences_created_at + ON user_search_space_preferences(created_at); + END IF; + END$$; + """ + ) + + # ===== STEP 5: Migrate user LLM preferences to user_search_space_preferences ===== + # For each user, create preferences for each of their search spaces + if all( + col in user_columns + for col in ["long_context_llm_id", "fast_llm_id", "strategic_llm_id"] + ): + op.execute( + """ + INSERT INTO user_search_space_preferences + (user_id, search_space_id, long_context_llm_id, fast_llm_id, strategic_llm_id, created_at) + SELECT + u.id as user_id, + ss.id as search_space_id, + u.long_context_llm_id, + u.fast_llm_id, + u.strategic_llm_id, + NOW() as created_at + FROM "user" u + CROSS JOIN searchspaces ss + WHERE ss.user_id = u.id + ON CONFLICT (user_id, search_space_id) DO NOTHING + """ + ) + + # ===== STEP 6: Remove LLM preference columns from user table ===== + # Get fresh list of foreign keys after previous operations + user_foreign_keys = [fk["name"] for fk in inspector.get_foreign_keys("user")] + + # Drop foreign key constraints if they exist + if "fk_user_long_context_llm_id_llm_configs" in user_foreign_keys: + op.drop_constraint( + "fk_user_long_context_llm_id_llm_configs", + "user", + type_="foreignkey", + ) + + if "fk_user_fast_llm_id_llm_configs" in user_foreign_keys: + op.drop_constraint( + "fk_user_fast_llm_id_llm_configs", + "user", + type_="foreignkey", + ) + + if "fk_user_strategic_llm_id_llm_configs" in user_foreign_keys: + op.drop_constraint( + "fk_user_strategic_llm_id_llm_configs", + "user", + type_="foreignkey", + ) + + # Drop columns from user table + if "long_context_llm_id" in user_columns: + op.drop_column("user", "long_context_llm_id") + + if "fast_llm_id" in user_columns: + op.drop_column("user", "fast_llm_id") + + if "strategic_llm_id" in user_columns: + op.drop_column("user", "strategic_llm_id") + + +def downgrade() -> None: + """ + Downgrade schema back to user-owned LLM configs. + + WARNING: This downgrade will result in data loss: + - LLM configs will be moved back to user ownership (first occurrence kept) + - Per-search-space user preferences will be consolidated to user level + - Additional LLM configs in search spaces beyond the first will be deleted + """ + + from sqlalchemy import inspect + + conn = op.get_bind() + inspector = inspect(conn) + + # Get existing columns and constraints + llm_config_columns = [col["name"] for col in inspector.get_columns("llm_configs")] + user_columns = [col["name"] for col in inspector.get_columns("user")] + + # ===== STEP 1: Add LLM preference columns back to user table ===== + if "long_context_llm_id" not in user_columns: + op.add_column( + "user", + sa.Column("long_context_llm_id", sa.Integer(), nullable=True), + ) + + if "fast_llm_id" not in user_columns: + op.add_column( + "user", + sa.Column("fast_llm_id", sa.Integer(), nullable=True), + ) + + if "strategic_llm_id" not in user_columns: + op.add_column( + "user", + sa.Column("strategic_llm_id", sa.Integer(), nullable=True), + ) + + # ===== STEP 2: Migrate preferences back to user table ===== + # Take the first preference for each user + op.execute( + """ + UPDATE "user" u + SET + long_context_llm_id = ussp.long_context_llm_id, + fast_llm_id = ussp.fast_llm_id, + strategic_llm_id = ussp.strategic_llm_id + FROM ( + SELECT DISTINCT ON (user_id) + user_id, + long_context_llm_id, + fast_llm_id, + strategic_llm_id + FROM user_search_space_preferences + ORDER BY user_id, created_at ASC + ) ussp + WHERE u.id = ussp.user_id + """ + ) + + # ===== STEP 3: Add foreign key constraints back to user table ===== + op.create_foreign_key( + "fk_user_long_context_llm_id_llm_configs", + "user", + "llm_configs", + ["long_context_llm_id"], + ["id"], + ondelete="SET NULL", + ) + + op.create_foreign_key( + "fk_user_fast_llm_id_llm_configs", + "user", + "llm_configs", + ["fast_llm_id"], + ["id"], + ondelete="SET NULL", + ) + + op.create_foreign_key( + "fk_user_strategic_llm_id_llm_configs", + "user", + "llm_configs", + ["strategic_llm_id"], + ["id"], + ondelete="SET NULL", + ) + + # ===== STEP 4: Drop user_search_space_preferences table ===== + op.execute("DROP TABLE IF EXISTS user_search_space_preferences CASCADE") + + # ===== STEP 5: Add user_id back to llm_configs ===== + if "user_id" not in llm_config_columns: + op.add_column( + "llm_configs", + sa.Column("user_id", postgresql.UUID(), nullable=True), + ) + + # Populate user_id from search_space + op.execute( + """ + UPDATE llm_configs lc + SET user_id = ss.user_id + FROM searchspaces ss + WHERE lc.search_space_id = ss.id + """ + ) + + # Make user_id NOT NULL + op.alter_column( + "llm_configs", + "user_id", + nullable=False, + ) + + # Add foreign key constraint for user_id + op.create_foreign_key( + "fk_llm_configs_user_id_user", + "llm_configs", + "user", + ["user_id"], + ["id"], + ondelete="CASCADE", + ) + + # ===== STEP 6: Remove search_space_id from llm_configs ===== + # Drop foreign key constraint + foreign_keys = [fk["name"] for fk in inspector.get_foreign_keys("llm_configs")] + if "fk_llm_configs_search_space_id" in foreign_keys: + op.drop_constraint( + "fk_llm_configs_search_space_id", + "llm_configs", + type_="foreignkey", + ) + + # Drop search_space_id column + if "search_space_id" in llm_config_columns: + op.drop_column("llm_configs", "search_space_id") diff --git a/surfsense_backend/alembic/versions/26_add_language_column_to_llm_configs.py b/surfsense_backend/alembic/versions/26_add_language_column_to_llm_configs.py new file mode 100644 index 000000000..e5cdc37d7 --- /dev/null +++ b/surfsense_backend/alembic/versions/26_add_language_column_to_llm_configs.py @@ -0,0 +1,69 @@ +"""Add language column to llm_configs + +Revision ID: 26 +Revises: 25 + +Changes: +1. Add language column to llm_configs table with default value of 'English' +""" + +from collections.abc import Sequence + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "26" +down_revision: str | None = "25" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Add language column to llm_configs table.""" + + from sqlalchemy import inspect + + conn = op.get_bind() + inspector = inspect(conn) + + # Get existing columns + llm_config_columns = [col["name"] for col in inspector.get_columns("llm_configs")] + + # Add language column if it doesn't exist + if "language" not in llm_config_columns: + op.add_column( + "llm_configs", + sa.Column( + "language", + sa.String(length=50), + nullable=True, + server_default="English", + ), + ) + + # Update existing rows to have 'English' as default + op.execute( + """ + UPDATE llm_configs + SET language = 'English' + WHERE language IS NULL + """ + ) + + +def downgrade() -> None: + """Remove language column from llm_configs table.""" + + from sqlalchemy import inspect + + conn = op.get_bind() + inspector = inspect(conn) + + # Get existing columns + llm_config_columns = [col["name"] for col in inspector.get_columns("llm_configs")] + + # Drop language column if it exists + if "language" in llm_config_columns: + op.drop_column("llm_configs", "language") diff --git a/surfsense_backend/alembic/versions/27_add_searxng_connector_enum.py b/surfsense_backend/alembic/versions/27_add_searxng_connector_enum.py new file mode 100644 index 000000000..7834a8671 --- /dev/null +++ b/surfsense_backend/alembic/versions/27_add_searxng_connector_enum.py @@ -0,0 +1,41 @@ +"""Add SearxNG connector enum value + +Revision ID: 27 +Revises: 26 +Create Date: 2025-01-18 00:00:00.000000 + +""" + +from collections.abc import Sequence + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "27" +down_revision: str | None = "26" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Safely add SEARXNG_API to searchsourceconnectortype enum.""" + op.execute( + """ + DO $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_type t + JOIN pg_enum e ON t.oid = e.enumtypid + WHERE t.typname = 'searchsourceconnectortype' AND e.enumlabel = 'SEARXNG_API' + ) THEN + ALTER TYPE searchsourceconnectortype ADD VALUE 'SEARXNG_API'; + END IF; + END + $$; + """ + ) + + +def downgrade() -> None: + """Downgrade not supported for enum edits.""" + pass diff --git a/surfsense_backend/alembic/versions/28_add_chinese_litellmprovider_enum.py b/surfsense_backend/alembic/versions/28_add_chinese_litellmprovider_enum.py new file mode 100644 index 000000000..af36c2ea1 --- /dev/null +++ b/surfsense_backend/alembic/versions/28_add_chinese_litellmprovider_enum.py @@ -0,0 +1,107 @@ +"""Add Chinese LLM providers to LiteLLMProvider enum + +Revision ID: 28 +Revises: 27 +""" + +from collections.abc import Sequence + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "28" +down_revision: str | None = "27" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """ + Add Chinese LLM providers to LiteLLMProvider enum. + + Adds support for: + - DEEPSEEK: DeepSeek AI models + - ALIBABA_QWEN: Alibaba Qwen models + - MOONSHOT: Moonshot AI models + - ZHIPU: Zhipu AI models + """ + + # Add DEEPSEEK to the enum if it doesn't already exist + op.execute( + """ + DO $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_enum + WHERE enumtypid = 'litellmprovider'::regtype + AND enumlabel = 'DEEPSEEK' + ) THEN + ALTER TYPE litellmprovider ADD VALUE 'DEEPSEEK'; + END IF; + END$$; + """ + ) + + # Add ALIBABA_QWEN to the enum if it doesn't already exist + op.execute( + """ + DO $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_enum + WHERE enumtypid = 'litellmprovider'::regtype + AND enumlabel = 'ALIBABA_QWEN' + ) THEN + ALTER TYPE litellmprovider ADD VALUE 'ALIBABA_QWEN'; + END IF; + END$$; + """ + ) + + # Add MOONSHOT to the enum if it doesn't already exist + op.execute( + """ + DO $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_enum + WHERE enumtypid = 'litellmprovider'::regtype + AND enumlabel = 'MOONSHOT' + ) THEN + ALTER TYPE litellmprovider ADD VALUE 'MOONSHOT'; + END IF; + END$$; + """ + ) + + # Add ZHIPU to the enum if it doesn't already exist + op.execute( + """ + DO $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_enum + WHERE enumtypid = 'litellmprovider'::regtype + AND enumlabel = 'ZHIPU' + ) THEN + ALTER TYPE litellmprovider ADD VALUE 'ZHIPU'; + END IF; + END$$; + """ + ) + + +def downgrade() -> None: + """ + Remove Chinese LLM providers from LiteLLMProvider enum. + + Note: PostgreSQL doesn't support removing enum values directly. + This would require recreating the enum type and updating all dependent objects. + For safety, this downgrade is a no-op. + + """ + # PostgreSQL doesn't support removing enum values directly + # This would require a complex migration recreating the enum + # PostgreSQL 不支持直接删除枚举值 + # 这需要复杂的迁移来重建枚举 + pass diff --git a/surfsense_backend/app/agents/podcaster/configuration.py b/surfsense_backend/app/agents/podcaster/configuration.py index c4c5f9e9c..453f12676 100644 --- a/surfsense_backend/app/agents/podcaster/configuration.py +++ b/surfsense_backend/app/agents/podcaster/configuration.py @@ -17,6 +17,7 @@ class Configuration: # and when you invoke the graph podcast_title: str user_id: str + search_space_id: int @classmethod def from_runnable_config( diff --git a/surfsense_backend/app/agents/podcaster/nodes.py b/surfsense_backend/app/agents/podcaster/nodes.py index 891928d90..bce9882d6 100644 --- a/surfsense_backend/app/agents/podcaster/nodes.py +++ b/surfsense_backend/app/agents/podcaster/nodes.py @@ -28,11 +28,12 @@ async def create_podcast_transcript( # Get configuration from runnable config configuration = Configuration.from_runnable_config(config) user_id = configuration.user_id + search_space_id = configuration.search_space_id # Get user's long context LLM - llm = await get_user_long_context_llm(state.db_session, user_id) + llm = await get_user_long_context_llm(state.db_session, user_id, search_space_id) if not llm: - error_message = f"No long context LLM configured for user {user_id}" + error_message = f"No long context LLM configured for user {user_id} in search space {search_space_id}" print(error_message) raise RuntimeError(error_message) diff --git a/surfsense_backend/app/agents/researcher/configuration.py b/surfsense_backend/app/agents/researcher/configuration.py index 3e81a59c0..24d8c819e 100644 --- a/surfsense_backend/app/agents/researcher/configuration.py +++ b/surfsense_backend/app/agents/researcher/configuration.py @@ -37,6 +37,7 @@ class Configuration: search_mode: SearchMode research_mode: ResearchMode document_ids_to_add_in_context: list[int] + language: str | None = None @classmethod def from_runnable_config( diff --git a/surfsense_backend/app/agents/researcher/nodes.py b/surfsense_backend/app/agents/researcher/nodes.py index 41e47a75d..2b113785a 100644 --- a/surfsense_backend/app/agents/researcher/nodes.py +++ b/surfsense_backend/app/agents/researcher/nodes.py @@ -577,6 +577,8 @@ async def write_answer_outline( user_query = configuration.user_query num_sections = configuration.num_sections user_id = configuration.user_id + search_space_id = configuration.search_space_id + language = configuration.language # Get language from configuration writer( { @@ -587,9 +589,9 @@ async def write_answer_outline( ) # Get user's strategic LLM - llm = await get_user_strategic_llm(state.db_session, user_id) + llm = await get_user_strategic_llm(state.db_session, user_id, search_space_id) if not llm: - error_message = f"No strategic LLM configured for user {user_id}" + error_message = f"No strategic LLM configured for user {user_id} in search space {search_space_id}" writer({"yield_value": streaming_service.format_error(error_message)}) raise RuntimeError(error_message) @@ -627,7 +629,7 @@ async def write_answer_outline( # Create messages for the LLM messages = [ - SystemMessage(content=get_answer_outline_system_prompt()), + SystemMessage(content=get_answer_outline_system_prompt(language=language)), HumanMessage(content=human_message_content), ] @@ -1010,7 +1012,10 @@ async def fetch_relevant_documents( source_object, tavily_chunks, ) = await connector_service.search_tavily( - user_query=reformulated_query, user_id=user_id, top_k=top_k + user_query=reformulated_query, + user_id=user_id, + search_space_id=search_space_id, + top_k=top_k, ) # Add to sources and raw documents @@ -1028,6 +1033,30 @@ async def fetch_relevant_documents( } ) + elif connector == "SEARXNG_API": + ( + source_object, + searx_chunks, + ) = await connector_service.search_searxng( + user_query=reformulated_query, + user_id=user_id, + search_space_id=search_space_id, + top_k=top_k, + ) + + if source_object: + all_sources.append(source_object) + all_raw_documents.extend(searx_chunks) + + if streaming_service and writer: + writer( + { + "yield_value": streaming_service.format_terminal_info_delta( + f"🌐 Found {len(searx_chunks)} SearxNG results related to your query" + ) + } + ) + elif connector == "LINKUP_API": linkup_mode = "standard" @@ -1037,6 +1066,7 @@ async def fetch_relevant_documents( ) = await connector_service.search_linkup( user_query=reformulated_query, user_id=user_id, + search_space_id=search_space_id, mode=linkup_mode, ) @@ -1850,6 +1880,7 @@ async def reformulate_user_query( user_query=user_query, session=state.db_session, user_id=configuration.user_id, + search_space_id=configuration.search_space_id, chat_history_str=chat_history_str, ) @@ -1994,6 +2025,7 @@ async def handle_qna_workflow( "relevant_documents": all_documents, # Use combined documents "user_id": configuration.user_id, "search_space_id": configuration.search_space_id, + "language": configuration.language, } } @@ -2089,6 +2121,7 @@ async def generate_further_questions( configuration = Configuration.from_runnable_config(config) chat_history = state.chat_history user_id = configuration.user_id + search_space_id = configuration.search_space_id streaming_service = state.streaming_service # Get reranked documents from the state (will be populated by sub-agents) @@ -2103,9 +2136,9 @@ async def generate_further_questions( ) # Get user's fast LLM - llm = await get_user_fast_llm(state.db_session, user_id) + llm = await get_user_fast_llm(state.db_session, user_id, search_space_id) if not llm: - error_message = f"No fast LLM configured for user {user_id}" + error_message = f"No fast LLM configured for user {user_id} in search space {search_space_id}" print(error_message) writer({"yield_value": streaming_service.format_error(error_message)}) diff --git a/surfsense_backend/app/agents/researcher/prompts.py b/surfsense_backend/app/agents/researcher/prompts.py index 44b218913..825772a24 100644 --- a/surfsense_backend/app/agents/researcher/prompts.py +++ b/surfsense_backend/app/agents/researcher/prompts.py @@ -1,9 +1,18 @@ import datetime -def get_answer_outline_system_prompt(): +def _build_language_instruction(language: str | None = None): + if language: + return f"\n\nIMPORTANT: Please respond in {language} language. All your responses, explanations, and analysis should be written in {language}." + return "" + + +def get_answer_outline_system_prompt(language: str | None = None) -> str: + language_instruction = _build_language_instruction(language) + return f""" Today's date: {datetime.datetime.now().strftime("%Y-%m-%d")} +{language_instruction} You are an expert research assistant specializing in structuring information. Your task is to create a detailed and logical research outline based on the user's query. This outline will serve as the blueprint for generating a comprehensive research report. diff --git a/surfsense_backend/app/agents/researcher/qna_agent/configuration.py b/surfsense_backend/app/agents/researcher/qna_agent/configuration.py index 5a4529e0d..ea107a575 100644 --- a/surfsense_backend/app/agents/researcher/qna_agent/configuration.py +++ b/surfsense_backend/app/agents/researcher/qna_agent/configuration.py @@ -20,6 +20,7 @@ class Configuration: ] # Documents provided directly to the agent for answering user_id: str # User identifier search_space_id: int # Search space identifier + language: str | None = None # Language for responses @classmethod def from_runnable_config( diff --git a/surfsense_backend/app/agents/researcher/qna_agent/nodes.py b/surfsense_backend/app/agents/researcher/qna_agent/nodes.py index 4e01bbb58..c4e79d685 100644 --- a/surfsense_backend/app/agents/researcher/qna_agent/nodes.py +++ b/surfsense_backend/app/agents/researcher/qna_agent/nodes.py @@ -101,11 +101,12 @@ async def answer_question(state: State, config: RunnableConfig) -> dict[str, Any documents = state.reranked_documents user_query = configuration.user_query user_id = configuration.user_id - + search_space_id = configuration.search_space_id + language = configuration.language # Get user's fast LLM - llm = await get_user_fast_llm(state.db_session, user_id) + llm = await get_user_fast_llm(state.db_session, user_id, search_space_id) if not llm: - error_message = f"No fast LLM configured for user {user_id}" + error_message = f"No fast LLM configured for user {user_id} in search space {search_space_id}" print(error_message) raise RuntimeError(error_message) @@ -126,7 +127,9 @@ async def answer_question(state: State, config: RunnableConfig) -> dict[str, Any """ # Use initial system prompt for token calculation - initial_system_prompt = get_qna_citation_system_prompt(chat_history_str) + initial_system_prompt = get_qna_citation_system_prompt( + chat_history_str, language + ) base_messages = [ SystemMessage(content=initial_system_prompt), HumanMessage(content=base_human_message_template), @@ -145,9 +148,9 @@ async def answer_question(state: State, config: RunnableConfig) -> dict[str, Any # Choose system prompt based on final document availability system_prompt = ( - get_qna_citation_system_prompt(chat_history_str) + get_qna_citation_system_prompt(chat_history_str, language) if has_documents - else get_qna_no_documents_system_prompt(chat_history_str) + else get_qna_no_documents_system_prompt(chat_history_str, language) ) # Generate documents section diff --git a/surfsense_backend/app/agents/researcher/qna_agent/prompts.py b/surfsense_backend/app/agents/researcher/qna_agent/prompts.py index 212788804..9c35f90cc 100644 --- a/surfsense_backend/app/agents/researcher/qna_agent/prompts.py +++ b/surfsense_backend/app/agents/researcher/qna_agent/prompts.py @@ -1,7 +1,11 @@ import datetime +from ..prompts import _build_language_instruction -def get_qna_citation_system_prompt(chat_history: str | None = None): + +def get_qna_citation_system_prompt( + chat_history: str | None = None, language: str | None = None +): chat_history_section = ( f""" @@ -16,9 +20,11 @@ NO CHAT HISTORY PROVIDED """ ) + # Add language instruction if specified + language_instruction = _build_language_instruction(language) return f""" Today's date: {datetime.datetime.now().strftime("%Y-%m-%d")} -You are SurfSense, an advanced AI research assistant that provides detailed, well-researched answers to user questions by synthesizing information from multiple personal knowledge sources. +You are SurfSense, an advanced AI research assistant that provides detailed, well-researched answers to user questions by synthesizing information from multiple personal knowledge sources.{language_instruction} {chat_history_section} - EXTENSION: "Web content saved via SurfSense browser extension" (personal browsing history) @@ -149,7 +155,9 @@ Make sure your response: """ -def get_qna_no_documents_system_prompt(chat_history: str | None = None): +def get_qna_no_documents_system_prompt( + chat_history: str | None = None, language: str | None = None +): chat_history_section = ( f""" @@ -164,9 +172,12 @@ NO CHAT HISTORY PROVIDED """ ) + # Add language instruction if specified + language_instruction = _build_language_instruction(language) + return f""" Today's date: {datetime.datetime.now().strftime("%Y-%m-%d")} -You are SurfSense, an advanced AI research assistant that provides helpful, detailed answers to user questions in a conversational manner. +You are SurfSense, an advanced AI research assistant that provides helpful, detailed answers to user questions in a conversational manner.{language_instruction} {chat_history_section} The user has asked a question but there are no specific documents from their personal knowledge base available to answer it. You should provide a helpful response based on: diff --git a/surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py b/surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py index 91a8bf84e..153cafac5 100644 --- a/surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py +++ b/surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py @@ -107,11 +107,12 @@ async def write_sub_section(state: State, config: RunnableConfig) -> dict[str, A configuration = Configuration.from_runnable_config(config) documents = state.reranked_documents user_id = configuration.user_id + search_space_id = configuration.search_space_id # Get user's fast LLM - llm = await get_user_fast_llm(state.db_session, user_id) + llm = await get_user_fast_llm(state.db_session, user_id, search_space_id) if not llm: - error_message = f"No fast LLM configured for user {user_id}" + error_message = f"No fast LLM configured for user {user_id} in search space {search_space_id}" print(error_message) raise RuntimeError(error_message) diff --git a/surfsense_backend/app/agents/researcher/sub_section_writer/prompts.py b/surfsense_backend/app/agents/researcher/sub_section_writer/prompts.py index c3d487671..3c34eb474 100644 --- a/surfsense_backend/app/agents/researcher/sub_section_writer/prompts.py +++ b/surfsense_backend/app/agents/researcher/sub_section_writer/prompts.py @@ -1,7 +1,11 @@ import datetime +from ..prompts import _build_language_instruction -def get_citation_system_prompt(chat_history: str | None = None): + +def get_citation_system_prompt( + chat_history: str | None = None, language: str | None = None +): chat_history_section = ( f""" @@ -16,9 +20,12 @@ NO CHAT HISTORY PROVIDED """ ) + # Add language instruction if specified + language_instruction = _build_language_instruction(language) + return f""" Today's date: {datetime.datetime.now().strftime("%Y-%m-%d")} -You are SurfSense, an advanced AI research assistant that synthesizes information from multiple knowledge sources to provide comprehensive, well-cited answers to user queries. +You are SurfSense, an advanced AI research assistant that synthesizes information from multiple knowledge sources to provide comprehensive, well-cited answers to user queries.{language_instruction} {chat_history_section} - EXTENSION: "Web content saved via SurfSense browser extension" (personal browsing history) @@ -156,7 +163,9 @@ Make sure your response: """ -def get_no_documents_system_prompt(chat_history: str | None = None): +def get_no_documents_system_prompt( + chat_history: str | None = None, language: str | None = None +): chat_history_section = ( f""" @@ -171,9 +180,12 @@ NO CHAT HISTORY PROVIDED """ ) + # Add language instruction if specified + language_instruction = _build_language_instruction(language) + return f""" Today's date: {datetime.datetime.now().strftime("%Y-%m-%d")} -You are SurfSense, an advanced AI research assistant that helps users create well-structured content for their documents and research. +You are SurfSense, an advanced AI research assistant that helps users create well-structured content for their documents and research.{language_instruction} {chat_history_section} You are writing content for a specific sub-section of a document. No specific documents from the user's personal knowledge base are available, so you should create content based on: diff --git a/surfsense_backend/app/connectors/google_calendar_connector.py b/surfsense_backend/app/connectors/google_calendar_connector.py index 2f8846cc0..164d230e0 100644 --- a/surfsense_backend/app/connectors/google_calendar_connector.py +++ b/surfsense_backend/app/connectors/google_calendar_connector.py @@ -31,15 +31,20 @@ class GoogleCalendarConnector: credentials: Credentials, session: AsyncSession, user_id: str, + connector_id: int | None = None, ): """ Initialize the GoogleCalendarConnector class. Args: credentials: Google OAuth Credentials object + session: Database session for updating connector + user_id: User ID (kept for backward compatibility) + connector_id: Optional connector ID for direct updates """ self._credentials = credentials self._session = session self._user_id = user_id + self._connector_id = connector_id self.service = None async def _get_credentials( @@ -84,17 +89,25 @@ class GoogleCalendarConnector: self._credentials.refresh(Request()) # Update the connector config in DB if self._session: - result = await self._session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.user_id == self._user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.GOOGLE_CALENDAR_CONNECTOR, + # Use connector_id if available, otherwise fall back to user_id query + if self._connector_id: + result = await self._session.execute( + select(SearchSourceConnector).filter( + SearchSourceConnector.id == self._connector_id + ) + ) + else: + result = await self._session.execute( + select(SearchSourceConnector).filter( + SearchSourceConnector.user_id == self._user_id, + SearchSourceConnector.connector_type + == SearchSourceConnectorType.GOOGLE_CALENDAR_CONNECTOR, + ) ) - ) connector = result.scalars().first() if connector is None: raise RuntimeError( - "GOOGLE_CALENDAR_CONNECTOR connector not found for current user; cannot persist refreshed token." + "GOOGLE_CALENDAR_CONNECTOR connector not found; cannot persist refreshed token." ) connector.config = json.loads(self._credentials.to_json()) flag_modified(connector, "config") diff --git a/surfsense_backend/app/connectors/google_gmail_connector.py b/surfsense_backend/app/connectors/google_gmail_connector.py index d012ade96..637680560 100644 --- a/surfsense_backend/app/connectors/google_gmail_connector.py +++ b/surfsense_backend/app/connectors/google_gmail_connector.py @@ -30,15 +30,20 @@ class GoogleGmailConnector: credentials: Credentials, session: AsyncSession, user_id: str, + connector_id: int | None = None, ): """ Initialize the GoogleGmailConnector class. Args: credentials: Google OAuth Credentials object + session: Database session for updating connector + user_id: User ID (kept for backward compatibility) + connector_id: Optional connector ID for direct updates """ self._credentials = credentials self._session = session self._user_id = user_id + self._connector_id = connector_id self.service = None async def _get_credentials( @@ -83,17 +88,25 @@ class GoogleGmailConnector: self._credentials.refresh(Request()) # Update the connector config in DB if self._session: - result = await self._session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.user_id == self._user_id, - SearchSourceConnector.connector_type - == SearchSourceConnectorType.GOOGLE_GMAIL_CONNECTOR, + # Use connector_id if available, otherwise fall back to user_id query + if self._connector_id: + result = await self._session.execute( + select(SearchSourceConnector).filter( + SearchSourceConnector.id == self._connector_id + ) + ) + else: + result = await self._session.execute( + select(SearchSourceConnector).filter( + SearchSourceConnector.user_id == self._user_id, + SearchSourceConnector.connector_type + == SearchSourceConnectorType.GOOGLE_GMAIL_CONNECTOR, + ) ) - ) connector = result.scalars().first() if connector is None: raise RuntimeError( - "GMAIL connector not found for current user; cannot persist refreshed token." + "GMAIL connector not found; cannot persist refreshed token." ) connector.config = json.loads(self._credentials.to_json()) flag_modified(connector, "config") diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py index a67c27a61..d2d30afc4 100644 --- a/surfsense_backend/app/db.py +++ b/surfsense_backend/app/db.py @@ -55,6 +55,7 @@ class DocumentType(str, Enum): class SearchSourceConnectorType(str, Enum): SERPER_API = "SERPER_API" # NOT IMPLEMENTED YET : DON'T REMEMBER WHY : MOST PROBABLY BECAUSE WE NEED TO CRAWL THE RESULTS RETURNED BY IT TAVILY_API = "TAVILY_API" + SEARXNG_API = "SEARXNG_API" LINKUP_API = "LINKUP_API" SLACK_CONNECTOR = "SLACK_CONNECTOR" NOTION_CONNECTOR = "NOTION_CONNECTOR" @@ -78,6 +79,11 @@ class ChatType(str, Enum): class LiteLLMProvider(str, Enum): + """ + Enum for LLM providers supported by LiteLLM. + LiteLLM 支持的 LLM 提供商枚举。 + """ + OPENAI = "OPENAI" ANTHROPIC = "ANTHROPIC" GROQ = "GROQ" @@ -101,6 +107,11 @@ class LiteLLMProvider(str, Enum): ALEPH_ALPHA = "ALEPH_ALPHA" PETALS = "PETALS" COMETAPI = "COMETAPI" + # Chinese LLM Providers (OpenAI-compatible) + DEEPSEEK = "DEEPSEEK" + ALIBABA_QWEN = "ALIBABA_QWEN" + MOONSHOT = "MOONSHOT" + ZHIPU = "ZHIPU" CUSTOM = "CUSTOM" @@ -252,7 +263,12 @@ class SearchSpace(BaseModel, TimestampMixin): class SearchSourceConnector(BaseModel, TimestampMixin): __tablename__ = "search_source_connectors" __table_args__ = ( - UniqueConstraint("user_id", "connector_type", name="uq_user_connector_type"), + UniqueConstraint( + "search_space_id", + "user_id", + "connector_type", + name="uq_searchspace_user_connector_type", + ), ) name = Column(String(100), nullable=False, index=True) @@ -261,6 +277,13 @@ class SearchSourceConnector(BaseModel, TimestampMixin): last_indexed_at = Column(TIMESTAMP(timezone=True), nullable=True) config = Column(JSON, nullable=False) + search_space_id = Column( + Integer, ForeignKey("searchspaces.id", ondelete="CASCADE"), nullable=False + ) + search_space = relationship( + "SearchSpace", back_populates="search_source_connectors" + ) + user_id = Column( UUID(as_uuid=True), ForeignKey("user.id", ondelete="CASCADE"), nullable=False ) @@ -315,13 +338,59 @@ class LLMConfig(BaseModel, TimestampMixin): api_key = Column(String, nullable=False) api_base = Column(String(500), nullable=True) + language = Column(String(50), nullable=True, default="English") + # For any other parameters that litellm supports litellm_params = Column(JSON, nullable=True, default={}) + search_space_id = Column( + Integer, ForeignKey("searchspaces.id", ondelete="CASCADE"), nullable=False + ) + search_space = relationship("SearchSpace", back_populates="llm_configs") + + +class UserSearchSpacePreference(BaseModel, TimestampMixin): + __tablename__ = "user_search_space_preferences" + __table_args__ = ( + UniqueConstraint( + "user_id", + "search_space_id", + name="uq_user_searchspace", + ), + ) + user_id = Column( UUID(as_uuid=True), ForeignKey("user.id", ondelete="CASCADE"), nullable=False ) - user = relationship("User", back_populates="llm_configs", foreign_keys=[user_id]) + search_space_id = Column( + Integer, ForeignKey("searchspaces.id", ondelete="CASCADE"), nullable=False + ) + + # User-specific LLM preferences for this search space + long_context_llm_id = Column( + Integer, ForeignKey("llm_configs.id", ondelete="SET NULL"), nullable=True + ) + fast_llm_id = Column( + Integer, ForeignKey("llm_configs.id", ondelete="SET NULL"), nullable=True + ) + strategic_llm_id = Column( + Integer, ForeignKey("llm_configs.id", ondelete="SET NULL"), nullable=True + ) + + # Future RBAC fields can be added here + # role = Column(String(50), nullable=True) # e.g., 'owner', 'editor', 'viewer' + # permissions = Column(JSON, nullable=True) + + user = relationship("User", back_populates="search_space_preferences") + search_space = relationship("SearchSpace", back_populates="user_preferences") + + long_context_llm = relationship( + "LLMConfig", foreign_keys=[long_context_llm_id], post_update=True + ) + fast_llm = relationship("LLMConfig", foreign_keys=[fast_llm_id], post_update=True) + strategic_llm = relationship( + "LLMConfig", foreign_keys=[strategic_llm_id], post_update=True + ) class Log(BaseModel, TimestampMixin): @@ -351,70 +420,22 @@ if config.AUTH_TYPE == "GOOGLE": "OAuthAccount", lazy="joined" ) search_spaces = relationship("SearchSpace", back_populates="user") - search_source_connectors = relationship( - "SearchSourceConnector", back_populates="user" - ) - llm_configs = relationship( - "LLMConfig", + search_space_preferences = relationship( + "UserSearchSpacePreference", back_populates="user", - foreign_keys="LLMConfig.user_id", cascade="all, delete-orphan", ) - long_context_llm_id = Column( - Integer, ForeignKey("llm_configs.id", ondelete="SET NULL"), nullable=True - ) - fast_llm_id = Column( - Integer, ForeignKey("llm_configs.id", ondelete="SET NULL"), nullable=True - ) - strategic_llm_id = Column( - Integer, ForeignKey("llm_configs.id", ondelete="SET NULL"), nullable=True - ) - - long_context_llm = relationship( - "LLMConfig", foreign_keys=[long_context_llm_id], post_update=True - ) - fast_llm = relationship( - "LLMConfig", foreign_keys=[fast_llm_id], post_update=True - ) - strategic_llm = relationship( - "LLMConfig", foreign_keys=[strategic_llm_id], post_update=True - ) - else: class User(SQLAlchemyBaseUserTableUUID, Base): search_spaces = relationship("SearchSpace", back_populates="user") - search_source_connectors = relationship( - "SearchSourceConnector", back_populates="user" - ) - llm_configs = relationship( - "LLMConfig", + search_space_preferences = relationship( + "UserSearchSpacePreference", back_populates="user", - foreign_keys="LLMConfig.user_id", cascade="all, delete-orphan", ) - long_context_llm_id = Column( - Integer, ForeignKey("llm_configs.id", ondelete="SET NULL"), nullable=True - ) - fast_llm_id = Column( - Integer, ForeignKey("llm_configs.id", ondelete="SET NULL"), nullable=True - ) - strategic_llm_id = Column( - Integer, ForeignKey("llm_configs.id", ondelete="SET NULL"), nullable=True - ) - - long_context_llm = relationship( - "LLMConfig", foreign_keys=[long_context_llm_id], post_update=True - ) - fast_llm = relationship( - "LLMConfig", foreign_keys=[fast_llm_id], post_update=True - ) - strategic_llm = relationship( - "LLMConfig", foreign_keys=[strategic_llm_id], post_update=True - ) - engine = create_async_engine(DATABASE_URL) async_session_maker = async_sessionmaker(engine, expire_on_commit=False) diff --git a/surfsense_backend/app/routes/airtable_add_connector_route.py b/surfsense_backend/app/routes/airtable_add_connector_route.py index 1820c26d2..d5959cf16 100644 --- a/surfsense_backend/app/routes/airtable_add_connector_route.py +++ b/surfsense_backend/app/routes/airtable_add_connector_route.py @@ -217,9 +217,10 @@ async def airtable_callback( scope=token_json.get("scope"), ) - # Check if connector already exists for this user + # Check if connector already exists for this search space and user existing_connector_result = await session.execute( select(SearchSourceConnector).filter( + SearchSourceConnector.search_space_id == space_id, SearchSourceConnector.user_id == user_id, SearchSourceConnector.connector_type == SearchSourceConnectorType.AIRTABLE_CONNECTOR, @@ -232,7 +233,9 @@ async def airtable_callback( existing_connector.config = credentials.to_dict() existing_connector.name = "Airtable Connector" existing_connector.is_indexable = True - logger.info(f"Updated existing Airtable connector for user {user_id}") + logger.info( + f"Updated existing Airtable connector for user {user_id} in space {space_id}" + ) else: # Create new connector new_connector = SearchSourceConnector( @@ -240,10 +243,13 @@ async def airtable_callback( connector_type=SearchSourceConnectorType.AIRTABLE_CONNECTOR, is_indexable=True, config=credentials.to_dict(), + search_space_id=space_id, user_id=user_id, ) session.add(new_connector) - logger.info(f"Created new Airtable connector for user {user_id}") + logger.info( + f"Created new Airtable connector for user {user_id} in space {space_id}" + ) try: await session.commit() diff --git a/surfsense_backend/app/routes/chats_routes.py b/surfsense_backend/app/routes/chats_routes.py index 1dcda505c..e003dc260 100644 --- a/surfsense_backend/app/routes/chats_routes.py +++ b/surfsense_backend/app/routes/chats_routes.py @@ -4,8 +4,9 @@ from langchain.schema import AIMessage, HumanMessage from sqlalchemy.exc import IntegrityError, OperationalError from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select +from sqlalchemy.orm import selectinload -from app.db import Chat, SearchSpace, User, get_async_session +from app.db import Chat, SearchSpace, User, UserSearchSpacePreference, get_async_session from app.schemas import ( AISDKChatRequest, ChatCreate, @@ -17,19 +18,17 @@ from app.tasks.stream_connector_search_results import stream_connector_search_re from app.users import current_active_user from app.utils.check_ownership import check_ownership from app.utils.validators import ( - validate_search_space_id, - validate_document_ids, validate_connectors, + validate_document_ids, + validate_messages, validate_research_mode, validate_search_mode, - validate_messages, + validate_search_space_id, ) router = APIRouter() - - @router.post("/chat") async def handle_chat_data( request: AISDKChatRequest, @@ -38,25 +37,68 @@ async def handle_chat_data( ): # Validate and sanitize all input data messages = validate_messages(request.messages) - + if messages[-1]["role"] != "user": raise HTTPException( status_code=400, detail="Last message must be a user message" ) user_query = messages[-1]["content"] - + # Extract and validate data from request request_data = request.data or {} search_space_id = validate_search_space_id(request_data.get("search_space_id")) research_mode = validate_research_mode(request_data.get("research_mode")) selected_connectors = validate_connectors(request_data.get("selected_connectors")) - document_ids_to_add_in_context = validate_document_ids(request_data.get("document_ids_to_add_in_context")) + document_ids_to_add_in_context = validate_document_ids( + request_data.get("document_ids_to_add_in_context") + ) search_mode_str = validate_search_mode(request_data.get("search_mode")) + # print("RESQUEST DATA:", request_data) + # print("SELECTED CONNECTORS:", selected_connectors) # Check if the search space belongs to the current user try: await check_ownership(session, SearchSpace, search_space_id, user) + language_result = await session.execute( + select(UserSearchSpacePreference) + .options( + selectinload(UserSearchSpacePreference.search_space).selectinload( + SearchSpace.llm_configs + ), + selectinload(UserSearchSpacePreference.long_context_llm), + selectinload(UserSearchSpacePreference.fast_llm), + selectinload(UserSearchSpacePreference.strategic_llm), + ) + .filter( + UserSearchSpacePreference.search_space_id == search_space_id, + UserSearchSpacePreference.user_id == user.id, + ) + ) + user_preference = language_result.scalars().first() + # print("UserSearchSpacePreference:", user_preference) + + language = None + if ( + user_preference + and user_preference.search_space + and user_preference.search_space.llm_configs + ): + llm_configs = user_preference.search_space.llm_configs + + for preferred_llm in [ + user_preference.fast_llm, + user_preference.long_context_llm, + user_preference.strategic_llm, + ]: + if preferred_llm and getattr(preferred_llm, "language", None): + language = preferred_llm.language + break + + if not language: + first_llm_config = llm_configs[0] + language = getattr(first_llm_config, "language", None) + except HTTPException: raise HTTPException( status_code=403, detail="You don't have access to this search space" @@ -80,6 +122,7 @@ async def handle_chat_data( langchain_chat_history, search_mode_str, document_ids_to_add_in_context, + language, ) ) @@ -132,21 +175,16 @@ async def read_chats( # Validate pagination parameters if skip < 0: raise HTTPException( - status_code=400, - detail="skip must be a non-negative integer" + status_code=400, detail="skip must be a non-negative integer" ) - + if limit <= 0 or limit > 1000: # Reasonable upper limit - raise HTTPException( - status_code=400, - detail="limit must be between 1 and 1000" - ) - + raise HTTPException(status_code=400, detail="limit must be between 1 and 1000") + # Validate search_space_id if provided if search_space_id is not None and search_space_id <= 0: raise HTTPException( - status_code=400, - detail="search_space_id must be a positive integer" + status_code=400, detail="search_space_id must be a positive integer" ) try: # Select specific fields excluding messages diff --git a/surfsense_backend/app/routes/documents_routes.py b/surfsense_backend/app/routes/documents_routes.py index dd7b56033..63d7d2178 100644 --- a/surfsense_backend/app/routes/documents_routes.py +++ b/surfsense_backend/app/routes/documents_routes.py @@ -1070,6 +1070,7 @@ async def process_file_in_background( }, ) except Exception as e: + await session.rollback() await task_logger.log_task_failure( log_entry, f"Failed to process file: {filename}", diff --git a/surfsense_backend/app/routes/google_calendar_add_connector_route.py b/surfsense_backend/app/routes/google_calendar_add_connector_route.py index a319c87ee..e1356809b 100644 --- a/surfsense_backend/app/routes/google_calendar_add_connector_route.py +++ b/surfsense_backend/app/routes/google_calendar_add_connector_route.py @@ -105,9 +105,10 @@ async def calendar_callback( creds_dict = json.loads(creds.to_json()) try: - # Check if a connector with the same type already exists for this user + # Check if a connector with the same type already exists for this search space and user result = await session.execute( select(SearchSourceConnector).filter( + SearchSourceConnector.search_space_id == space_id, SearchSourceConnector.user_id == user_id, SearchSourceConnector.connector_type == SearchSourceConnectorType.GOOGLE_CALENDAR_CONNECTOR, @@ -117,12 +118,13 @@ async def calendar_callback( if existing_connector: raise HTTPException( status_code=409, - detail="A GOOGLE_CALENDAR_CONNECTOR connector already exists. Each user can have only one connector of each type.", + detail="A GOOGLE_CALENDAR_CONNECTOR connector already exists in this search space. Each search space can have only one connector of each type per user.", ) db_connector = SearchSourceConnector( name="Google Calendar Connector", connector_type=SearchSourceConnectorType.GOOGLE_CALENDAR_CONNECTOR, config=creds_dict, + search_space_id=space_id, user_id=user_id, is_indexable=True, ) diff --git a/surfsense_backend/app/routes/google_gmail_add_connector_route.py b/surfsense_backend/app/routes/google_gmail_add_connector_route.py index 01c6e7fb9..59badd071 100644 --- a/surfsense_backend/app/routes/google_gmail_add_connector_route.py +++ b/surfsense_backend/app/routes/google_gmail_add_connector_route.py @@ -104,9 +104,10 @@ async def gmail_callback( creds_dict = json.loads(creds.to_json()) try: - # Check if a connector with the same type already exists for this user + # Check if a connector with the same type already exists for this search space and user result = await session.execute( select(SearchSourceConnector).filter( + SearchSourceConnector.search_space_id == space_id, SearchSourceConnector.user_id == user_id, SearchSourceConnector.connector_type == SearchSourceConnectorType.GOOGLE_GMAIL_CONNECTOR, @@ -116,12 +117,13 @@ async def gmail_callback( if existing_connector: raise HTTPException( status_code=409, - detail="A GOOGLE_GMAIL_CONNECTOR connector already exists. Each user can have only one connector of each type.", + detail="A GOOGLE_GMAIL_CONNECTOR connector already exists in this search space. Each search space can have only one connector of each type per user.", ) db_connector = SearchSourceConnector( name="Google Gmail Connector", connector_type=SearchSourceConnectorType.GOOGLE_GMAIL_CONNECTOR, config=creds_dict, + search_space_id=space_id, user_id=user_id, is_indexable=True, ) diff --git a/surfsense_backend/app/routes/llm_config_routes.py b/surfsense_backend/app/routes/llm_config_routes.py index ce76dc9bc..ec8ea5846 100644 --- a/surfsense_backend/app/routes/llm_config_routes.py +++ b/surfsense_backend/app/routes/llm_config_routes.py @@ -2,15 +2,72 @@ from fastapi import APIRouter, Depends, HTTPException from pydantic import BaseModel from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select +from sqlalchemy.orm import selectinload -from app.db import LLMConfig, User, get_async_session +from app.db import ( + LLMConfig, + SearchSpace, + User, + UserSearchSpacePreference, + get_async_session, +) from app.schemas import LLMConfigCreate, LLMConfigRead, LLMConfigUpdate from app.users import current_active_user -from app.utils.check_ownership import check_ownership router = APIRouter() +# Helper function to check search space access +async def check_search_space_access( + session: AsyncSession, search_space_id: int, user: User +) -> SearchSpace: + """Verify that the user has access to the search space""" + result = await session.execute( + select(SearchSpace).filter( + SearchSpace.id == search_space_id, SearchSpace.user_id == user.id + ) + ) + search_space = result.scalars().first() + if not search_space: + raise HTTPException( + status_code=404, + detail="Search space not found or you don't have permission to access it", + ) + return search_space + + +# Helper function to get or create user search space preference +async def get_or_create_user_preference( + session: AsyncSession, user_id, search_space_id: int +) -> UserSearchSpacePreference: + """Get or create user preference for a search space""" + result = await session.execute( + select(UserSearchSpacePreference) + .filter( + UserSearchSpacePreference.user_id == user_id, + UserSearchSpacePreference.search_space_id == search_space_id, + ) + .options( + selectinload(UserSearchSpacePreference.long_context_llm), + selectinload(UserSearchSpacePreference.fast_llm), + selectinload(UserSearchSpacePreference.strategic_llm), + ) + ) + preference = result.scalars().first() + + if not preference: + # Create new preference entry + preference = UserSearchSpacePreference( + user_id=user_id, + search_space_id=search_space_id, + ) + session.add(preference) + await session.commit() + await session.refresh(preference) + + return preference + + class LLMPreferencesUpdate(BaseModel): """Schema for updating user LLM preferences""" @@ -36,9 +93,12 @@ async def create_llm_config( session: AsyncSession = Depends(get_async_session), user: User = Depends(current_active_user), ): - """Create a new LLM configuration for the authenticated user""" + """Create a new LLM configuration for a search space""" try: - db_llm_config = LLMConfig(**llm_config.model_dump(), user_id=user.id) + # Verify user has access to the search space + await check_search_space_access(session, llm_config.search_space_id, user) + + db_llm_config = LLMConfig(**llm_config.model_dump()) session.add(db_llm_config) await session.commit() await session.refresh(db_llm_config) @@ -54,20 +114,26 @@ async def create_llm_config( @router.get("/llm-configs/", response_model=list[LLMConfigRead]) async def read_llm_configs( + search_space_id: int, skip: int = 0, limit: int = 200, session: AsyncSession = Depends(get_async_session), user: User = Depends(current_active_user), ): - """Get all LLM configurations for the authenticated user""" + """Get all LLM configurations for a search space""" try: + # Verify user has access to the search space + await check_search_space_access(session, search_space_id, user) + result = await session.execute( select(LLMConfig) - .filter(LLMConfig.user_id == user.id) + .filter(LLMConfig.search_space_id == search_space_id) .offset(skip) .limit(limit) ) return result.scalars().all() + except HTTPException: + raise except Exception as e: raise HTTPException( status_code=500, detail=f"Failed to fetch LLM configurations: {e!s}" @@ -82,7 +148,18 @@ async def read_llm_config( ): """Get a specific LLM configuration by ID""" try: - llm_config = await check_ownership(session, LLMConfig, llm_config_id, user) + # Get the LLM config + result = await session.execute( + select(LLMConfig).filter(LLMConfig.id == llm_config_id) + ) + llm_config = result.scalars().first() + + if not llm_config: + raise HTTPException(status_code=404, detail="LLM configuration not found") + + # Verify user has access to the search space + await check_search_space_access(session, llm_config.search_space_id, user) + return llm_config except HTTPException: raise @@ -101,7 +178,18 @@ async def update_llm_config( ): """Update an existing LLM configuration""" try: - db_llm_config = await check_ownership(session, LLMConfig, llm_config_id, user) + # Get the LLM config + result = await session.execute( + select(LLMConfig).filter(LLMConfig.id == llm_config_id) + ) + db_llm_config = result.scalars().first() + + if not db_llm_config: + raise HTTPException(status_code=404, detail="LLM configuration not found") + + # Verify user has access to the search space + await check_search_space_access(session, db_llm_config.search_space_id, user) + update_data = llm_config_update.model_dump(exclude_unset=True) for key, value in update_data.items(): @@ -127,7 +215,18 @@ async def delete_llm_config( ): """Delete an LLM configuration""" try: - db_llm_config = await check_ownership(session, LLMConfig, llm_config_id, user) + # Get the LLM config + result = await session.execute( + select(LLMConfig).filter(LLMConfig.id == llm_config_id) + ) + db_llm_config = result.scalars().first() + + if not db_llm_config: + raise HTTPException(status_code=404, detail="LLM configuration not found") + + # Verify user has access to the search space + await check_search_space_access(session, db_llm_config.search_space_id, user) + await session.delete(db_llm_config) await session.commit() return {"message": "LLM configuration deleted successfully"} @@ -143,99 +242,114 @@ async def delete_llm_config( # User LLM Preferences endpoints -@router.get("/users/me/llm-preferences", response_model=LLMPreferencesRead) +@router.get( + "/search-spaces/{search_space_id}/llm-preferences", + response_model=LLMPreferencesRead, +) async def get_user_llm_preferences( + search_space_id: int, session: AsyncSession = Depends(get_async_session), user: User = Depends(current_active_user), ): - """Get the current user's LLM preferences""" + """Get the current user's LLM preferences for a specific search space""" try: - # Refresh user to get latest relationships - await session.refresh(user) + # Verify user has access to the search space + await check_search_space_access(session, search_space_id, user) - result = { - "long_context_llm_id": user.long_context_llm_id, - "fast_llm_id": user.fast_llm_id, - "strategic_llm_id": user.strategic_llm_id, - "long_context_llm": None, - "fast_llm": None, - "strategic_llm": None, + # Get or create user preference for this search space + preference = await get_or_create_user_preference( + session, user.id, search_space_id + ) + + return { + "long_context_llm_id": preference.long_context_llm_id, + "fast_llm_id": preference.fast_llm_id, + "strategic_llm_id": preference.strategic_llm_id, + "long_context_llm": preference.long_context_llm, + "fast_llm": preference.fast_llm, + "strategic_llm": preference.strategic_llm, } - - # Fetch the actual LLM configs if they exist - if user.long_context_llm_id: - long_context_llm = await session.execute( - select(LLMConfig).filter( - LLMConfig.id == user.long_context_llm_id, - LLMConfig.user_id == user.id, - ) - ) - llm_config = long_context_llm.scalars().first() - if llm_config: - result["long_context_llm"] = llm_config - - if user.fast_llm_id: - fast_llm = await session.execute( - select(LLMConfig).filter( - LLMConfig.id == user.fast_llm_id, LLMConfig.user_id == user.id - ) - ) - llm_config = fast_llm.scalars().first() - if llm_config: - result["fast_llm"] = llm_config - - if user.strategic_llm_id: - strategic_llm = await session.execute( - select(LLMConfig).filter( - LLMConfig.id == user.strategic_llm_id, LLMConfig.user_id == user.id - ) - ) - llm_config = strategic_llm.scalars().first() - if llm_config: - result["strategic_llm"] = llm_config - - return result + except HTTPException: + raise except Exception as e: raise HTTPException( status_code=500, detail=f"Failed to fetch LLM preferences: {e!s}" ) from e -@router.put("/users/me/llm-preferences", response_model=LLMPreferencesRead) +@router.put( + "/search-spaces/{search_space_id}/llm-preferences", + response_model=LLMPreferencesRead, +) async def update_user_llm_preferences( + search_space_id: int, preferences: LLMPreferencesUpdate, session: AsyncSession = Depends(get_async_session), user: User = Depends(current_active_user), ): - """Update the current user's LLM preferences""" + """Update the current user's LLM preferences for a specific search space""" try: - # Validate that all provided LLM config IDs belong to the user + # Verify user has access to the search space + await check_search_space_access(session, search_space_id, user) + + # Get or create user preference for this search space + preference = await get_or_create_user_preference( + session, user.id, search_space_id + ) + + # Validate that all provided LLM config IDs belong to the search space update_data = preferences.model_dump(exclude_unset=True) + # Store language from configs to validate consistency + languages = set() + for _key, llm_config_id in update_data.items(): if llm_config_id is not None: - # Verify ownership of the LLM config + # Verify the LLM config belongs to the search space result = await session.execute( select(LLMConfig).filter( - LLMConfig.id == llm_config_id, LLMConfig.user_id == user.id + LLMConfig.id == llm_config_id, + LLMConfig.search_space_id == search_space_id, ) ) llm_config = result.scalars().first() if not llm_config: raise HTTPException( status_code=404, - detail=f"LLM configuration {llm_config_id} not found or you don't have permission to access it", + detail=f"LLM configuration {llm_config_id} not found in this search space", ) + # Collect language for consistency check + languages.add(llm_config.language) + + # Check if all selected LLM configs have the same language + if len(languages) > 1: + raise HTTPException( + status_code=400, + detail="All selected LLM configurations must have the same language setting", + ) + # Update user preferences for key, value in update_data.items(): - setattr(user, key, value) + setattr(preference, key, value) await session.commit() - await session.refresh(user) + await session.refresh(preference) + + # Reload relationships + await session.refresh( + preference, ["long_context_llm", "fast_llm", "strategic_llm"] + ) # Return updated preferences - return await get_user_llm_preferences(session, user) + return { + "long_context_llm_id": preference.long_context_llm_id, + "fast_llm_id": preference.fast_llm_id, + "strategic_llm_id": preference.strategic_llm_id, + "long_context_llm": preference.long_context_llm, + "fast_llm": preference.fast_llm, + "strategic_llm": preference.strategic_llm, + } except HTTPException: raise except Exception as e: diff --git a/surfsense_backend/app/routes/luma_add_connector_route.py b/surfsense_backend/app/routes/luma_add_connector_route.py index 37fd0c3df..04d840a08 100644 --- a/surfsense_backend/app/routes/luma_add_connector_route.py +++ b/surfsense_backend/app/routes/luma_add_connector_route.py @@ -47,9 +47,10 @@ async def add_luma_connector( HTTPException: If connector already exists or validation fails """ try: - # Check if a Luma connector already exists for this user + # Check if a Luma connector already exists for this search space and user result = await session.execute( select(SearchSourceConnector).filter( + SearchSourceConnector.search_space_id == request.space_id, SearchSourceConnector.user_id == user.id, SearchSourceConnector.connector_type == SearchSourceConnectorType.LUMA_CONNECTOR, @@ -64,7 +65,9 @@ async def add_luma_connector( await session.commit() await session.refresh(existing_connector) - logger.info(f"Updated existing Luma connector for user {user.id}") + logger.info( + f"Updated existing Luma connector for user {user.id} in space {request.space_id}" + ) return { "message": "Luma connector updated successfully", @@ -77,6 +80,7 @@ async def add_luma_connector( name="Luma Event Connector", connector_type=SearchSourceConnectorType.LUMA_CONNECTOR, config={"api_key": request.api_key}, + search_space_id=request.space_id, user_id=user.id, is_indexable=True, ) @@ -113,13 +117,15 @@ async def add_luma_connector( @router.delete("/connectors/luma") async def delete_luma_connector( + space_id: int, user: User = Depends(current_active_user), session: AsyncSession = Depends(get_async_session), ): """ - Delete the Luma connector for the authenticated user. + Delete the Luma connector for the authenticated user in a specific search space. Args: + space_id: Search space ID user: Current authenticated user session: Database session @@ -132,6 +138,7 @@ async def delete_luma_connector( try: result = await session.execute( select(SearchSourceConnector).filter( + SearchSourceConnector.search_space_id == space_id, SearchSourceConnector.user_id == user.id, SearchSourceConnector.connector_type == SearchSourceConnectorType.LUMA_CONNECTOR, @@ -165,13 +172,15 @@ async def delete_luma_connector( @router.get("/connectors/luma/test") async def test_luma_connector( + space_id: int, user: User = Depends(current_active_user), session: AsyncSession = Depends(get_async_session), ): """ - Test the Luma connector for the authenticated user. + Test the Luma connector for the authenticated user in a specific search space. Args: + space_id: Search space ID user: Current authenticated user session: Database session @@ -182,9 +191,10 @@ async def test_luma_connector( HTTPException: If connector doesn't exist or test fails """ try: - # Get the Luma connector for this user + # Get the Luma connector for this search space and user result = await session.execute( select(SearchSourceConnector).filter( + SearchSourceConnector.search_space_id == space_id, SearchSourceConnector.user_id == user.id, SearchSourceConnector.connector_type == SearchSourceConnectorType.LUMA_CONNECTOR, diff --git a/surfsense_backend/app/routes/search_source_connectors_routes.py b/surfsense_backend/app/routes/search_source_connectors_routes.py index 6250e0c07..fc0f0b892 100644 --- a/surfsense_backend/app/routes/search_source_connectors_routes.py +++ b/surfsense_backend/app/routes/search_source_connectors_routes.py @@ -1,13 +1,13 @@ """ SearchSourceConnector routes for CRUD operations: POST /search-source-connectors/ - Create a new connector -GET /search-source-connectors/ - List all connectors for the current user +GET /search-source-connectors/ - List all connectors for the current user (optionally filtered by search space) GET /search-source-connectors/{connector_id} - Get a specific connector PUT /search-source-connectors/{connector_id} - Update a specific connector DELETE /search-source-connectors/{connector_id} - Delete a specific connector POST /search-source-connectors/{connector_id}/index - Index content from a connector to a search space -Note: Each user can have only one connector of each type (SERPER_API, TAVILY_API, SLACK_CONNECTOR, NOTION_CONNECTOR, GITHUB_CONNECTOR, LINEAR_CONNECTOR, DISCORD_CONNECTOR, LUMA_CONNECTOR). +Note: Each search space can have only one connector of each type per user (based on search_space_id, user_id, and connector_type). """ import logging @@ -93,19 +93,26 @@ async def list_github_repositories( @router.post("/search-source-connectors/", response_model=SearchSourceConnectorRead) async def create_search_source_connector( connector: SearchSourceConnectorCreate, + search_space_id: int = Query( + ..., description="ID of the search space to associate the connector with" + ), session: AsyncSession = Depends(get_async_session), user: User = Depends(current_active_user), ): """ Create a new search source connector. - Each user can have only one connector of each type (SERPER_API, TAVILY_API, SLACK_CONNECTOR, etc.). + Each search space can have only one connector of each type per user (based on search_space_id, user_id, and connector_type). The config must contain the appropriate keys for the connector type. """ try: - # Check if a connector with the same type already exists for this user + # Check if the search space belongs to the user + await check_ownership(session, SearchSpace, search_space_id, user) + + # Check if a connector with the same type already exists for this search space and user result = await session.execute( select(SearchSourceConnector).filter( + SearchSourceConnector.search_space_id == search_space_id, SearchSourceConnector.user_id == user.id, SearchSourceConnector.connector_type == connector.connector_type, ) @@ -114,9 +121,11 @@ async def create_search_source_connector( if existing_connector: raise HTTPException( status_code=409, - detail=f"A connector with type {connector.connector_type} already exists. Each user can have only one connector of each type.", + detail=f"A connector with type {connector.connector_type} already exists in this search space. Each search space can have only one connector of each type per user.", ) - db_connector = SearchSourceConnector(**connector.model_dump(), user_id=user.id) + db_connector = SearchSourceConnector( + **connector.model_dump(), search_space_id=search_space_id, user_id=user.id + ) session.add(db_connector) await session.commit() await session.refresh(db_connector) @@ -128,7 +137,7 @@ async def create_search_source_connector( await session.rollback() raise HTTPException( status_code=409, - detail=f"Integrity error: A connector with this type already exists. {e!s}", + detail=f"Integrity error: A connector with this type already exists in this search space. {e!s}", ) from e except HTTPException: await session.rollback() @@ -152,13 +161,19 @@ async def read_search_source_connectors( session: AsyncSession = Depends(get_async_session), user: User = Depends(current_active_user), ): - """List all search source connectors for the current user.""" + """List all search source connectors for the current user, optionally filtered by search space.""" try: query = select(SearchSourceConnector).filter( SearchSourceConnector.user_id == user.id ) - # No need to filter by search_space_id as connectors are user-owned, not search space specific + # Filter by search_space_id if provided + if search_space_id is not None: + # Verify the search space belongs to the user + await check_ownership(session, SearchSpace, search_space_id, user) + query = query.filter( + SearchSourceConnector.search_space_id == search_space_id + ) result = await session.execute(query.offset(skip).limit(limit)) return result.scalars().all() @@ -255,6 +270,8 @@ async def update_search_source_connector( if key == "connector_type" and value != db_connector.connector_type: result = await session.execute( select(SearchSourceConnector).filter( + SearchSourceConnector.search_space_id + == db_connector.search_space_id, SearchSourceConnector.user_id == user.id, SearchSourceConnector.connector_type == value, SearchSourceConnector.id != connector_id, @@ -264,7 +281,7 @@ async def update_search_source_connector( if existing_connector: raise HTTPException( status_code=409, - detail=f"A connector with type {value} already exists. Each user can have only one connector of each type.", + detail=f"A connector with type {value} already exists in this search space. Each search space can have only one connector of each type per user.", ) setattr(db_connector, key, value) diff --git a/surfsense_backend/app/schemas/llm_config.py b/surfsense_backend/app/schemas/llm_config.py index c3c003397..285c15665 100644 --- a/surfsense_backend/app/schemas/llm_config.py +++ b/surfsense_backend/app/schemas/llm_config.py @@ -1,4 +1,3 @@ -import uuid from datetime import datetime from typing import Any @@ -27,10 +26,15 @@ class LLMConfigBase(BaseModel): litellm_params: dict[str, Any] | None = Field( default=None, description="Additional LiteLLM parameters" ) + language: str | None = Field( + default="English", max_length=50, description="Language for the LLM" + ) class LLMConfigCreate(LLMConfigBase): - pass + search_space_id: int = Field( + ..., description="Search space ID to associate the LLM config with" + ) class LLMConfigUpdate(BaseModel): @@ -48,6 +52,9 @@ class LLMConfigUpdate(BaseModel): api_base: str | None = Field( None, max_length=500, description="Optional API base URL" ) + language: str | None = Field( + None, max_length=50, description="Language for the LLM" + ) litellm_params: dict[str, Any] | None = Field( None, description="Additional LiteLLM parameters" ) @@ -56,6 +63,6 @@ class LLMConfigUpdate(BaseModel): class LLMConfigRead(LLMConfigBase, IDModel, TimestampModel): id: int created_at: datetime - user_id: uuid.UUID + search_space_id: int model_config = ConfigDict(from_attributes=True) diff --git a/surfsense_backend/app/schemas/search_source_connector.py b/surfsense_backend/app/schemas/search_source_connector.py index aac6c415c..b84eeb07d 100644 --- a/surfsense_backend/app/schemas/search_source_connector.py +++ b/surfsense_backend/app/schemas/search_source_connector.py @@ -39,6 +39,7 @@ class SearchSourceConnectorUpdate(BaseModel): class SearchSourceConnectorRead(SearchSourceConnectorBase, IDModel, TimestampModel): + search_space_id: int user_id: uuid.UUID model_config = ConfigDict(from_attributes=True) diff --git a/surfsense_backend/app/services/connector_service.py b/surfsense_backend/app/services/connector_service.py index 938500a67..ca323f510 100644 --- a/surfsense_backend/app/services/connector_service.py +++ b/surfsense_backend/app/services/connector_service.py @@ -1,6 +1,8 @@ import asyncio from typing import Any +from urllib.parse import urljoin +import httpx from linkup import LinkupClient from sqlalchemy import func from sqlalchemy.ext.asyncio import AsyncSession @@ -236,28 +238,37 @@ class ConnectorService: return transformed_results async def get_connector_by_type( - self, user_id: str, connector_type: SearchSourceConnectorType + self, + user_id: str, + connector_type: SearchSourceConnectorType, + search_space_id: int | None = None, ) -> SearchSourceConnector | None: """ - Get a connector by type for a specific user + Get a connector by type for a specific user and optionally a search space Args: user_id: The user's ID connector_type: The connector type to retrieve + search_space_id: Optional search space ID to filter by Returns: Optional[SearchSourceConnector]: The connector if found, None otherwise """ - result = await self.session.execute( - select(SearchSourceConnector).filter( - SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type == connector_type, - ) + query = select(SearchSourceConnector).filter( + SearchSourceConnector.user_id == user_id, + SearchSourceConnector.connector_type == connector_type, ) + + if search_space_id is not None: + query = query.filter( + SearchSourceConnector.search_space_id == search_space_id + ) + + result = await self.session.execute(query) return result.scalars().first() async def search_tavily( - self, user_query: str, user_id: str, top_k: int = 20 + self, user_query: str, user_id: str, search_space_id: int, top_k: int = 20 ) -> tuple: """ Search using Tavily API and return both the source information and documents @@ -265,6 +276,7 @@ class ConnectorService: Args: user_query: The user's query user_id: The user's ID + search_space_id: The search space ID top_k: Maximum number of results to return Returns: @@ -272,7 +284,7 @@ class ConnectorService: """ # Get Tavily connector configuration tavily_connector = await self.get_connector_by_type( - user_id, SearchSourceConnectorType.TAVILY_API + user_id, SearchSourceConnectorType.TAVILY_API, search_space_id ) if not tavily_connector: @@ -362,6 +374,192 @@ class ConnectorService: "sources": [], }, [] + async def search_searxng( + self, + user_query: str, + user_id: str, + search_space_id: int, + top_k: int = 20, + ) -> tuple: + """ + Search using a configured SearxNG instance and return both sources and documents. + """ + searx_connector = await self.get_connector_by_type( + user_id, SearchSourceConnectorType.SEARXNG_API, search_space_id + ) + + if not searx_connector: + return { + "id": 11, + "name": "SearxNG Search", + "type": "SEARXNG_API", + "sources": [], + }, [] + + config = searx_connector.config or {} + host = config.get("SEARXNG_HOST") + + if not host: + print("SearxNG connector is missing SEARXNG_HOST configuration") + return { + "id": 11, + "name": "SearxNG Search", + "type": "SEARXNG_API", + "sources": [], + }, [] + + api_key = config.get("SEARXNG_API_KEY") + engines = config.get("SEARXNG_ENGINES") + categories = config.get("SEARXNG_CATEGORIES") + language = config.get("SEARXNG_LANGUAGE") + safesearch = config.get("SEARXNG_SAFESEARCH") + + def _parse_bool(value: Any, default: bool = True) -> bool: + if isinstance(value, bool): + return value + if isinstance(value, str): + lowered = value.strip().lower() + if lowered in {"true", "1", "yes", "on"}: + return True + if lowered in {"false", "0", "no", "off"}: + return False + return default + + verify_ssl = _parse_bool(config.get("SEARXNG_VERIFY_SSL", True)) + + safesearch_value: int | None = None + if isinstance(safesearch, str): + safesearch_clean = safesearch.strip() + if safesearch_clean.isdigit(): + safesearch_value = int(safesearch_clean) + elif isinstance(safesearch, int | float): + safesearch_value = int(safesearch) + + if safesearch_value is not None and not (0 <= safesearch_value <= 2): + safesearch_value = None + + def _format_list(value: Any) -> str | None: + if value is None: + return None + if isinstance(value, str): + value = value.strip() + return value or None + if isinstance(value, list | tuple | set): + cleaned = [str(item).strip() for item in value if str(item).strip()] + return ",".join(cleaned) if cleaned else None + return str(value) + + params: dict[str, Any] = { + "q": user_query, + "format": "json", + "language": language or "", + "limit": max(1, min(top_k, 50)), + } + + engines_param = _format_list(engines) + if engines_param: + params["engines"] = engines_param + + categories_param = _format_list(categories) + if categories_param: + params["categories"] = categories_param + + if safesearch_value is not None: + params["safesearch"] = safesearch_value + + if not params.get("language"): + params.pop("language") + + headers = {"Accept": "application/json"} + if api_key: + headers["X-API-KEY"] = api_key + + searx_endpoint = urljoin(host if host.endswith("/") else f"{host}/", "search") + + try: + async with httpx.AsyncClient(timeout=20.0, verify=verify_ssl) as client: + response = await client.get( + searx_endpoint, + params=params, + headers=headers, + ) + response.raise_for_status() + except httpx.HTTPError as exc: + print(f"Error searching with SearxNG: {exc!s}") + return { + "id": 11, + "name": "SearxNG Search", + "type": "SEARXNG_API", + "sources": [], + }, [] + + try: + data = response.json() + except ValueError: + print("Failed to decode JSON response from SearxNG") + return { + "id": 11, + "name": "SearxNG Search", + "type": "SEARXNG_API", + "sources": [], + }, [] + + searx_results = data.get("results", []) + if not searx_results: + return { + "id": 11, + "name": "SearxNG Search", + "type": "SEARXNG_API", + "sources": [], + }, [] + + sources_list: list[dict[str, Any]] = [] + documents: list[dict[str, Any]] = [] + + async with self.counter_lock: + for result in searx_results: + description = result.get("content") or result.get("snippet") or "" + if len(description) > 160: + description = f"{description[:157]}..." + + source = { + "id": self.source_id_counter, + "title": result.get("title", "SearxNG Result"), + "description": description, + "url": result.get("url", ""), + } + sources_list.append(source) + + metadata = { + "url": result.get("url", ""), + "engines": result.get("engines", []), + "category": result.get("category"), + "source": "SEARXNG_API", + } + + document = { + "chunk_id": self.source_id_counter, + "content": description or result.get("content", ""), + "score": result.get("score", 0.0), + "document": { + "id": self.source_id_counter, + "title": result.get("title", "SearxNG Result"), + "document_type": "SEARXNG_API", + "metadata": metadata, + }, + } + documents.append(document) + self.source_id_counter += 1 + + result_object = { + "id": 11, + "name": "SearxNG Search", + "type": "SEARXNG_API", + "sources": sources_list, + } + + return result_object, documents + async def search_slack( self, user_query: str, @@ -1637,7 +1835,11 @@ class ConnectorService: return result_object, clickup_chunks async def search_linkup( - self, user_query: str, user_id: str, mode: str = "standard" + self, + user_query: str, + user_id: str, + search_space_id: int, + mode: str = "standard", ) -> tuple: """ Search using Linkup API and return both the source information and documents @@ -1645,6 +1847,7 @@ class ConnectorService: Args: user_query: The user's query user_id: The user's ID + search_space_id: The search space ID mode: Search depth mode, can be "standard" or "deep" Returns: @@ -1652,7 +1855,7 @@ class ConnectorService: """ # Get Linkup connector configuration linkup_connector = await self.get_connector_by_type( - user_id, SearchSourceConnectorType.LINKUP_API + user_id, SearchSourceConnectorType.LINKUP_API, search_space_id ) if not linkup_connector: diff --git a/surfsense_backend/app/services/llm_service.py b/surfsense_backend/app/services/llm_service.py index 9135e49dc..e322eb401 100644 --- a/surfsense_backend/app/services/llm_service.py +++ b/surfsense_backend/app/services/llm_service.py @@ -1,10 +1,14 @@ import logging +import litellm from langchain_litellm import ChatLiteLLM from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select -from app.db import LLMConfig, User +from app.db import LLMConfig, UserSearchSpacePreference + +# Configure litellm to automatically drop unsupported parameters +litellm.drop_params = True logger = logging.getLogger(__name__) @@ -16,61 +20,74 @@ class LLMRole: async def get_user_llm_instance( - session: AsyncSession, user_id: str, role: str + session: AsyncSession, user_id: str, search_space_id: int, role: str ) -> ChatLiteLLM | None: """ - Get a ChatLiteLLM instance for a specific user and role. + Get a ChatLiteLLM instance for a specific user, search space, and role. Args: session: Database session user_id: User ID + search_space_id: Search Space ID role: LLM role ('long_context', 'fast', or 'strategic') Returns: ChatLiteLLM instance or None if not found """ try: - # Get user with their LLM preferences - result = await session.execute(select(User).where(User.id == user_id)) - user = result.scalars().first() + # Get user's LLM preferences for this search space + result = await session.execute( + select(UserSearchSpacePreference).where( + UserSearchSpacePreference.user_id == user_id, + UserSearchSpacePreference.search_space_id == search_space_id, + ) + ) + preference = result.scalars().first() - if not user: - logger.error(f"User {user_id} not found") + if not preference: + logger.error( + f"No LLM preferences found for user {user_id} in search space {search_space_id}" + ) return None # Get the appropriate LLM config ID based on role llm_config_id = None if role == LLMRole.LONG_CONTEXT: - llm_config_id = user.long_context_llm_id + llm_config_id = preference.long_context_llm_id elif role == LLMRole.FAST: - llm_config_id = user.fast_llm_id + llm_config_id = preference.fast_llm_id elif role == LLMRole.STRATEGIC: - llm_config_id = user.strategic_llm_id + llm_config_id = preference.strategic_llm_id else: logger.error(f"Invalid LLM role: {role}") return None if not llm_config_id: - logger.error(f"No {role} LLM configured for user {user_id}") + logger.error( + f"No {role} LLM configured for user {user_id} in search space {search_space_id}" + ) return None # Get the LLM configuration result = await session.execute( select(LLMConfig).where( - LLMConfig.id == llm_config_id, LLMConfig.user_id == user_id + LLMConfig.id == llm_config_id, + LLMConfig.search_space_id == search_space_id, ) ) llm_config = result.scalars().first() if not llm_config: - logger.error(f"LLM config {llm_config_id} not found for user {user_id}") + logger.error( + f"LLM config {llm_config_id} not found in search space {search_space_id}" + ) return None - # Build the model string for litellm + # Build the model string for litellm / 构建 LiteLLM 的模型字符串 if llm_config.custom_provider: model_string = f"{llm_config.custom_provider}/{llm_config.model_name}" else: - # Map provider enum to litellm format + # Map provider enum to litellm format / 将提供商枚举映射为 LiteLLM 格式 provider_map = { "OPENAI": "openai", "ANTHROPIC": "anthropic", @@ -82,6 +99,11 @@ async def get_user_llm_instance( "AZURE_OPENAI": "azure", "OPENROUTER": "openrouter", "COMETAPI": "cometapi", + # Chinese LLM providers (OpenAI-compatible) + "DEEPSEEK": "openai", # DeepSeek uses OpenAI-compatible API + "ALIBABA_QWEN": "openai", # Qwen uses OpenAI-compatible API + "MOONSHOT": "openai", # Moonshot (Kimi) uses OpenAI-compatible API + "ZHIPU": "openai", # Zhipu (GLM) uses OpenAI-compatible API # Add more mappings as needed } provider_prefix = provider_map.get( @@ -113,19 +135,25 @@ async def get_user_llm_instance( async def get_user_long_context_llm( - session: AsyncSession, user_id: str + session: AsyncSession, user_id: str, search_space_id: int ) -> ChatLiteLLM | None: - """Get user's long context LLM instance.""" - return await get_user_llm_instance(session, user_id, LLMRole.LONG_CONTEXT) + """Get user's long context LLM instance for a specific search space.""" + return await get_user_llm_instance( + session, user_id, search_space_id, LLMRole.LONG_CONTEXT + ) -async def get_user_fast_llm(session: AsyncSession, user_id: str) -> ChatLiteLLM | None: - """Get user's fast LLM instance.""" - return await get_user_llm_instance(session, user_id, LLMRole.FAST) +async def get_user_fast_llm( + session: AsyncSession, user_id: str, search_space_id: int +) -> ChatLiteLLM | None: + """Get user's fast LLM instance for a specific search space.""" + return await get_user_llm_instance(session, user_id, search_space_id, LLMRole.FAST) async def get_user_strategic_llm( - session: AsyncSession, user_id: str + session: AsyncSession, user_id: str, search_space_id: int ) -> ChatLiteLLM | None: - """Get user's strategic LLM instance.""" - return await get_user_llm_instance(session, user_id, LLMRole.STRATEGIC) + """Get user's strategic LLM instance for a specific search space.""" + return await get_user_llm_instance( + session, user_id, search_space_id, LLMRole.STRATEGIC + ) diff --git a/surfsense_backend/app/services/query_service.py b/surfsense_backend/app/services/query_service.py index 4a4bc59be..d2759ab27 100644 --- a/surfsense_backend/app/services/query_service.py +++ b/surfsense_backend/app/services/query_service.py @@ -17,6 +17,7 @@ class QueryService: user_query: str, session: AsyncSession, user_id: str, + search_space_id: int, chat_history_str: str | None = None, ) -> str: """ @@ -27,6 +28,7 @@ class QueryService: user_query: The original user query session: Database session for accessing user LLM configs user_id: User ID to get their specific LLM configuration + search_space_id: Search Space ID to get user's LLM preferences chat_history_str: Optional chat history string Returns: @@ -37,10 +39,10 @@ class QueryService: try: # Get the user's strategic LLM instance - llm = await get_user_strategic_llm(session, user_id) + llm = await get_user_strategic_llm(session, user_id, search_space_id) if not llm: print( - f"Warning: No strategic LLM configured for user {user_id}. Using original query." + f"Warning: No strategic LLM configured for user {user_id} in search space {search_space_id}. Using original query." ) return user_query diff --git a/surfsense_backend/app/services/task_logging_service.py b/surfsense_backend/app/services/task_logging_service.py index 39316b71f..6ba9d0432 100644 --- a/surfsense_backend/app/services/task_logging_service.py +++ b/surfsense_backend/app/services/task_logging_service.py @@ -1,3 +1,4 @@ +import contextlib import logging from datetime import datetime from typing import Any @@ -73,6 +74,14 @@ class TaskLoggingService: Returns: Log: The updated log entry """ + # Ensure session is in a valid state + if not self.session.is_active: + await self.session.rollback() + + # Refresh log_entry to avoid expired state + with contextlib.suppress(Exception): + await self.session.refresh(log_entry) + # Update the existing log entry log_entry.status = LogStatus.SUCCESS log_entry.message = message @@ -114,6 +123,14 @@ class TaskLoggingService: Returns: Log: The updated log entry """ + # Ensure session is in a valid state + if not self.session.is_active: + await self.session.rollback() + + # Refresh log_entry to avoid expired state + with contextlib.suppress(Exception): + await self.session.refresh(log_entry) + # Update the existing log entry log_entry.status = LogStatus.FAILED log_entry.level = LogLevel.ERROR @@ -161,6 +178,14 @@ class TaskLoggingService: Returns: Log: The updated log entry """ + # Ensure session is in a valid state + if not self.session.is_active: + await self.session.rollback() + + # Refresh log_entry to avoid expired state + with contextlib.suppress(Exception): + await self.session.refresh(log_entry) + log_entry.message = progress_message if progress_metadata: diff --git a/surfsense_backend/app/tasks/connector_indexers/airtable_indexer.py b/surfsense_backend/app/tasks/connector_indexers/airtable_indexer.py index 7ba2f2d44..0cc21bb47 100644 --- a/surfsense_backend/app/tasks/connector_indexers/airtable_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/airtable_indexer.py @@ -260,7 +260,9 @@ async def index_airtable_records( continue # Generate document summary - user_llm = await get_user_long_context_llm(session, user_id) + user_llm = await get_user_long_context_llm( + session, user_id, search_space_id + ) if user_llm: document_metadata = { diff --git a/surfsense_backend/app/tasks/connector_indexers/clickup_indexer.py b/surfsense_backend/app/tasks/connector_indexers/clickup_indexer.py index 3120fcbc7..5ee7342fa 100644 --- a/surfsense_backend/app/tasks/connector_indexers/clickup_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/clickup_indexer.py @@ -222,7 +222,9 @@ async def index_clickup_tasks( continue # Generate summary with metadata - user_llm = await get_user_long_context_llm(session, user_id) + user_llm = await get_user_long_context_llm( + session, user_id, search_space_id + ) if user_llm: document_metadata = { diff --git a/surfsense_backend/app/tasks/connector_indexers/confluence_indexer.py b/surfsense_backend/app/tasks/connector_indexers/confluence_indexer.py index 625992922..28cb3b1f4 100644 --- a/surfsense_backend/app/tasks/connector_indexers/confluence_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/confluence_indexer.py @@ -233,7 +233,9 @@ async def index_confluence_pages( continue # Generate summary with metadata - user_llm = await get_user_long_context_llm(session, user_id) + user_llm = await get_user_long_context_llm( + session, user_id, search_space_id + ) comment_count = len(comments) if user_llm: diff --git a/surfsense_backend/app/tasks/connector_indexers/discord_indexer.py b/surfsense_backend/app/tasks/connector_indexers/discord_indexer.py index c538f12f7..08c995f64 100644 --- a/surfsense_backend/app/tasks/connector_indexers/discord_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/discord_indexer.py @@ -325,7 +325,9 @@ async def index_discord_messages( continue # Get user's long context LLM - user_llm = await get_user_long_context_llm(session, user_id) + user_llm = await get_user_long_context_llm( + session, user_id, search_space_id + ) if not user_llm: logger.error( f"No long context LLM configured for user {user_id}" diff --git a/surfsense_backend/app/tasks/connector_indexers/github_indexer.py b/surfsense_backend/app/tasks/connector_indexers/github_indexer.py index ba01e3979..9cc0c0993 100644 --- a/surfsense_backend/app/tasks/connector_indexers/github_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/github_indexer.py @@ -213,7 +213,9 @@ async def index_github_repos( continue # Generate summary with metadata - user_llm = await get_user_long_context_llm(session, user_id) + user_llm = await get_user_long_context_llm( + session, user_id, search_space_id + ) if user_llm: # Extract file extension from file path file_extension = ( diff --git a/surfsense_backend/app/tasks/connector_indexers/google_calendar_indexer.py b/surfsense_backend/app/tasks/connector_indexers/google_calendar_indexer.py index abc492581..be5169612 100644 --- a/surfsense_backend/app/tasks/connector_indexers/google_calendar_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/google_calendar_indexer.py @@ -113,7 +113,10 @@ async def index_google_calendar_events( ) calendar_client = GoogleCalendarConnector( - credentials=credentials, session=session, user_id=user_id + credentials=credentials, + session=session, + user_id=user_id, + connector_id=connector_id, ) # Calculate date range @@ -263,7 +266,9 @@ async def index_google_calendar_events( continue # Generate summary with metadata - user_llm = await get_user_long_context_llm(session, user_id) + user_llm = await get_user_long_context_llm( + session, user_id, search_space_id + ) if user_llm: document_metadata = { diff --git a/surfsense_backend/app/tasks/connector_indexers/google_gmail_indexer.py b/surfsense_backend/app/tasks/connector_indexers/google_gmail_indexer.py index 531cfca0e..872e19d03 100644 --- a/surfsense_backend/app/tasks/connector_indexers/google_gmail_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/google_gmail_indexer.py @@ -127,7 +127,9 @@ async def index_google_gmail_messages( ) # Initialize Google gmail connector - gmail_connector = GoogleGmailConnector(credentials, session, user_id) + gmail_connector = GoogleGmailConnector( + credentials, session, user_id, connector_id + ) # Fetch recent Google gmail messages logger.info(f"Fetching recent emails for connector {connector_id}") @@ -208,7 +210,9 @@ async def index_google_gmail_messages( continue # Generate summary with metadata - user_llm = await get_user_long_context_llm(session, user_id) + user_llm = await get_user_long_context_llm( + session, user_id, search_space_id + ) if user_llm: document_metadata = { diff --git a/surfsense_backend/app/tasks/connector_indexers/jira_indexer.py b/surfsense_backend/app/tasks/connector_indexers/jira_indexer.py index c199faed0..e9d556954 100644 --- a/surfsense_backend/app/tasks/connector_indexers/jira_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/jira_indexer.py @@ -216,7 +216,9 @@ async def index_jira_issues( continue # Generate summary with metadata - user_llm = await get_user_long_context_llm(session, user_id) + user_llm = await get_user_long_context_llm( + session, user_id, search_space_id + ) comment_count = len(formatted_issue.get("comments", [])) if user_llm: diff --git a/surfsense_backend/app/tasks/connector_indexers/linear_indexer.py b/surfsense_backend/app/tasks/connector_indexers/linear_indexer.py index 6ca145357..aca1e2040 100644 --- a/surfsense_backend/app/tasks/connector_indexers/linear_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/linear_indexer.py @@ -228,7 +228,9 @@ async def index_linear_issues( continue # Generate summary with metadata - user_llm = await get_user_long_context_llm(session, user_id) + user_llm = await get_user_long_context_llm( + session, user_id, search_space_id + ) state = formatted_issue.get("state", "Unknown") description = formatted_issue.get("description", "") comment_count = len(formatted_issue.get("comments", [])) diff --git a/surfsense_backend/app/tasks/connector_indexers/luma_indexer.py b/surfsense_backend/app/tasks/connector_indexers/luma_indexer.py index d7b6d3058..3d8970654 100644 --- a/surfsense_backend/app/tasks/connector_indexers/luma_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/luma_indexer.py @@ -270,7 +270,9 @@ async def index_luma_events( continue # Generate summary with metadata - user_llm = await get_user_long_context_llm(session, user_id) + user_llm = await get_user_long_context_llm( + session, user_id, search_space_id + ) if user_llm: document_metadata = { diff --git a/surfsense_backend/app/tasks/connector_indexers/notion_indexer.py b/surfsense_backend/app/tasks/connector_indexers/notion_indexer.py index a6c8853a3..b290f86da 100644 --- a/surfsense_backend/app/tasks/connector_indexers/notion_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/notion_indexer.py @@ -299,7 +299,9 @@ async def index_notion_pages( continue # Get user's long context LLM - user_llm = await get_user_long_context_llm(session, user_id) + user_llm = await get_user_long_context_llm( + session, user_id, search_space_id + ) if not user_llm: logger.error(f"No long context LLM configured for user {user_id}") skipped_pages.append(f"{page_title} (no LLM configured)") diff --git a/surfsense_backend/app/tasks/document_processors/extension_processor.py b/surfsense_backend/app/tasks/document_processors/extension_processor.py index 8f8433148..ed25b8fbd 100644 --- a/surfsense_backend/app/tasks/document_processors/extension_processor.py +++ b/surfsense_backend/app/tasks/document_processors/extension_processor.py @@ -104,9 +104,11 @@ async def add_extension_received_document( return existing_document # Get user's long context LLM - user_llm = await get_user_long_context_llm(session, user_id) + user_llm = await get_user_long_context_llm(session, user_id, search_space_id) if not user_llm: - raise RuntimeError(f"No long context LLM configured for user {user_id}") + raise RuntimeError( + f"No long context LLM configured for user {user_id} in search space {search_space_id}" + ) # Generate summary with metadata document_metadata = { diff --git a/surfsense_backend/app/tasks/document_processors/file_processors.py b/surfsense_backend/app/tasks/document_processors/file_processors.py index 3803f4b2b..573b2c28c 100644 --- a/surfsense_backend/app/tasks/document_processors/file_processors.py +++ b/surfsense_backend/app/tasks/document_processors/file_processors.py @@ -60,9 +60,11 @@ async def add_received_file_document_using_unstructured( # TODO: Check if file_markdown exceeds token limit of embedding model # Get user's long context LLM - user_llm = await get_user_long_context_llm(session, user_id) + user_llm = await get_user_long_context_llm(session, user_id, search_space_id) if not user_llm: - raise RuntimeError(f"No long context LLM configured for user {user_id}") + raise RuntimeError( + f"No long context LLM configured for user {user_id} in search space {search_space_id}" + ) # Generate summary with metadata document_metadata = { @@ -140,9 +142,11 @@ async def add_received_file_document_using_llamacloud( return existing_document # Get user's long context LLM - user_llm = await get_user_long_context_llm(session, user_id) + user_llm = await get_user_long_context_llm(session, user_id, search_space_id) if not user_llm: - raise RuntimeError(f"No long context LLM configured for user {user_id}") + raise RuntimeError( + f"No long context LLM configured for user {user_id} in search space {search_space_id}" + ) # Generate summary with metadata document_metadata = { @@ -221,9 +225,11 @@ async def add_received_file_document_using_docling( return existing_document # Get user's long context LLM - user_llm = await get_user_long_context_llm(session, user_id) + user_llm = await get_user_long_context_llm(session, user_id, search_space_id) if not user_llm: - raise RuntimeError(f"No long context LLM configured for user {user_id}") + raise RuntimeError( + f"No long context LLM configured for user {user_id} in search space {search_space_id}" + ) # Generate summary using chunked processing for large documents from app.services.docling_service import create_docling_service diff --git a/surfsense_backend/app/tasks/document_processors/markdown_processor.py b/surfsense_backend/app/tasks/document_processors/markdown_processor.py index 493b046af..fa3c79d81 100644 --- a/surfsense_backend/app/tasks/document_processors/markdown_processor.py +++ b/surfsense_backend/app/tasks/document_processors/markdown_processor.py @@ -75,9 +75,11 @@ async def add_received_markdown_file_document( return existing_document # Get user's long context LLM - user_llm = await get_user_long_context_llm(session, user_id) + user_llm = await get_user_long_context_llm(session, user_id, search_space_id) if not user_llm: - raise RuntimeError(f"No long context LLM configured for user {user_id}") + raise RuntimeError( + f"No long context LLM configured for user {user_id} in search space {search_space_id}" + ) # Generate summary with metadata document_metadata = { diff --git a/surfsense_backend/app/tasks/document_processors/url_crawler.py b/surfsense_backend/app/tasks/document_processors/url_crawler.py index eddcda388..682086112 100644 --- a/surfsense_backend/app/tasks/document_processors/url_crawler.py +++ b/surfsense_backend/app/tasks/document_processors/url_crawler.py @@ -161,9 +161,11 @@ async def add_crawled_url_document( ) # Get user's long context LLM - user_llm = await get_user_long_context_llm(session, user_id) + user_llm = await get_user_long_context_llm(session, user_id, search_space_id) if not user_llm: - raise RuntimeError(f"No long context LLM configured for user {user_id}") + raise RuntimeError( + f"No long context LLM configured for user {user_id} in search space {search_space_id}" + ) # Generate summary await task_logger.log_task_progress( diff --git a/surfsense_backend/app/tasks/document_processors/youtube_processor.py b/surfsense_backend/app/tasks/document_processors/youtube_processor.py index 37981f6ae..a28a7f186 100644 --- a/surfsense_backend/app/tasks/document_processors/youtube_processor.py +++ b/surfsense_backend/app/tasks/document_processors/youtube_processor.py @@ -234,9 +234,11 @@ async def add_youtube_video_document( ) # Get user's long context LLM - user_llm = await get_user_long_context_llm(session, user_id) + user_llm = await get_user_long_context_llm(session, user_id, search_space_id) if not user_llm: - raise RuntimeError(f"No long context LLM configured for user {user_id}") + raise RuntimeError( + f"No long context LLM configured for user {user_id} in search space {search_space_id}" + ) # Generate summary await task_logger.log_task_progress( diff --git a/surfsense_backend/app/tasks/podcast_tasks.py b/surfsense_backend/app/tasks/podcast_tasks.py index 312ae5bc3..e5f828ef2 100644 --- a/surfsense_backend/app/tasks/podcast_tasks.py +++ b/surfsense_backend/app/tasks/podcast_tasks.py @@ -98,6 +98,7 @@ async def generate_chat_podcast( "configurable": { "podcast_title": "SurfSense", "user_id": str(user_id), + "search_space_id": search_space_id, } } # Initialize state with database session and streaming service diff --git a/surfsense_backend/app/tasks/stream_connector_search_results.py b/surfsense_backend/app/tasks/stream_connector_search_results.py index ead6a89e7..dd1ae4ce5 100644 --- a/surfsense_backend/app/tasks/stream_connector_search_results.py +++ b/surfsense_backend/app/tasks/stream_connector_search_results.py @@ -20,6 +20,7 @@ async def stream_connector_search_results( langchain_chat_history: list[Any], search_mode_str: str, document_ids_to_add_in_context: list[int], + language: str | None = None, ) -> AsyncGenerator[str, None]: """ Stream connector search results to the client @@ -66,8 +67,10 @@ async def stream_connector_search_results( "search_mode": search_mode, "research_mode": research_mode, "document_ids_to_add_in_context": document_ids_to_add_in_context, + "language": language, # Add language to the configuration } } + # print(f"Researcher configuration: {config['configurable']}") # Debug print # Initialize state with database session and streaming service initial_state = State( db_session=session, diff --git a/surfsense_backend/app/utils/validators.py b/surfsense_backend/app/utils/validators.py index 32b406b9d..0d6bd5341 100644 --- a/surfsense_backend/app/utils/validators.py +++ b/surfsense_backend/app/utils/validators.py @@ -16,89 +16,80 @@ from fastapi import HTTPException def validate_search_space_id(search_space_id: Any) -> int: """ Validate and convert search_space_id to integer. - + Args: search_space_id: The search space ID to validate - + Returns: int: Validated search space ID - + Raises: HTTPException: If validation fails """ if search_space_id is None: - raise HTTPException( - status_code=400, - detail="search_space_id is required" - ) - + raise HTTPException(status_code=400, detail="search_space_id is required") + if isinstance(search_space_id, bool): raise HTTPException( - status_code=400, - detail="search_space_id must be an integer, not a boolean" + status_code=400, detail="search_space_id must be an integer, not a boolean" ) - + if isinstance(search_space_id, int): if search_space_id <= 0: raise HTTPException( - status_code=400, - detail="search_space_id must be a positive integer" - + status_code=400, detail="search_space_id must be a positive integer" ) return search_space_id - + if isinstance(search_space_id, str): # Check if it's a valid integer string if not search_space_id.strip(): raise HTTPException( - status_code=400, - detail="search_space_id cannot be empty" + status_code=400, detail="search_space_id cannot be empty" ) - + # Check for valid integer format (no leading zeros, no decimal points) - if not re.match(r'^[1-9]\d*$', search_space_id.strip()): + if not re.match(r"^[1-9]\d*$", search_space_id.strip()): raise HTTPException( status_code=400, - detail="search_space_id must be a valid positive integer" + detail="search_space_id must be a valid positive integer", ) - + value = int(search_space_id.strip()) # Regex already guarantees value > 0, but check retained for clarity if value <= 0: raise HTTPException( - status_code=400, - detail="search_space_id must be a positive integer" + status_code=400, detail="search_space_id must be a positive integer" ) return value - + raise HTTPException( status_code=400, - detail="search_space_id must be an integer or string representation of an integer" + detail="search_space_id must be an integer or string representation of an integer", ) def validate_document_ids(document_ids: Any) -> list[int]: """ Validate and convert document_ids to list of integers. - + Args: document_ids: The document IDs to validate - + Returns: List[int]: Validated list of document IDs - + Raises: HTTPException: If validation fails """ if document_ids is None: return [] - + if not isinstance(document_ids, list): raise HTTPException( - status_code=400, - detail="document_ids_to_add_in_context must be a list" + status_code=400, detail="document_ids_to_add_in_context must be a list" ) - + validated_ids = [] for i, doc_id in enumerate(document_ids): if isinstance(doc_id, bool): @@ -111,119 +102,110 @@ def validate_document_ids(document_ids: Any) -> list[int]: if doc_id <= 0: raise HTTPException( status_code=400, - detail=f"document_ids_to_add_in_context[{i}] must be a positive integer" + detail=f"document_ids_to_add_in_context[{i}] must be a positive integer", ) validated_ids.append(doc_id) elif isinstance(doc_id, str): if not doc_id.strip(): raise HTTPException( status_code=400, - detail=f"document_ids_to_add_in_context[{i}] cannot be empty" + detail=f"document_ids_to_add_in_context[{i}] cannot be empty", ) - - if not re.match(r'^[1-9]\d*$', doc_id.strip()): + + if not re.match(r"^[1-9]\d*$", doc_id.strip()): raise HTTPException( status_code=400, - detail=f"document_ids_to_add_in_context[{i}] must be a valid positive integer" + detail=f"document_ids_to_add_in_context[{i}] must be a valid positive integer", ) - + value = int(doc_id.strip()) # Regex already guarantees value > 0 if value <= 0: raise HTTPException( status_code=400, - detail=f"document_ids_to_add_in_context[{i}] must be a positive integer" + detail=f"document_ids_to_add_in_context[{i}] must be a positive integer", ) validated_ids.append(value) else: raise HTTPException( status_code=400, - detail=f"document_ids_to_add_in_context[{i}] must be an integer or string representation of an integer" + detail=f"document_ids_to_add_in_context[{i}] must be an integer or string representation of an integer", ) - + return validated_ids def validate_connectors(connectors: Any) -> list[str]: """ Validate selected_connectors list. - + Args: connectors: The connectors to validate - + Returns: List[str]: Validated list of connector names - + Raises: HTTPException: If validation fails """ if connectors is None: return [] - + if not isinstance(connectors, list): raise HTTPException( - status_code=400, - detail="selected_connectors must be a list" + status_code=400, detail="selected_connectors must be a list" ) - + validated_connectors = [] for i, connector in enumerate(connectors): if not isinstance(connector, str): raise HTTPException( - status_code=400, - detail=f"selected_connectors[{i}] must be a string" + status_code=400, detail=f"selected_connectors[{i}] must be a string" ) - + if not connector.strip(): raise HTTPException( - status_code=400, - detail=f"selected_connectors[{i}] cannot be empty" + status_code=400, detail=f"selected_connectors[{i}] cannot be empty" ) - + trimmed = connector.strip() - if not re.fullmatch(r'[\w\-_]+', trimmed): + if not re.fullmatch(r"[\w\-_]+", trimmed): raise HTTPException( status_code=400, - detail=f"selected_connectors[{i}] contains invalid characters" + detail=f"selected_connectors[{i}] contains invalid characters", ) validated_connectors.append(trimmed) - + return validated_connectors def validate_research_mode(research_mode: Any) -> str: """ Validate research_mode parameter. - + Args: research_mode: The research mode to validate - + Returns: str: Validated research mode - + Raises: HTTPException: If validation fails """ if research_mode is None: return "QNA" # Default value - + if not isinstance(research_mode, str): - raise HTTPException( - status_code=400, - detail="research_mode must be a string" - ) + raise HTTPException(status_code=400, detail="research_mode must be a string") normalized_mode = research_mode.strip().upper() if not normalized_mode: - raise HTTPException( - status_code=400, - detail="research_mode cannot be empty" - ) + raise HTTPException(status_code=400, detail="research_mode cannot be empty") valid_modes = ["REPORT_GENERAL", "REPORT_DEEP", "REPORT_DEEPER", "QNA"] if normalized_mode not in valid_modes: raise HTTPException( status_code=400, - detail=f"research_mode must be one of: {', '.join(valid_modes)}" + detail=f"research_mode must be one of: {', '.join(valid_modes)}", ) return normalized_mode @@ -231,36 +213,30 @@ def validate_research_mode(research_mode: Any) -> str: def validate_search_mode(search_mode: Any) -> str: """ Validate search_mode parameter. - + Args: search_mode: The search mode to validate - + Returns: str: Validated search mode - + Raises: HTTPException: If validation fails """ if search_mode is None: return "CHUNKS" # Default value - + if not isinstance(search_mode, str): - raise HTTPException( - status_code=400, - detail="search_mode must be a string" - ) + raise HTTPException(status_code=400, detail="search_mode must be a string") normalized_mode = search_mode.strip().upper() if not normalized_mode: - raise HTTPException( - status_code=400, - detail="search_mode cannot be empty" - ) + raise HTTPException(status_code=400, detail="search_mode cannot be empty") valid_modes = ["CHUNKS", "DOCUMENTS"] if normalized_mode not in valid_modes: raise HTTPException( status_code=400, - detail=f"search_mode must be one of: {', '.join(valid_modes)}" + detail=f"search_mode must be one of: {', '.join(valid_modes)}", ) return normalized_mode @@ -268,185 +244,155 @@ def validate_search_mode(search_mode: Any) -> str: def validate_messages(messages: Any) -> list[dict]: """ Validate messages structure. - + Args: messages: The messages to validate - + Returns: List[dict]: Validated messages - + Raises: HTTPException: If validation fails """ if not isinstance(messages, list): - raise HTTPException( - status_code=400, - detail="messages must be a list" - ) - + raise HTTPException(status_code=400, detail="messages must be a list") + if not messages: - raise HTTPException( - status_code=400, - detail="messages cannot be empty" - ) - + raise HTTPException(status_code=400, detail="messages cannot be empty") + validated_messages = [] for i, message in enumerate(messages): if not isinstance(message, dict): raise HTTPException( - status_code=400, - detail=f"messages[{i}] must be a dictionary" + status_code=400, detail=f"messages[{i}] must be a dictionary" ) - + if "role" not in message: raise HTTPException( - status_code=400, - detail=f"messages[{i}] must have a 'role' field" + status_code=400, detail=f"messages[{i}] must have a 'role' field" ) - + if "content" not in message: raise HTTPException( - status_code=400, - detail=f"messages[{i}] must have a 'content' field" + status_code=400, detail=f"messages[{i}] must have a 'content' field" ) - + role = message["role"] if not isinstance(role, str) or role not in ["user", "assistant", "system"]: raise HTTPException( status_code=400, - detail=f"messages[{i}].role must be 'user', 'assistant', or 'system'" + detail=f"messages[{i}].role must be 'user', 'assistant', or 'system'", ) - + content = message["content"] if not isinstance(content, str): raise HTTPException( - status_code=400, - detail=f"messages[{i}].content must be a string" + status_code=400, detail=f"messages[{i}].content must be a string" ) - + if not content.strip(): raise HTTPException( - status_code=400, - detail=f"messages[{i}].content cannot be empty" + status_code=400, detail=f"messages[{i}].content cannot be empty" ) - + # Trim content and enforce max length (10,000 chars) sanitized_content = content.strip() if len(sanitized_content) > 10000: # Reasonable limit raise HTTPException( status_code=400, - detail=f"messages[{i}].content is too long (max 10000 characters)" + detail=f"messages[{i}].content is too long (max 10000 characters)", ) - - validated_messages.append({ - "role": role, - "content": sanitized_content - }) - + + validated_messages.append({"role": role, "content": sanitized_content}) + return validated_messages def validate_email(email: str) -> str: """ Validate email address using pyvalidators library. - + Args: email: The email address to validate - + Returns: str: Validated email address - + Raises: HTTPException: If validation fails """ if not email or not email.strip(): - raise HTTPException( - status_code=400, - detail="Email address is required" - ) - + raise HTTPException(status_code=400, detail="Email address is required") + email = email.strip() - + if not validators.email(email): - raise HTTPException( - status_code=400, - detail="Invalid email address format" - ) - + raise HTTPException(status_code=400, detail="Invalid email address format") + return email def validate_url(url: str) -> str: """ Validate URL using pyvalidators library. - + Args: url: The URL to validate - + Returns: str: Validated URL - + Raises: HTTPException: If validation fails """ if not url or not url.strip(): - raise HTTPException( - status_code=400, - detail="URL is required" - ) - + raise HTTPException(status_code=400, detail="URL is required") + url = url.strip() - + if not validators.url(url): - raise HTTPException( - status_code=400, - detail="Invalid URL format" - ) - + raise HTTPException(status_code=400, detail="Invalid URL format") + return url def validate_uuid(uuid_string: str) -> str: """ Validate UUID using pyvalidators library. - + Args: uuid_string: The UUID string to validate - + Returns: str: Validated UUID string - + Raises: HTTPException: If validation fails """ if not uuid_string or not uuid_string.strip(): - raise HTTPException( - status_code=400, - detail="UUID is required" - ) - + raise HTTPException(status_code=400, detail="UUID is required") + uuid_string = uuid_string.strip() - + if not validators.uuid(uuid_string): - raise HTTPException( - status_code=400, - detail="Invalid UUID format" - ) - + raise HTTPException(status_code=400, detail="Invalid UUID format") + return uuid_string -def validate_connector_config(connector_type: str | Any, config: dict[str, Any]) -> dict[str, Any]: +def validate_connector_config( + connector_type: str | Any, config: dict[str, Any] +) -> dict[str, Any]: """ Validate connector configuration based on connector type. - + Args: connector_type: The type of connector (string or enum) config: The configuration dictionary to validate - + Returns: dict: Validated configuration - + Raises: ValueError: If validation fails """ @@ -454,110 +400,125 @@ def validate_connector_config(connector_type: str | Any, config: dict[str, Any]) raise ValueError("config must be a dictionary of connector settings") # Convert enum to string if needed - connector_type_str = str(connector_type).split('.')[-1] if hasattr(connector_type, 'value') else str(connector_type) - + connector_type_str = ( + str(connector_type).split(".")[-1] + if hasattr(connector_type, "value") + else str(connector_type) + ) + # Validation function helpers def validate_email_field(key: str, connector_name: str) -> None: if not validators.email(config.get(key, "")): raise ValueError(f"Invalid email format for {connector_name} connector") - + def validate_url_field(key: str, connector_name: str) -> None: - if not validators.url(config.get(key, "")): + if not validators.url(config.get(key, "").strip(), simple_host=True): raise ValueError(f"Invalid base URL format for {connector_name} connector") - + def validate_list_field(key: str, field_name: str) -> None: value = config.get(key) if not isinstance(value, list) or not value: raise ValueError(f"{field_name} must be a non-empty list of strings") - + # Lookup table for connector validation rules connector_rules = { - "SERPER_API": { - "required": ["SERPER_API_KEY"], - "validators": {} - }, - "TAVILY_API": { - "required": ["TAVILY_API_KEY"], - "validators": {} - }, - "LINKUP_API": { - "required": ["LINKUP_API_KEY"], - "validators": {} - }, - "SLACK_CONNECTOR": { - "required": ["SLACK_BOT_TOKEN"], - "validators": {} + "SERPER_API": {"required": ["SERPER_API_KEY"], "validators": {}}, + "TAVILY_API": {"required": ["TAVILY_API_KEY"], "validators": {}}, + "SEARXNG_API": { + "required": ["SEARXNG_HOST"], + "optional": [ + "SEARXNG_API_KEY", + "SEARXNG_ENGINES", + "SEARXNG_CATEGORIES", + "SEARXNG_LANGUAGE", + "SEARXNG_SAFESEARCH", + "SEARXNG_VERIFY_SSL", + ], + "validators": { + "SEARXNG_HOST": lambda: validate_url_field("SEARXNG_HOST", "SearxNG") + }, }, + "LINKUP_API": {"required": ["LINKUP_API_KEY"], "validators": {}}, + "SLACK_CONNECTOR": {"required": ["SLACK_BOT_TOKEN"], "validators": {}}, "NOTION_CONNECTOR": { "required": ["NOTION_INTEGRATION_TOKEN"], - "validators": {} + "validators": {}, }, "GITHUB_CONNECTOR": { "required": ["GITHUB_PAT", "repo_full_names"], "validators": { - "repo_full_names": lambda: validate_list_field("repo_full_names", "repo_full_names") - } - }, - "LINEAR_CONNECTOR": { - "required": ["LINEAR_API_KEY"], - "validators": {} - }, - "DISCORD_CONNECTOR": { - "required": ["DISCORD_BOT_TOKEN"], - "validators": {} + "repo_full_names": lambda: validate_list_field( + "repo_full_names", "repo_full_names" + ) + }, }, + "LINEAR_CONNECTOR": {"required": ["LINEAR_API_KEY"], "validators": {}}, + "DISCORD_CONNECTOR": {"required": ["DISCORD_BOT_TOKEN"], "validators": {}}, "JIRA_CONNECTOR": { "required": ["JIRA_EMAIL", "JIRA_API_TOKEN", "JIRA_BASE_URL"], "validators": { "JIRA_EMAIL": lambda: validate_email_field("JIRA_EMAIL", "JIRA"), - "JIRA_BASE_URL": lambda: validate_url_field("JIRA_BASE_URL", "JIRA") - } + "JIRA_BASE_URL": lambda: validate_url_field("JIRA_BASE_URL", "JIRA"), + }, }, "CONFLUENCE_CONNECTOR": { - "required": ["CONFLUENCE_BASE_URL", "CONFLUENCE_EMAIL", "CONFLUENCE_API_TOKEN"], + "required": [ + "CONFLUENCE_BASE_URL", + "CONFLUENCE_EMAIL", + "CONFLUENCE_API_TOKEN", + ], "validators": { - "CONFLUENCE_EMAIL": lambda: validate_email_field("CONFLUENCE_EMAIL", "Confluence"), - "CONFLUENCE_BASE_URL": lambda: validate_url_field("CONFLUENCE_BASE_URL", "Confluence") - } + "CONFLUENCE_EMAIL": lambda: validate_email_field( + "CONFLUENCE_EMAIL", "Confluence" + ), + "CONFLUENCE_BASE_URL": lambda: validate_url_field( + "CONFLUENCE_BASE_URL", "Confluence" + ), + }, }, - "CLICKUP_CONNECTOR": { - "required": ["CLICKUP_API_TOKEN"], - "validators": {} - }, - "GOOGLE_CALENDAR_CONNECTOR": { - "required": ["token", "refresh_token", "token_uri", "client_id", "expiry", "scopes", "client_secret"], - "validators": {}, - "allow_none_or_empty": False # Special flag for Google connectors - }, - "GOOGLE_GMAIL_CONNECTOR": { - "required": ["token", "refresh_token", "token_uri", "client_id", "expiry", "scopes", "client_secret"], - "validators": {}, - "allow_none_or_empty": False - }, - "AIRTABLE_CONNECTOR": { - "required": ["AIRTABLE_API_KEY", "AIRTABLE_BASE_ID"], - "validators": {} - }, - "LUMA_CONNECTOR": { - "required": ["LUMA_API_KEY"], - "validators": {} - } + "CLICKUP_CONNECTOR": {"required": ["CLICKUP_API_TOKEN"], "validators": {}}, + # "GOOGLE_CALENDAR_CONNECTOR": { + # "required": ["token", "refresh_token", "token_uri", "client_id", "expiry", "scopes", "client_secret"], + # "validators": {}, + # "allow_none_or_empty": False # Special flag for Google connectors + # }, + # "GOOGLE_GMAIL_CONNECTOR": { + # "required": ["token", "refresh_token", "token_uri", "client_id", "expiry", "scopes", "client_secret"], + # "validators": {}, + # "allow_none_or_empty": False + # }, + # "AIRTABLE_CONNECTOR": { + # "required": ["AIRTABLE_API_KEY", "AIRTABLE_BASE_ID"], + # "validators": {} + # }, + "LUMA_CONNECTOR": {"required": ["LUMA_API_KEY"], "validators": {}}, } - + rules = connector_rules.get(connector_type_str) if not rules: return config # Unknown connector type, pass through - - # Validate required keys match exactly - if set(config.keys()) != set(rules["required"]): + + required_keys = set(rules["required"]) + optional_keys = set(rules.get("optional", [])) + config_keys = set(config.keys()) + + # Validate that no unexpected keys are present + if not config_keys.issubset(required_keys | optional_keys): + allowed_keys = list(required_keys | optional_keys) raise ValueError( - f"For {connector_type_str} connector type, config must only contain these keys: {rules['required']}" + f"For {connector_type_str} connector type, config may only contain these keys: {allowed_keys}" ) - + + # Validate that all required keys are present + if not required_keys.issubset(config_keys): + raise ValueError( + f"For {connector_type_str} connector type, config must include these keys: {sorted(required_keys)}" + ) + # Apply custom validators first (these check format before emptiness) for validator_func in rules["validators"].values(): validator_func() - + # Validate each field is not empty for key in rules["required"]: # Special handling for Google connectors that don't allow None or empty strings @@ -568,5 +529,5 @@ def validate_connector_config(connector_type: str | Any, config: dict[str, Any]) # Standard check: field must have a truthy value if not config.get(key): raise ValueError(f"{key} cannot be empty") - + return config diff --git a/surfsense_web/app/dashboard/[search_space_id]/client-layout.tsx b/surfsense_web/app/dashboard/[search_space_id]/client-layout.tsx index 6610cb046..f3c1531a8 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/client-layout.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/client-layout.tsx @@ -1,12 +1,16 @@ "use client"; +import { Loader2 } from "lucide-react"; +import { usePathname, useRouter } from "next/navigation"; import type React from "react"; -import { useState } from "react"; +import { useEffect, useState } from "react"; import { DashboardBreadcrumb } from "@/components/dashboard-breadcrumb"; import { AppSidebarProvider } from "@/components/sidebar/AppSidebarProvider"; import { ThemeTogglerComponent } from "@/components/theme/theme-toggle"; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; import { Separator } from "@/components/ui/separator"; import { SidebarInset, SidebarProvider, SidebarTrigger } from "@/components/ui/sidebar"; +import { useLLMPreferences } from "@/hooks/use-llm-configs"; export function DashboardClientLayout({ children, @@ -19,6 +23,16 @@ export function DashboardClientLayout({ navSecondary: any[]; navMain: any[]; }) { + const router = useRouter(); + const pathname = usePathname(); + const searchSpaceIdNum = Number(searchSpaceId); + + const { loading, error, isOnboardingComplete } = useLLMPreferences(searchSpaceIdNum); + const [hasCheckedOnboarding, setHasCheckedOnboarding] = useState(false); + + // Skip onboarding check if we're already on the onboarding page + const isOnboardingPage = pathname?.includes("/onboard"); + const [open, setOpen] = useState(() => { try { const match = document.cookie.match(/(?:^|; )sidebar_state=([^;]+)/); @@ -29,6 +43,68 @@ export function DashboardClientLayout({ return true; }); + useEffect(() => { + // Skip check if already on onboarding page + if (isOnboardingPage) { + setHasCheckedOnboarding(true); + return; + } + + // Only check once after preferences have loaded + if (!loading && !hasCheckedOnboarding) { + const onboardingComplete = isOnboardingComplete(); + + if (!onboardingComplete) { + router.push(`/dashboard/${searchSpaceId}/onboard`); + } + + setHasCheckedOnboarding(true); + } + }, [ + loading, + isOnboardingComplete, + isOnboardingPage, + router, + searchSpaceId, + hasCheckedOnboarding, + ]); + + // Show loading screen while checking onboarding status (only on first load) + if (!hasCheckedOnboarding && loading && !isOnboardingPage) { + return ( +
+ + + Loading Configuration + Checking your LLM preferences... + + + + + +
+ ); + } + + // Show error screen if there's an error loading preferences (but not on onboarding page) + if (error && !hasCheckedOnboarding && !isOnboardingPage) { + return ( +
+ + + + Configuration Error + + Failed to load your LLM configuration + + +

{error}

+ + +
+ ); + } + return ( {/* Use AppSidebarProvider which fetches user, search space, and recent chats */} diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx index e7e195759..eb1d53895 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx @@ -41,7 +41,7 @@ import { import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { useSearchSourceConnectors } from "@/hooks/useSearchSourceConnectors"; +import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; import { cn } from "@/lib/utils"; // Helper function to format date with time @@ -65,7 +65,7 @@ export default function ConnectorsPage() { const today = new Date(); const { connectors, isLoading, error, deleteConnector, indexConnector } = - useSearchSourceConnectors(); + useSearchSourceConnectors(false, parseInt(searchSpaceId)); const [connectorToDelete, setConnectorToDelete] = useState(null); const [indexingConnectorId, setIndexingConnectorId] = useState(null); const [datePickerOpen, setDatePickerOpen] = useState(false); @@ -141,28 +141,6 @@ export default function ConnectorsPage() { } }; - const getDisabledEndDates = (date: Date) => { - const connector = connectors.find((c) => c.id === selectedConnectorForIndexing); - - switch (connector?.connector_type) { - case EnumConnectorName.GOOGLE_CALENDAR_CONNECTOR: - return startDate ? date < startDate : false; - default: - return date > today || (startDate ? date < startDate : false); - } - }; - - const getDisabledStartDates = (date: Date) => { - const connector = connectors.find((c) => c.id === selectedConnectorForIndexing); - - switch (connector?.connector_type) { - case EnumConnectorName.GOOGLE_CALENDAR_CONNECTOR: - return endDate ? date > endDate : false; - default: - return date > today || (endDate ? date > endDate : false); - } - }; - return (
@@ -389,13 +366,7 @@ export default function ConnectorsPage() { - +
diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/edit/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/edit/page.tsx index 75248e4d0..87844e8c8 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/edit/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/edit/page.tsx @@ -20,7 +20,7 @@ import { } from "@/components/ui/card"; import { Form } from "@/components/ui/form"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { useConnectorEditPage } from "@/hooks/useConnectorEditPage"; +import { useConnectorEditPage } from "@/hooks/use-connector-edit-page"; // Import Utils, Types, Hook, and Components import { getConnectorTypeDisplay } from "@/lib/connectors/utils"; diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/page.tsx index ae8910f95..41a8492fd 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/page.tsx @@ -24,7 +24,7 @@ import { Input } from "@/components/ui/input"; import { type SearchSourceConnector, useSearchSourceConnectors, -} from "@/hooks/useSearchSourceConnectors"; +} from "@/hooks/use-search-source-connectors"; // Define the form schema with Zod const apiConnectorFormSchema = z.object({ @@ -83,7 +83,7 @@ export default function EditConnectorPage() { const searchSpaceId = params.search_space_id as string; const connectorId = parseInt(params.connector_id as string, 10); - const { connectors, updateConnector } = useSearchSourceConnectors(); + const { connectors, updateConnector } = useSearchSourceConnectors(false, parseInt(searchSpaceId)); const [connector, setConnector] = useState(null); const [isLoading, setIsLoading] = useState(true); const [isSubmitting, setIsSubmitting] = useState(false); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/airtable-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/airtable-connector/page.tsx index 18163482c..2d82877b3 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/airtable-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/airtable-connector/page.tsx @@ -21,7 +21,7 @@ import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; import { type SearchSourceConnector, useSearchSourceConnectors, -} from "@/hooks/useSearchSourceConnectors"; +} from "@/hooks/use-search-source-connectors"; export default function AirtableConnectorPage() { const router = useRouter(); @@ -30,10 +30,10 @@ export default function AirtableConnectorPage() { const [isConnecting, setIsConnecting] = useState(false); const [doesConnectorExist, setDoesConnectorExist] = useState(false); - const { fetchConnectors } = useSearchSourceConnectors(); + const { fetchConnectors } = useSearchSourceConnectors(true, parseInt(searchSpaceId)); useEffect(() => { - fetchConnectors().then((data) => { + fetchConnectors(parseInt(searchSpaceId)).then((data) => { const connector = data.find( (c: SearchSourceConnector) => c.connector_type === EnumConnectorName.AIRTABLE_CONNECTOR ); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/clickup-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/clickup-connector/page.tsx index 0914a1c00..cc86b1680 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/clickup-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/clickup-connector/page.tsx @@ -22,7 +22,7 @@ import { import { Input } from "@/components/ui/input"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { useSearchSourceConnectors } from "@/hooks/useSearchSourceConnectors"; +import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; // Define the form schema with Zod const clickupConnectorFormSchema = z.object({ @@ -69,7 +69,7 @@ export default function ClickUpConnectorPage() { last_indexed_at: null, }; - await createConnector(connectorData); + await createConnector(connectorData, parseInt(searchSpaceId)); toast.success("ClickUp connector created successfully!"); router.push(`/dashboard/${searchSpaceId}/connectors`); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/confluence-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/confluence-connector/page.tsx index d135b91cc..361afd5ff 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/confluence-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/confluence-connector/page.tsx @@ -24,7 +24,7 @@ import { Input } from "@/components/ui/input"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { useSearchSourceConnectors } from "@/hooks/useSearchSourceConnectors"; +import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; // Define the form schema with Zod const confluenceConnectorFormSchema = z.object({ @@ -77,17 +77,20 @@ export default function ConfluenceConnectorPage() { const onSubmit = async (values: ConfluenceConnectorFormValues) => { setIsSubmitting(true); try { - await createConnector({ - name: values.name, - connector_type: EnumConnectorName.CONFLUENCE_CONNECTOR, - config: { - CONFLUENCE_BASE_URL: values.base_url, - CONFLUENCE_EMAIL: values.email, - CONFLUENCE_API_TOKEN: values.api_token, + await createConnector( + { + name: values.name, + connector_type: EnumConnectorName.CONFLUENCE_CONNECTOR, + config: { + CONFLUENCE_BASE_URL: values.base_url, + CONFLUENCE_EMAIL: values.email, + CONFLUENCE_API_TOKEN: values.api_token, + }, + is_indexable: true, + last_indexed_at: null, }, - is_indexable: true, - last_indexed_at: null, - }); + parseInt(searchSpaceId) + ); toast.success("Confluence connector created successfully!"); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/discord-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/discord-connector/page.tsx index 2f01e52f4..573190945 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/discord-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/discord-connector/page.tsx @@ -37,7 +37,7 @@ import { Input } from "@/components/ui/input"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { useSearchSourceConnectors } from "@/hooks/useSearchSourceConnectors"; +import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; // Define the form schema with Zod const discordConnectorFormSchema = z.object({ @@ -73,15 +73,18 @@ export default function DiscordConnectorPage() { const onSubmit = async (values: DiscordConnectorFormValues) => { setIsSubmitting(true); try { - await createConnector({ - name: values.name, - connector_type: EnumConnectorName.DISCORD_CONNECTOR, - config: { - DISCORD_BOT_TOKEN: values.bot_token, + await createConnector( + { + name: values.name, + connector_type: EnumConnectorName.DISCORD_CONNECTOR, + config: { + DISCORD_BOT_TOKEN: values.bot_token, + }, + is_indexable: true, + last_indexed_at: null, }, - is_indexable: true, - last_indexed_at: null, - }); + parseInt(searchSpaceId) + ); toast.success("Discord connector created successfully!"); router.push(`/dashboard/${searchSpaceId}/connectors`); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/github-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/github-connector/page.tsx index 7491b06ef..e5a6d24f7 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/github-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/github-connector/page.tsx @@ -39,7 +39,7 @@ import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; // Assuming useSearchSourceConnectors hook exists and works similarly -import { useSearchSourceConnectors } from "@/hooks/useSearchSourceConnectors"; +import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; // Define the form schema with Zod for GitHub PAT entry step const githubPatFormSchema = z.object({ @@ -148,16 +148,19 @@ export default function GithubConnectorPage() { setIsCreatingConnector(true); try { - await createConnector({ - name: connectorName, // Use the stored name - connector_type: EnumConnectorName.GITHUB_CONNECTOR, - config: { - GITHUB_PAT: validatedPat, // Use the stored validated PAT - repo_full_names: selectedRepos, // Add the selected repo names + await createConnector( + { + name: connectorName, // Use the stored name + connector_type: EnumConnectorName.GITHUB_CONNECTOR, + config: { + GITHUB_PAT: validatedPat, // Use the stored validated PAT + repo_full_names: selectedRepos, // Add the selected repo names + }, + is_indexable: true, + last_indexed_at: null, }, - is_indexable: true, - last_indexed_at: null, - }); + parseInt(searchSpaceId) + ); toast.success("GitHub connector created successfully!"); router.push(`/dashboard/${searchSpaceId}/connectors`); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-calendar-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-calendar-connector/page.tsx index 4bb33c3f6..2fdc95671 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-calendar-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-calendar-connector/page.tsx @@ -23,7 +23,7 @@ import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; import { type SearchSourceConnector, useSearchSourceConnectors, -} from "@/hooks/useSearchSourceConnectors"; +} from "@/hooks/use-search-source-connectors"; export default function GoogleCalendarConnectorPage() { const router = useRouter(); @@ -32,10 +32,10 @@ export default function GoogleCalendarConnectorPage() { const [isConnecting, setIsConnecting] = useState(false); const [doesConnectorExist, setDoesConnectorExist] = useState(false); - const { fetchConnectors } = useSearchSourceConnectors(); + const { fetchConnectors } = useSearchSourceConnectors(true, parseInt(searchSpaceId)); useEffect(() => { - fetchConnectors().then((data) => { + fetchConnectors(parseInt(searchSpaceId)).then((data) => { const connector = data.find( (c: SearchSourceConnector) => c.connector_type === EnumConnectorName.GOOGLE_CALENDAR_CONNECTOR diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-gmail-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-gmail-connector/page.tsx index 7ce2b645a..c1354d03e 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-gmail-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-gmail-connector/page.tsx @@ -23,7 +23,7 @@ import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; import { type SearchSourceConnector, useSearchSourceConnectors, -} from "@/hooks/useSearchSourceConnectors"; +} from "@/hooks/use-search-source-connectors"; export default function GoogleGmailConnectorPage() { const router = useRouter(); @@ -32,10 +32,10 @@ export default function GoogleGmailConnectorPage() { const [isConnecting, setIsConnecting] = useState(false); const [doesConnectorExist, setDoesConnectorExist] = useState(false); - const { fetchConnectors } = useSearchSourceConnectors(); + const { fetchConnectors } = useSearchSourceConnectors(true, parseInt(searchSpaceId)); useEffect(() => { - fetchConnectors().then((data) => { + fetchConnectors(parseInt(searchSpaceId)).then((data) => { const connector = data.find( (c: SearchSourceConnector) => c.connector_type === EnumConnectorName.GOOGLE_GMAIL_CONNECTOR ); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/jira-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/jira-connector/page.tsx index 174047a15..6d3e92e42 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/jira-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/jira-connector/page.tsx @@ -37,7 +37,7 @@ import { Input } from "@/components/ui/input"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { useSearchSourceConnectors } from "@/hooks/useSearchSourceConnectors"; +import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; // Define the form schema with Zod const jiraConnectorFormSchema = z.object({ @@ -90,17 +90,20 @@ export default function JiraConnectorPage() { const onSubmit = async (values: JiraConnectorFormValues) => { setIsSubmitting(true); try { - await createConnector({ - name: values.name, - connector_type: EnumConnectorName.JIRA_CONNECTOR, - config: { - JIRA_BASE_URL: values.base_url, - JIRA_EMAIL: values.email, - JIRA_API_TOKEN: values.api_token, + await createConnector( + { + name: values.name, + connector_type: EnumConnectorName.JIRA_CONNECTOR, + config: { + JIRA_BASE_URL: values.base_url, + JIRA_EMAIL: values.email, + JIRA_API_TOKEN: values.api_token, + }, + is_indexable: true, + last_indexed_at: null, }, - is_indexable: true, - last_indexed_at: null, - }); + parseInt(searchSpaceId) + ); toast.success("Jira connector created successfully!"); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/linear-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/linear-connector/page.tsx index d5d940b9d..b1d2a51e6 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/linear-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/linear-connector/page.tsx @@ -37,7 +37,7 @@ import { Input } from "@/components/ui/input"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { useSearchSourceConnectors } from "@/hooks/useSearchSourceConnectors"; +import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; // Define the form schema with Zod const linearConnectorFormSchema = z.object({ @@ -77,15 +77,18 @@ export default function LinearConnectorPage() { const onSubmit = async (values: LinearConnectorFormValues) => { setIsSubmitting(true); try { - await createConnector({ - name: values.name, - connector_type: EnumConnectorName.LINEAR_CONNECTOR, - config: { - LINEAR_API_KEY: values.api_key, + await createConnector( + { + name: values.name, + connector_type: EnumConnectorName.LINEAR_CONNECTOR, + config: { + LINEAR_API_KEY: values.api_key, + }, + is_indexable: true, + last_indexed_at: null, }, - is_indexable: true, - last_indexed_at: null, - }); + parseInt(searchSpaceId) + ); toast.success("Linear connector created successfully!"); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/linkup-api/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/linkup-api/page.tsx index 311ea2c3f..a4467fb1d 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/linkup-api/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/linkup-api/page.tsx @@ -30,7 +30,7 @@ import { import { Input } from "@/components/ui/input"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { useSearchSourceConnectors } from "@/hooks/useSearchSourceConnectors"; +import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; // Define the form schema with Zod const linkupApiFormSchema = z.object({ @@ -65,15 +65,18 @@ export default function LinkupApiPage() { const onSubmit = async (values: LinkupApiFormValues) => { setIsSubmitting(true); try { - await createConnector({ - name: values.name, - connector_type: EnumConnectorName.LINKUP_API, - config: { - LINKUP_API_KEY: values.api_key, + await createConnector( + { + name: values.name, + connector_type: EnumConnectorName.LINKUP_API, + config: { + LINKUP_API_KEY: values.api_key, + }, + is_indexable: false, + last_indexed_at: null, }, - is_indexable: false, - last_indexed_at: null, - }); + parseInt(searchSpaceId) + ); toast.success("Linkup API connector created successfully!"); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/luma-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/luma-connector/page.tsx index 864c7fdba..ca9b716d5 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/luma-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/luma-connector/page.tsx @@ -33,7 +33,7 @@ import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; import { type SearchSourceConnector, useSearchSourceConnectors, -} from "@/hooks/useSearchSourceConnectors"; +} from "@/hooks/use-search-source-connectors"; // Define the form schema with Zod const lumaConnectorFormSchema = z.object({ @@ -55,7 +55,10 @@ export default function LumaConnectorPage() { const [isSubmitting, setIsSubmitting] = useState(false); const [doesConnectorExist, setDoesConnectorExist] = useState(false); - const { fetchConnectors, createConnector } = useSearchSourceConnectors(); + const { fetchConnectors, createConnector } = useSearchSourceConnectors( + true, + parseInt(searchSpaceId) + ); // Initialize the form const form = useForm({ @@ -67,29 +70,38 @@ export default function LumaConnectorPage() { }); useEffect(() => { - fetchConnectors().then((data) => { - const connector = data.find( - (c: SearchSourceConnector) => c.connector_type === EnumConnectorName.LUMA_CONNECTOR - ); - if (connector) { - setDoesConnectorExist(true); - } - }); - }, [fetchConnectors]); + fetchConnectors(parseInt(searchSpaceId)) + .then((data) => { + if (data && Array.isArray(data)) { + const connector = data.find( + (c: SearchSourceConnector) => c.connector_type === EnumConnectorName.LUMA_CONNECTOR + ); + if (connector) { + setDoesConnectorExist(true); + } + } + }) + .catch((error) => { + console.error("Error fetching connectors:", error); + }); + }, [fetchConnectors, searchSpaceId]); // Handle form submission const onSubmit = async (values: LumaConnectorFormValues) => { setIsSubmitting(true); try { - await createConnector({ - name: values.name, - connector_type: EnumConnectorName.LUMA_CONNECTOR, - config: { - LUMA_API_KEY: values.api_key, + await createConnector( + { + name: values.name, + connector_type: EnumConnectorName.LUMA_CONNECTOR, + config: { + LUMA_API_KEY: values.api_key, + }, + is_indexable: true, + last_indexed_at: null, }, - is_indexable: true, - last_indexed_at: null, - }); + parseInt(searchSpaceId) + ); toast.success("Luma connector created successfully!"); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/notion-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/notion-connector/page.tsx index 2a447e611..5e0a4bbb3 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/notion-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/notion-connector/page.tsx @@ -37,7 +37,7 @@ import { Input } from "@/components/ui/input"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { useSearchSourceConnectors } from "@/hooks/useSearchSourceConnectors"; +import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; // Define the form schema with Zod const notionConnectorFormSchema = z.object({ @@ -72,15 +72,18 @@ export default function NotionConnectorPage() { const onSubmit = async (values: NotionConnectorFormValues) => { setIsSubmitting(true); try { - await createConnector({ - name: values.name, - connector_type: EnumConnectorName.NOTION_CONNECTOR, - config: { - NOTION_INTEGRATION_TOKEN: values.integration_token, + await createConnector( + { + name: values.name, + connector_type: EnumConnectorName.NOTION_CONNECTOR, + config: { + NOTION_INTEGRATION_TOKEN: values.integration_token, + }, + is_indexable: true, + last_indexed_at: null, }, - is_indexable: true, - last_indexed_at: null, - }); + parseInt(searchSpaceId) + ); toast.success("Notion connector created successfully!"); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/page.tsx index a8bb9cfe8..1c5aa049d 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/page.tsx @@ -45,6 +45,13 @@ const connectorCategories: ConnectorCategory[] = [ icon: getConnectorIcon(EnumConnectorName.TAVILY_API, "h-6 w-6"), status: "available", }, + { + id: "searxng", + title: "SearxNG", + description: "Use your own SearxNG meta-search instance for web results.", + icon: getConnectorIcon(EnumConnectorName.SEARXNG_API, "h-6 w-6"), + status: "available", + }, { id: "linkup-api", title: "Linkup API", diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/searxng/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/searxng/page.tsx new file mode 100644 index 000000000..4fd406706 --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/searxng/page.tsx @@ -0,0 +1,364 @@ +"use client"; + +import { zodResolver } from "@hookform/resolvers/zod"; +import { ArrowLeft, Check, Info, Loader2 } from "lucide-react"; +import { motion } from "motion/react"; +import { useParams, useRouter } from "next/navigation"; +import { useState } from "react"; +import { useForm } from "react-hook-form"; +import { toast } from "sonner"; +import * as z from "zod"; +import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; +import { Button } from "@/components/ui/button"; +import { + Card, + CardContent, + CardDescription, + CardFooter, + CardHeader, + CardTitle, +} from "@/components/ui/card"; +import { + Form, + FormControl, + FormDescription, + FormField, + FormItem, + FormLabel, + FormMessage, +} from "@/components/ui/form"; +import { Input } from "@/components/ui/input"; +import { Switch } from "@/components/ui/switch"; +import { EnumConnectorName } from "@/contracts/enums/connector"; +import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; +import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; + +const searxngFormSchema = z.object({ + name: z.string().min(3, { + message: "Connector name must be at least 3 characters.", + }), + host: z + .string({ required_error: "Host is required." }) + .url({ message: "Enter a valid SearxNG host URL (e.g. https://searxng.example.org)." }), + api_key: z.string().optional(), + engines: z.string().optional(), + categories: z.string().optional(), + language: z.string().optional(), + safesearch: z + .string() + .regex(/^[0-2]?$/, { message: "SafeSearch must be 0, 1, or 2." }) + .optional(), + verify_ssl: z.boolean().default(true), +}); + +type SearxngFormValues = z.infer; + +const parseCommaSeparated = (value?: string | null) => { + if (!value) return undefined; + const items = value + .split(",") + .map((item) => item.trim()) + .filter((item) => item.length > 0); + return items.length > 0 ? items : undefined; +}; + +export default function SearxngConnectorPage() { + const router = useRouter(); + const params = useParams(); + const searchSpaceId = params.search_space_id as string; + const [isSubmitting, setIsSubmitting] = useState(false); + const { createConnector } = useSearchSourceConnectors(); + + const form = useForm({ + resolver: zodResolver(searxngFormSchema), + defaultValues: { + name: "SearxNG Connector", + host: "", + api_key: "", + engines: "", + categories: "", + language: "", + safesearch: "", + verify_ssl: true, + }, + }); + + const onSubmit = async (values: SearxngFormValues) => { + setIsSubmitting(true); + try { + const config: Record = { + SEARXNG_HOST: values.host.trim(), + }; + + const apiKey = values.api_key?.trim(); + if (apiKey) config.SEARXNG_API_KEY = apiKey; + + const engines = parseCommaSeparated(values.engines); + if (engines) config.SEARXNG_ENGINES = engines; + + const categories = parseCommaSeparated(values.categories); + if (categories) config.SEARXNG_CATEGORIES = categories; + + const language = values.language?.trim(); + if (language) config.SEARXNG_LANGUAGE = language; + + const safesearch = values.safesearch?.trim(); + if (safesearch) { + const parsed = Number(safesearch); + if (!Number.isNaN(parsed)) { + config.SEARXNG_SAFESEARCH = parsed; + } + } + + // Include verify flag only when disabled to keep config minimal + if (values.verify_ssl === false) { + config.SEARXNG_VERIFY_SSL = false; + } + + await createConnector( + { + name: values.name, + connector_type: EnumConnectorName.SEARXNG_API, + config, + is_indexable: false, + last_indexed_at: null, + }, + parseInt(searchSpaceId) + ); + + toast.success("SearxNG connector created successfully!"); + router.push(`/dashboard/${searchSpaceId}/connectors`); + } catch (error) { + console.error("Error creating SearxNG connector:", error); + toast.error(error instanceof Error ? error.message : "Failed to create connector"); + } finally { + setIsSubmitting(false); + } + }; + + return ( +
+ + +
+
+
+ {getConnectorIcon(EnumConnectorName.SEARXNG_API, "h-6 w-6")} +
+
+

Connect SearxNG

+

+ Bring your self-hosted SearxNG meta-search engine into SurfSense. +

+
+
+
+ + + + + Connect SearxNG + + Integrate SurfSense with any SearxNG instance to broaden your search coverage while + preserving privacy and control. + + + + + + SearxNG Instance Required + + You need access to a running SearxNG instance. Refer to the{" "} + + SearxNG installation guide + {" "} + for setup instructions. If your instance requires an API key, include it below. + + + +
+ + ( + + Connector Name + + + + A friendly name to identify this connector. + + + )} + /> + + ( + + SearxNG Host + + + + + Provide the full base URL to your SearxNG instance. Include the protocol + (http/https). + + + + )} + /> + + ( + + API Key (optional) + + + + + Leave empty if your SearxNG instance does not enforce API keys. + + + + )} + /> + +
+ ( + + Engines (optional) + + + + + Comma-separated list to target specific engines. + + + + )} + /> + + ( + + Categories (optional) + + + + + Comma-separated list of SearxNG categories. + + + + )} + /> +
+ +
+ ( + + Preferred Language (optional) + + + + + IETF language tag (e.g. en, en-US). Leave blank to inherit defaults. + + + + )} + /> + + ( + + SafeSearch Level (optional) + + + + + Set 0, 1, or 2 to adjust SafeSearch filtering. Leave blank to use the + instance default. + + + + )} + /> +
+ + ( + +
+ Verify SSL Certificates + + Disable only when connecting to instances with self-signed certificates. + +
+ + + + + )} + /> + + + + + + +
+ + +
+ ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/serper-api/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/serper-api/page.tsx index 0d38d4314..10e5da6f7 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/serper-api/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/serper-api/page.tsx @@ -30,7 +30,7 @@ import { import { Input } from "@/components/ui/input"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { useSearchSourceConnectors } from "@/hooks/useSearchSourceConnectors"; +import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; // Define the form schema with Zod const serperApiFormSchema = z.object({ @@ -65,15 +65,18 @@ export default function SerperApiPage() { const onSubmit = async (values: SerperApiFormValues) => { setIsSubmitting(true); try { - await createConnector({ - name: values.name, - connector_type: EnumConnectorName.SERPER_API, - config: { - SERPER_API_KEY: values.api_key, + await createConnector( + { + name: values.name, + connector_type: EnumConnectorName.SERPER_API, + config: { + SERPER_API_KEY: values.api_key, + }, + is_indexable: false, + last_indexed_at: null, }, - is_indexable: false, - last_indexed_at: null, - }); + parseInt(searchSpaceId) + ); toast.success("Serper API connector created successfully!"); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/slack-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/slack-connector/page.tsx index 1cf18a89c..e22d69d6d 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/slack-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/slack-connector/page.tsx @@ -37,7 +37,7 @@ import { Input } from "@/components/ui/input"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { useSearchSourceConnectors } from "@/hooks/useSearchSourceConnectors"; +import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; // Define the form schema with Zod const slackConnectorFormSchema = z.object({ @@ -72,15 +72,18 @@ export default function SlackConnectorPage() { const onSubmit = async (values: SlackConnectorFormValues) => { setIsSubmitting(true); try { - await createConnector({ - name: values.name, - connector_type: EnumConnectorName.SLACK_CONNECTOR, - config: { - SLACK_BOT_TOKEN: values.bot_token, + await createConnector( + { + name: values.name, + connector_type: EnumConnectorName.SLACK_CONNECTOR, + config: { + SLACK_BOT_TOKEN: values.bot_token, + }, + is_indexable: true, + last_indexed_at: null, }, - is_indexable: true, - last_indexed_at: null, - }); + parseInt(searchSpaceId) + ); toast.success("Slack connector created successfully!"); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/tavily-api/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/tavily-api/page.tsx index 7fc80b10d..f744241b2 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/tavily-api/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/tavily-api/page.tsx @@ -30,7 +30,7 @@ import { import { Input } from "@/components/ui/input"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { useSearchSourceConnectors } from "@/hooks/useSearchSourceConnectors"; +import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; // Define the form schema with Zod const tavilyApiFormSchema = z.object({ @@ -65,15 +65,18 @@ export default function TavilyApiPage() { const onSubmit = async (values: TavilyApiFormValues) => { setIsSubmitting(true); try { - await createConnector({ - name: values.name, - connector_type: EnumConnectorName.TAVILY_API, - config: { - TAVILY_API_KEY: values.api_key, + await createConnector( + { + name: values.name, + connector_type: EnumConnectorName.TAVILY_API, + config: { + TAVILY_API_KEY: values.api_key, + }, + is_indexable: false, + last_indexed_at: null, }, - is_indexable: false, - last_indexed_at: null, - }); + parseInt(searchSpaceId) + ); toast.success("Tavily API connector created successfully!"); diff --git a/surfsense_web/app/dashboard/[search_space_id]/layout.tsx b/surfsense_web/app/dashboard/[search_space_id]/layout.tsx index d0e04fe68..b012484aa 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/layout.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/layout.tsx @@ -33,6 +33,12 @@ export default function DashboardLayout({ icon: "SquareTerminal", items: [], }, + { + title: "Manage LLMs", + url: `/dashboard/${search_space_id}/settings`, + icon: "Settings2", + items: [], + }, { title: "Documents", diff --git a/surfsense_web/app/dashboard/[search_space_id]/logs/(manage)/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/logs/(manage)/page.tsx index f8a0a593e..f89e4153f 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/logs/(manage)/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/logs/(manage)/page.tsx @@ -122,6 +122,55 @@ const logStatusConfig = { FAILED: { icon: X, color: "text-red-600", bgColor: "bg-red-50" }, } as const; +function MessageDetails({ + message, + taskName, + metadata, + createdAt, + children, +}: { + message: string; + taskName?: string; + metadata?: any; + createdAt?: string; + children: React.ReactNode; +}) { + return ( + + {children} + +
+
+ Log details + {createdAt && ( +

+ {new Date(createdAt).toLocaleString()} +

+ )} +
+
+ Close +
+
+ +
+ {taskName && ( +
+ {taskName} +
+ )} + +
+ {message} +
+
+ + + + + ); +} + const columns: ColumnDef[] = [ { id: "select", @@ -219,18 +268,29 @@ const columns: ColumnDef[] = [ cell: ({ row }) => { const message = row.getValue("message") as string; const taskName = row.original.log_metadata?.task_name; + const createdAt = row.getValue("created_at") as string; return ( -
- {taskName && ( -
- {taskName} + +
+ {taskName && ( +
+ {taskName} +
+ )} +
+ {message.length > 100 ? `${message.substring(0, 100)}...` : message}
- )} -
- {message.length > 100 ? `${message.substring(0, 100)}...` : message}
-
+ ); }, size: 400, @@ -839,7 +899,7 @@ function LogsTable({ animate={{ opacity: 1, y: 0 }} transition={{ delay: 0.3 }} > - +
{table.getHeaderGroups().map((headerGroup: any) => ( @@ -847,7 +907,11 @@ function LogsTable({ {header.isPlaceholder ? null : header.column.getCanSort() ? ( - - )} - - {/* Source Information */} -
-

Source Information

-
- {node?.metadata?.title || "Untitled"} -
-
- {node?.text || "No content available"} -
-
- - - )} - - {/* API-fetched document content */} - {!isDirectRenderSource && document && ( - -
- {/* Document Metadata */} - {document.document_metadata && Object.keys(document.document_metadata).length > 0 && ( -
-

Document Information

-
- {Object.entries(document.document_metadata).map(([key, value]) => ( -
-
- {key.replace(/_/g, " ")}: -
-
{String(value)}
-
- ))} -
-
- )} - - {/* External Link */} - {node?.url && ( -
- -
- )} - - {/* Chunks */} -
-
- {/* Header row: header and button side by side */} -
-

Document Content

- {document.content && ( - - - Summary - {summaryOpen ? ( - - ) : ( - - )} - - - )} -
- {/* Expanded summary content: always full width, below the row */} - {document.content && ( - - -
- -
- - - )} -
- - {document.chunks.map((chunk, idx) => ( -
-
- - Chunk {idx + 1} of {document.chunks.length} - - {chunk.id === chunkId && ( - - Referenced Chunk - - )} -
-
- -
-
- ))} -
-
- - )} - - + ); }; diff --git a/surfsense_web/components/chat/ChatInputGroup.tsx b/surfsense_web/components/chat/ChatInputGroup.tsx index 97584c230..c3877c108 100644 --- a/surfsense_web/components/chat/ChatInputGroup.tsx +++ b/surfsense_web/components/chat/ChatInputGroup.tsx @@ -27,7 +27,7 @@ import { import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; import { type Document, useDocuments } from "@/hooks/use-documents"; import { useLLMConfigs, useLLMPreferences } from "@/hooks/use-llm-configs"; -import { useSearchSourceConnectors } from "@/hooks/useSearchSourceConnectors"; +import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; const DocumentSelector = React.memo( ({ @@ -124,19 +124,20 @@ const ConnectorSelector = React.memo( onSelectionChange?: (connectorTypes: string[]) => void; selectedConnectors?: string[]; }) => { + const { search_space_id } = useParams(); const [isOpen, setIsOpen] = useState(false); const { connectorSourceItems, isLoading, isLoaded, fetchConnectors } = - useSearchSourceConnectors(true); + useSearchSourceConnectors(true, Number(search_space_id)); const handleOpenChange = useCallback( (open: boolean) => { setIsOpen(open); if (open && !isLoaded) { - fetchConnectors(); + fetchConnectors(Number(search_space_id)); } }, - [fetchConnectors, isLoaded] + [fetchConnectors, isLoaded, search_space_id] ); const handleConnectorToggle = useCallback( @@ -331,8 +332,11 @@ const ResearchModeSelector = React.memo( ResearchModeSelector.displayName = "ResearchModeSelector"; const LLMSelector = React.memo(() => { - const { llmConfigs, loading: llmLoading, error } = useLLMConfigs(); - const { preferences, updatePreferences, loading: preferencesLoading } = useLLMPreferences(); + const { search_space_id } = useParams(); + const searchSpaceId = Number(search_space_id); + + const { llmConfigs, loading: llmLoading, error } = useLLMConfigs(searchSpaceId); + const { preferences, updatePreferences, loading: preferencesLoading } = useLLMPreferences(searchSpaceId); const isLoading = llmLoading || preferencesLoading; diff --git a/surfsense_web/components/chat/ChatSources.tsx b/surfsense_web/components/chat/ChatSources.tsx index 625a81fa1..5f205d005 100644 --- a/surfsense_web/components/chat/ChatSources.tsx +++ b/surfsense_web/components/chat/ChatSources.tsx @@ -9,12 +9,14 @@ import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/com import { Sheet, SheetContent, SheetHeader, SheetTitle, SheetTrigger } from "@/components/ui/sheet"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; +import { SourceDetailSheet } from "./SourceDetailSheet"; interface Source { id: string; title: string; description: string; url: string; + sourceType: string; } interface SourceGroup { @@ -48,6 +50,9 @@ function getSourceIcon(type: string) { function SourceCard({ source }: { source: Source }) { const hasUrl = source.url && source.url.trim() !== ""; + const chunkId = Number(source.id); + const sourceType = source.sourceType; + const [isOpen, setIsOpen] = useState(false); // Clean up the description for better display const cleanDescription = source.description @@ -55,31 +60,54 @@ function SourceCard({ source }: { source: Source }) { .replace(/\n+/g, " ") .trim(); + const handleUrlClick = (e: React.MouseEvent, url: string) => { + e.preventDefault(); + e.stopPropagation(); + window.open(url, "_blank", "noopener,noreferrer"); + }; + return ( - - -
- - {source.title} - - {hasUrl && ( - - )} -
- - - - {cleanDescription} - - - + + + + +
+ + {source.title} + +
+ + #{chunkId} + + {hasUrl && ( + + )} +
+
+ + + + {cleanDescription} + + + + + ); } @@ -126,6 +154,7 @@ export default function ChatSourcesDisplay({ message }: { message: Message }) { title: node.metadata.title, description: node.text, url: node.url || "", + sourceType: sourceType, })), }); } diff --git a/surfsense_web/components/chat/SourceDetailSheet.tsx b/surfsense_web/components/chat/SourceDetailSheet.tsx new file mode 100644 index 000000000..138d004db --- /dev/null +++ b/surfsense_web/components/chat/SourceDetailSheet.tsx @@ -0,0 +1,245 @@ +"use client"; + +import { ChevronDown, ChevronUp, ExternalLink, Loader2 } from "lucide-react"; +import type React from "react"; +import { type ReactNode, useEffect, useRef, useState } from "react"; +import { MarkdownViewer } from "@/components/markdown-viewer"; +import { Button } from "@/components/ui/button"; +import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"; +import { ScrollArea } from "@/components/ui/scroll-area"; +import { + Sheet, + SheetContent, + SheetDescription, + SheetHeader, + SheetTitle, +} from "@/components/ui/sheet"; +import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; +import { useDocumentByChunk } from "@/hooks/use-document-by-chunk"; +import { cn } from "@/lib/utils"; + +interface SourceDetailSheetProps { + open: boolean; + onOpenChange: (open: boolean) => void; + chunkId: number; + sourceType: string; + title: string; + description?: string; + url?: string; + children?: ReactNode; +} + +const formatDocumentType = (type: string) => { + return type + .split("_") + .map((word) => word.charAt(0) + word.slice(1).toLowerCase()) + .join(" "); +}; + +export function SourceDetailSheet({ + open, + onOpenChange, + chunkId, + sourceType, + title, + description, + url, + children, +}: SourceDetailSheetProps) { + const { document, loading, error, fetchDocumentByChunk, clearDocument } = useDocumentByChunk(); + const chunksContainerRef = useRef(null); + const highlightedChunkRef = useRef(null); + const [summaryOpen, setSummaryOpen] = useState(false); + + // Check if this is a source type that should render directly from node + const isDirectRenderSource = + sourceType === "TAVILY_API" || sourceType === "LINKUP_API" || sourceType === "SEARXNG_API"; + + useEffect(() => { + if (open && chunkId && !isDirectRenderSource) { + fetchDocumentByChunk(chunkId); + } else if (!open && !isDirectRenderSource) { + clearDocument(); + } + }, [open, chunkId, isDirectRenderSource, fetchDocumentByChunk, clearDocument]); + + useEffect(() => { + // Scroll to highlighted chunk when document loads + if (document && highlightedChunkRef.current && chunksContainerRef.current) { + setTimeout(() => { + highlightedChunkRef.current?.scrollIntoView({ + behavior: "smooth", + block: "start", + }); + }, 100); + } + }, [document]); + + const handleUrlClick = (e: React.MouseEvent, clickUrl: string) => { + e.preventDefault(); + e.stopPropagation(); + window.open(clickUrl, "_blank", "noopener,noreferrer"); + }; + + return ( + + {children} + + + + {getConnectorIcon(sourceType)} + {document?.title || title} + + + {document + ? formatDocumentType(document.document_type) + : sourceType && formatDocumentType(sourceType)} + + + + {!isDirectRenderSource && loading && ( +
+ +
+ )} + + {!isDirectRenderSource && error && ( +
+

{error}

+
+ )} + + {/* Direct render for web search providers */} + {isDirectRenderSource && ( + +
+ {/* External Link */} + {url && ( +
+ +
+ )} + + {/* Source Information */} +
+

Source Information

+
+ {title || "Untitled"} +
+
+ {description || "No content available"} +
+
+
+ + )} + + {/* API-fetched document content */} + {!isDirectRenderSource && document && ( + +
+ {/* Document Metadata */} + {document.document_metadata && Object.keys(document.document_metadata).length > 0 && ( +
+

Document Information

+
+ {Object.entries(document.document_metadata).map(([key, value]) => ( +
+
+ {key.replace(/_/g, " ")}: +
+
{String(value)}
+
+ ))} +
+
+ )} + + {/* External Link */} + {url && ( +
+ +
+ )} + + {/* Chunks */} +
+
+ {/* Header row: header and button side by side */} +
+

Document Content

+ {document.content && ( + + + Summary + {summaryOpen ? ( + + ) : ( + + )} + + + )} +
+ {/* Expanded summary content: always full width, below the row */} + {document.content && ( + + +
+ +
+ + + )} +
+ + {document.chunks.map((chunk, idx) => ( +
+
+ + Chunk {idx + 1} of {document.chunks.length} + + {chunk.id === chunkId && ( + + Referenced Chunk + + )} +
+
+ +
+
+ ))} +
+
+ + )} + + + ); +} diff --git a/surfsense_web/components/editConnector/types.ts b/surfsense_web/components/editConnector/types.ts index 4e91ef305..c069972ba 100644 --- a/surfsense_web/components/editConnector/types.ts +++ b/surfsense_web/components/editConnector/types.ts @@ -30,6 +30,13 @@ export const editConnectorSchema = z.object({ NOTION_INTEGRATION_TOKEN: z.string().optional(), SERPER_API_KEY: z.string().optional(), TAVILY_API_KEY: z.string().optional(), + SEARXNG_HOST: z.string().optional(), + SEARXNG_API_KEY: z.string().optional(), + SEARXNG_ENGINES: z.string().optional(), + SEARXNG_CATEGORIES: z.string().optional(), + SEARXNG_LANGUAGE: z.string().optional(), + SEARXNG_SAFESEARCH: z.string().optional(), + SEARXNG_VERIFY_SSL: z.string().optional(), LINEAR_API_KEY: z.string().optional(), LINKUP_API_KEY: z.string().optional(), DISCORD_BOT_TOKEN: z.string().optional(), diff --git a/surfsense_web/components/inference-params-editor.tsx b/surfsense_web/components/inference-params-editor.tsx new file mode 100644 index 000000000..df198cfa0 --- /dev/null +++ b/surfsense_web/components/inference-params-editor.tsx @@ -0,0 +1,138 @@ +"use client"; + +import { useState } from "react"; +import { Plus, Trash2 } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; + +interface InferenceParamsEditorProps { + params: Record; + setParams: (newParams: Record) => void; +} + +const PARAM_KEYS = ["temperature", "max_tokens", "top_k", "top_p"] as const; + +export default function InferenceParamsEditor({ + params, + setParams, +}: InferenceParamsEditorProps) { + const [selectedKey, setSelectedKey] = useState(""); + const [value, setValue] = useState(""); + + const handleAdd = () => { + if (!selectedKey || value === "") return; + + if (params[selectedKey]) { + alert(`${selectedKey} already exists`); + return; + } + + const numericValue = Number(value); + + if ((selectedKey === "temperature" || selectedKey === "top_p") && (isNaN(numericValue) || numericValue < 0 || numericValue > 1)) { + alert("Value must be a number between 0 and 1"); + return; + } + + if ((selectedKey === "max_tokens" || selectedKey === "top_k") && (!Number.isInteger(numericValue) || numericValue < 0)) { + alert("Value must be a non-negative integer"); + return; + } + + setParams({ + ...params, + [selectedKey]: isNaN(numericValue) ? value : numericValue, + }); + + setSelectedKey(""); + setValue(""); + }; + + const handleDelete = (key: string) => { + const newParams = { ...params }; + delete newParams[key]; + setParams(newParams); + }; + + return ( +
+
+
+ + +
+ +
+ + setValue(e.target.value)} + className="w-full" + /> +
+ + +
+ +
+ + {Object.keys(params).length > 0 && ( +
+
+ + + + + + + + + {Object.entries(params).map(([key, val]) => ( + + + + + + ))} + +
KeyValueActions
{key}{val.toString()} + +
+
+ )} +
+ ); +} diff --git a/surfsense_web/components/onboard/add-provider-step.tsx b/surfsense_web/components/onboard/add-provider-step.tsx index e054a8cfe..6517fe0a1 100644 --- a/surfsense_web/components/onboard/add-provider-step.tsx +++ b/surfsense_web/components/onboard/add-provider-step.tsx @@ -17,37 +17,24 @@ import { SelectTrigger, SelectValue, } from "@/components/ui/select"; +import { LLM_PROVIDERS } from "@/contracts/enums/llm-providers"; +import { LANGUAGES } from "@/contracts/enums/languages"; import { type CreateLLMConfig, useLLMConfigs } from "@/hooks/use-llm-configs"; -const LLM_PROVIDERS = [ - { value: "OPENAI", label: "OpenAI", example: "gpt-4o, gpt-4, gpt-3.5-turbo" }, - { - value: "ANTHROPIC", - label: "Anthropic", - example: "claude-3-5-sonnet-20241022, claude-3-opus-20240229", - }, - { value: "GROQ", label: "Groq", example: "llama3-70b-8192, mixtral-8x7b-32768" }, - { value: "COHERE", label: "Cohere", example: "command-r-plus, command-r" }, - { value: "HUGGINGFACE", label: "HuggingFace", example: "microsoft/DialoGPT-medium" }, - { value: "AZURE_OPENAI", label: "Azure OpenAI", example: "gpt-4, gpt-35-turbo" }, - { value: "GOOGLE", label: "Google", example: "gemini-pro, gemini-pro-vision" }, - { value: "AWS_BEDROCK", label: "AWS Bedrock", example: "anthropic.claude-v2" }, - { value: "OLLAMA", label: "Ollama", example: "llama2, codellama" }, - { value: "MISTRAL", label: "Mistral", example: "mistral-large-latest, mistral-medium" }, - { value: "TOGETHER_AI", label: "Together AI", example: "togethercomputer/llama-2-70b-chat" }, - { value: "REPLICATE", label: "Replicate", example: "meta/llama-2-70b-chat" }, - { value: "OPENROUTER", label: "OpenRouter", example: "anthropic/claude-opus-4.1, openai/gpt-5" }, - { value: "COMETAPI", label: "CometAPI", example: "gpt-4o, claude-3-5-sonnet-20241022" }, - { value: "CUSTOM", label: "Custom Provider", example: "your-custom-model" }, -]; +import InferenceParamsEditor from "../inference-params-editor"; interface AddProviderStepProps { + searchSpaceId: number; onConfigCreated?: () => void; onConfigDeleted?: () => void; } -export function AddProviderStep({ onConfigCreated, onConfigDeleted }: AddProviderStepProps) { - const { llmConfigs, createLLMConfig, deleteLLMConfig } = useLLMConfigs(); +export function AddProviderStep({ + searchSpaceId, + onConfigCreated, + onConfigDeleted, +}: AddProviderStepProps) { + const { llmConfigs, createLLMConfig, deleteLLMConfig } = useLLMConfigs(searchSpaceId); const [isAddingNew, setIsAddingNew] = useState(false); const [formData, setFormData] = useState({ name: "", @@ -56,7 +43,9 @@ export function AddProviderStep({ onConfigCreated, onConfigDeleted }: AddProvide model_name: "", api_key: "", api_base: "", + language: "English", litellm_params: {}, + search_space_id: searchSpaceId, }); const [isSubmitting, setIsSubmitting] = useState(false); @@ -83,7 +72,9 @@ export function AddProviderStep({ onConfigCreated, onConfigDeleted }: AddProvide model_name: "", api_key: "", api_base: "", + language: "English", litellm_params: {}, + search_space_id: searchSpaceId, }); setIsAddingNew(false); // Notify parent component that a config was created @@ -93,6 +84,10 @@ export function AddProviderStep({ onConfigCreated, onConfigDeleted }: AddProvide const selectedProvider = LLM_PROVIDERS.find((p) => p.value === formData.provider); + const handleParamsChange = (newParams: Record) => { + setFormData((prev) => ({ ...prev, litellm_params: newParams })); + }; + return (
{/* Info Alert */} @@ -127,6 +122,7 @@ export function AddProviderStep({ onConfigCreated, onConfigDeleted }: AddProvide

Model: {config.model_name} + {config.language && ` • Language: ${config.language}`} {config.api_base && ` • Base: ${config.api_base}`}

@@ -177,7 +173,7 @@ export function AddProviderStep({ onConfigCreated, onConfigDeleted }: AddProvide
-
+
+ + {/* language */} +
+ + +
+
{formData.provider === "CUSTOM" && ( @@ -260,6 +277,14 @@ export function AddProviderStep({ onConfigCreated, onConfigDeleted }: AddProvide />
+ {/* Optional Inference Parameters */} +
+ +
+
@@ -516,7 +454,9 @@ export function ModelConfigManager() { model_name: "", api_key: "", api_base: "", + language: "", litellm_params: {}, + search_space_id: searchSpaceId, }); } }} @@ -549,22 +489,13 @@ export function ModelConfigManager() {
- + {formData.provider && ( -
-
- {LLM_PROVIDERS.find((p) => p.value === formData.provider)?.label} -
-
•
-
- {LLM_PROVIDERS.find((p) => p.value === formData.provider)?.description} -
-
+ + {LLM_PROVIDERS.find((p) => p.value === formData.provider)?.label} + )} @@ -613,6 +544,25 @@ export function ModelConfigManager() { )}
+
+ + +
+
- + handleInputChange("api_base", e.target.value)} /> + {selectedProvider?.apiBase && formData.api_base === selectedProvider.apiBase && ( +

+ + Using recommended API endpoint for {selectedProvider.label} +

+ )} + {selectedProvider?.apiBase && !formData.api_base && ( +

+ + ⚠️ API Base URL is required for {selectedProvider.label}. Click to auto-fill: + +

+ )} +
+ + {/* Optional Inference Parameters */} +
+ + setFormData((prev) => ({ ...prev, litellm_params: newParams })) + } + />
@@ -658,7 +644,9 @@ export function ModelConfigManager() { model_name: "", api_key: "", api_base: "", + language: "", litellm_params: {}, + search_space_id: searchSpaceId, }); }} disabled={isSubmitting} diff --git a/surfsense_web/contracts/enums/connector.ts b/surfsense_web/contracts/enums/connector.ts index a49314e2e..39c6fcfcf 100644 --- a/surfsense_web/contracts/enums/connector.ts +++ b/surfsense_web/contracts/enums/connector.ts @@ -1,6 +1,7 @@ export enum EnumConnectorName { SERPER_API = "SERPER_API", TAVILY_API = "TAVILY_API", + SEARXNG_API = "SEARXNG_API", LINKUP_API = "LINKUP_API", SLACK_CONNECTOR = "SLACK_CONNECTOR", NOTION_CONNECTOR = "NOTION_CONNECTOR", diff --git a/surfsense_web/contracts/enums/connectorIcons.tsx b/surfsense_web/contracts/enums/connectorIcons.tsx index f34a10398..1d17f49e1 100644 --- a/surfsense_web/contracts/enums/connectorIcons.tsx +++ b/surfsense_web/contracts/enums/connectorIcons.tsx @@ -32,6 +32,8 @@ export const getConnectorIcon = (connectorType: EnumConnectorName | string, clas return ; case EnumConnectorName.TAVILY_API: return ; + case EnumConnectorName.SEARXNG_API: + return ; case EnumConnectorName.SLACK_CONNECTOR: return ; case EnumConnectorName.NOTION_CONNECTOR: diff --git a/surfsense_web/contracts/enums/languages.ts b/surfsense_web/contracts/enums/languages.ts new file mode 100644 index 000000000..21a555a09 --- /dev/null +++ b/surfsense_web/contracts/enums/languages.ts @@ -0,0 +1,69 @@ +export interface Language { + value: string; + label: string; +} + +export const LANGUAGES: Language[] = [ + { value: "English", label: "English" }, + { value: "Spanish", label: "Spanish" }, + { value: "French", label: "French" }, + { value: "German", label: "German" }, + { value: "Italian", label: "Italian" }, + { value: "Portuguese", label: "Portuguese" }, + { value: "Russian", label: "Russian" }, + { value: "Chinese", label: "Chinese (Simplified)" }, + { value: "Chinese-traditional", label: "Chinese (Traditional)" }, + { value: "Japanese", label: "Japanese" }, + { value: "Korean", label: "Korean" }, + { value: "Arabic", label: "Arabic" }, + { value: "Hindi", label: "Hindi" }, + { value: "Dutch", label: "Dutch" }, + { value: "Swedish", label: "Swedish" }, + { value: "Norwegian", label: "Norwegian" }, + { value: "Danish", label: "Danish" }, + { value: "Finnish", label: "Finnish" }, + { value: "Polish", label: "Polish" }, + { value: "Czech", label: "Czech" }, + { value: "Hungarian", label: "Hungarian" }, + { value: "Romanian", label: "Romanian" }, + { value: "Bulgarian", label: "Bulgarian" }, + { value: "Croatian", label: "Croatian" }, + { value: "Serbian", label: "Serbian" }, + { value: "Slovenian", label: "Slovenian" }, + { value: "Slovak", label: "Slovak" }, + { value: "Lithuanian", label: "Lithuanian" }, + { value: "Latvian", label: "Latvian" }, + { value: "Estonian", label: "Estonian" }, + { value: "Greek", label: "Greek" }, + { value: "Turkish", label: "Turkish" }, + { value: "Hebrew", label: "Hebrew" }, + { value: "Thai", label: "Thai" }, + { value: "Vietnamese", label: "Vietnamese" }, + { value: "Indonesian", label: "Indonesian" }, + { value: "Malay", label: "Malay" }, + { value: "Tagalog", label: "Filipino/Tagalog" }, + { value: "Bengali", label: "Bengali" }, + { value: "Tamil", label: "Tamil" }, + { value: "Telugu", label: "Telugu" }, + { value: "Marathi", label: "Marathi" }, + { value: "Gujarati", label: "Gujarati" }, + { value: "Kannada", label: "Kannada" }, + { value: "Malayalam", label: "Malayalam" }, + { value: "Punjabi", label: "Punjabi" }, + { value: "Urdu", label: "Urdu" }, + { value: "Persian", label: "Persian/Farsi" }, + { value: "Swahili", label: "Swahili" }, + { value: "Afrikaans", label: "Afrikaans" }, + { value: "Amharic", label: "Amharic" }, + { value: "Ukrainian", label: "Ukrainian" }, + { value: "Belarusian", label: "Belarusian" }, + { value: "Georgian", label: "Georgian" }, + { value: "Armenian", label: "Armenian" }, + { value: "Azerbaijani", label: "Azerbaijani" }, + { value: "Kazakh", label: "Kazakh" }, + { value: "Uzbek", label: "Uzbek" }, + { value: "Kyrgyz", label: "Kyrgyz" }, + { value: "Tajik", label: "Tajik" }, + { value: "Turkmen", label: "Turkmen" }, + { value: "Mongolian", label: "Mongolian" }, +]; diff --git a/surfsense_web/contracts/enums/llm-providers.ts b/surfsense_web/contracts/enums/llm-providers.ts new file mode 100644 index 000000000..814ea16cf --- /dev/null +++ b/surfsense_web/contracts/enums/llm-providers.ts @@ -0,0 +1,129 @@ +export interface LLMProvider { + value: string; + label: string; + example: string; + description: string; + apiBase?: string; // Default API Base URL for the provider / 提供商的默认 API Base URL +} + +export const LLM_PROVIDERS: LLMProvider[] = [ + { + value: "OPENAI", + label: "OpenAI", + example: "gpt-4o, gpt-4, gpt-3.5-turbo", + description: "Industry-leading GPT models with broad capabilities", + }, + { + value: "ANTHROPIC", + label: "Anthropic", + example: "claude-3-5-sonnet-20241022, claude-3-opus-20240229", + description: "Claude models with strong reasoning and long context windows", + }, + { + value: "GROQ", + label: "Groq", + example: "llama3-70b-8192, mixtral-8x7b-32768", + description: "Lightning-fast inference with custom LPU hardware", + }, + { + value: "COHERE", + label: "Cohere", + example: "command-r-plus, command-r", + description: "Enterprise NLP models optimized for business applications", + }, + { + value: "HUGGINGFACE", + label: "HuggingFace", + example: "microsoft/DialoGPT-medium", + description: "Access thousands of open-source models", + }, + { + value: "AZURE_OPENAI", + label: "Azure OpenAI", + example: "gpt-4, gpt-35-turbo", + description: "OpenAI models with Microsoft Azure enterprise features", + }, + { + value: "GOOGLE", + label: "Google", + example: "gemini-pro, gemini-pro-vision", + description: "Gemini models with multimodal capabilities", + }, + { + value: "AWS_BEDROCK", + label: "AWS Bedrock", + example: "anthropic.claude-v2", + description: "Fully managed foundation models on AWS infrastructure", + }, + { + value: "OLLAMA", + label: "Ollama", + example: "llama2, codellama", + description: "Run open-source models locally on your machine", + }, + { + value: "MISTRAL", + label: "Mistral", + example: "mistral-large-latest, mistral-medium", + description: "High-performance open-source models from Europe", + }, + { + value: "TOGETHER_AI", + label: "Together AI", + example: "togethercomputer/llama-2-70b-chat", + description: "Scalable cloud platform for open-source models", + }, + { + value: "REPLICATE", + label: "Replicate", + example: "meta/llama-2-70b-chat", + description: "Cloud API for running machine learning models", + }, + { + value: "OPENROUTER", + label: "OpenRouter", + example: "anthropic/claude-opus-4.1, openai/gpt-5", + description: "Unified API gateway for multiple LLM providers", + }, + { + value: "COMETAPI", + label: "CometAPI", + example: "gpt-5-mini, claude-sonnet-4-5", + description: "Access 500+ AI models through one unified API", + }, + // Chinese LLM Providers / 国产 LLM 提供商 + { + value: "DEEPSEEK", + label: "DeepSeek", + example: "deepseek-chat, deepseek-coder", + description: "Chinese high-performance AI models", + apiBase: "https://api.deepseek.com", + }, + { + value: "ALIBABA_QWEN", + label: "Qwen", + example: "qwen-max, qwen-plus, qwen-turbo", + description: "Alibaba Cloud Qwen LLM", + apiBase: "https://dashscope.aliyuncs.com/compatible-mode/v1", + }, + { + value: "MOONSHOT", + label: "Kimi", + example: "moonshot-v1-8k, moonshot-v1-32k, moonshot-v1-128k", + description: "Moonshot AI Kimi models", + apiBase: "https://api.moonshot.cn/v1", + }, + { + value: "ZHIPU", + label: "GLM", + example: "glm-4, glm-4-flash, glm-3-turbo", + description: "Zhipu AI GLM series models", + apiBase: "https://open.bigmodel.cn/api/paas/v4", + }, + { + value: "CUSTOM", + label: "Custom Provider", + example: "your-custom-model", + description: "Connect to your own custom model endpoint", + }, +]; diff --git a/surfsense_web/hooks/index.ts b/surfsense_web/hooks/index.ts index 908d2adf5..76d96f6c7 100644 --- a/surfsense_web/hooks/index.ts +++ b/surfsense_web/hooks/index.ts @@ -1,3 +1,3 @@ export * from "./use-document-by-chunk"; export * from "./use-logs"; -export * from "./useSearchSourceConnectors"; +export * from "./use-search-source-connectors"; diff --git a/surfsense_web/hooks/useChat.ts b/surfsense_web/hooks/use-chat.ts similarity index 100% rename from surfsense_web/hooks/useChat.ts rename to surfsense_web/hooks/use-chat.ts diff --git a/surfsense_web/hooks/useConnectorEditPage.ts b/surfsense_web/hooks/use-connector-edit-page.ts similarity index 70% rename from surfsense_web/hooks/useConnectorEditPage.ts rename to surfsense_web/hooks/use-connector-edit-page.ts index fb5b005a9..ea6a3fe10 100644 --- a/surfsense_web/hooks/useConnectorEditPage.ts +++ b/surfsense_web/hooks/use-connector-edit-page.ts @@ -14,11 +14,47 @@ import { import { type SearchSourceConnector, useSearchSourceConnectors, -} from "@/hooks/useSearchSourceConnectors"; +} from "@/hooks/use-search-source-connectors"; + +const normalizeListInput = (value: unknown): string[] => { + if (Array.isArray(value)) { + return value.map((item) => String(item).trim()).filter((item) => item.length > 0); + } + if (typeof value === "string") { + return value + .split(",") + .map((item) => item.trim()) + .filter((item) => item.length > 0); + } + return []; +}; + +const arraysEqual = (a: string[], b: string[]): boolean => { + if (a.length !== b.length) return false; + return a.every((value, index) => value === b[index]); +}; + +const normalizeBoolean = (value: unknown): boolean | null => { + if (typeof value === "boolean") return value; + if (typeof value === "string") { + const lowered = value.trim().toLowerCase(); + if (["true", "1", "yes", "on"].includes(lowered)) return true; + if (["false", "0", "no", "off"].includes(lowered)) return false; + } + if (typeof value === "number") { + if (value === 1) return true; + if (value === 0) return false; + } + return null; +}; export function useConnectorEditPage(connectorId: number, searchSpaceId: string) { const router = useRouter(); - const { connectors, updateConnector, isLoading: connectorsLoading } = useSearchSourceConnectors(); + const { + connectors, + updateConnector, + isLoading: connectorsLoading, + } = useSearchSourceConnectors(false, parseInt(searchSpaceId)); // State managed by the hook const [connector, setConnector] = useState(null); @@ -44,6 +80,13 @@ export function useConnectorEditPage(connectorId: number, searchSpaceId: string) NOTION_INTEGRATION_TOKEN: "", SERPER_API_KEY: "", TAVILY_API_KEY: "", + SEARXNG_HOST: "", + SEARXNG_API_KEY: "", + SEARXNG_ENGINES: "", + SEARXNG_CATEGORIES: "", + SEARXNG_LANGUAGE: "", + SEARXNG_SAFESEARCH: "", + SEARXNG_VERIFY_SSL: "", LINEAR_API_KEY: "", DISCORD_BOT_TOKEN: "", CONFLUENCE_BASE_URL: "", @@ -70,6 +113,23 @@ export function useConnectorEditPage(connectorId: number, searchSpaceId: string) NOTION_INTEGRATION_TOKEN: config.NOTION_INTEGRATION_TOKEN || "", SERPER_API_KEY: config.SERPER_API_KEY || "", TAVILY_API_KEY: config.TAVILY_API_KEY || "", + SEARXNG_HOST: config.SEARXNG_HOST || "", + SEARXNG_API_KEY: config.SEARXNG_API_KEY || "", + SEARXNG_ENGINES: Array.isArray(config.SEARXNG_ENGINES) + ? config.SEARXNG_ENGINES.join(", ") + : config.SEARXNG_ENGINES || "", + SEARXNG_CATEGORIES: Array.isArray(config.SEARXNG_CATEGORIES) + ? config.SEARXNG_CATEGORIES.join(", ") + : config.SEARXNG_CATEGORIES || "", + SEARXNG_LANGUAGE: config.SEARXNG_LANGUAGE || "", + SEARXNG_SAFESEARCH: + config.SEARXNG_SAFESEARCH !== undefined && config.SEARXNG_SAFESEARCH !== null + ? String(config.SEARXNG_SAFESEARCH) + : "", + SEARXNG_VERIFY_SSL: + config.SEARXNG_VERIFY_SSL !== undefined && config.SEARXNG_VERIFY_SSL !== null + ? String(config.SEARXNG_VERIFY_SSL) + : "", LINEAR_API_KEY: config.LINEAR_API_KEY || "", LINKUP_API_KEY: config.LINKUP_API_KEY || "", DISCORD_BOT_TOKEN: config.DISCORD_BOT_TOKEN || "", @@ -234,6 +294,93 @@ export function useConnectorEditPage(connectorId: number, searchSpaceId: string) newConfig = { TAVILY_API_KEY: formData.TAVILY_API_KEY }; } break; + case "SEARXNG_API": { + const host = (formData.SEARXNG_HOST || "").trim(); + if (!host) { + toast.error("SearxNG host is required."); + setIsSaving(false); + return; + } + + const candidateConfig: Record = { SEARXNG_HOST: host }; + let hasChanges = host !== (originalConfig.SEARXNG_HOST || "").trim(); + + const apiKey = (formData.SEARXNG_API_KEY || "").trim(); + const originalApiKey = (originalConfig.SEARXNG_API_KEY || "").trim(); + if (apiKey !== originalApiKey) { + candidateConfig.SEARXNG_API_KEY = apiKey || null; + hasChanges = true; + } + + const newEngines = normalizeListInput(formData.SEARXNG_ENGINES || ""); + const originalEngines = normalizeListInput(originalConfig.SEARXNG_ENGINES); + if (!arraysEqual(newEngines, originalEngines)) { + candidateConfig.SEARXNG_ENGINES = newEngines; + hasChanges = true; + } + + const newCategories = normalizeListInput(formData.SEARXNG_CATEGORIES || ""); + const originalCategories = normalizeListInput(originalConfig.SEARXNG_CATEGORIES); + if (!arraysEqual(newCategories, originalCategories)) { + candidateConfig.SEARXNG_CATEGORIES = newCategories; + hasChanges = true; + } + + const language = (formData.SEARXNG_LANGUAGE || "").trim(); + const originalLanguage = (originalConfig.SEARXNG_LANGUAGE || "").trim(); + if (language !== originalLanguage) { + candidateConfig.SEARXNG_LANGUAGE = language || null; + hasChanges = true; + } + + const safesearchRaw = (formData.SEARXNG_SAFESEARCH || "").trim(); + const originalSafesearch = originalConfig.SEARXNG_SAFESEARCH; + if (safesearchRaw) { + const parsed = Number(safesearchRaw); + if ( + Number.isNaN(parsed) || + !Number.isInteger(parsed) || + parsed < 0 || + parsed > 2 + ) { + toast.error("SearxNG SafeSearch must be 0, 1, or 2."); + setIsSaving(false); + return; + } + if (parsed !== Number(originalSafesearch)) { + candidateConfig.SEARXNG_SAFESEARCH = parsed; + hasChanges = true; + } + } else if (originalSafesearch !== undefined && originalSafesearch !== null) { + candidateConfig.SEARXNG_SAFESEARCH = null; + hasChanges = true; + } + + const verifyRaw = (formData.SEARXNG_VERIFY_SSL || "").trim().toLowerCase(); + const originalVerifyBool = normalizeBoolean(originalConfig.SEARXNG_VERIFY_SSL); + if (verifyRaw) { + let parsedBool: boolean | null = null; + if (["true", "1", "yes", "on"].includes(verifyRaw)) parsedBool = true; + else if (["false", "0", "no", "off"].includes(verifyRaw)) parsedBool = false; + if (parsedBool === null) { + toast.error("SearxNG SSL verification must be true or false."); + setIsSaving(false); + return; + } + if (parsedBool !== originalVerifyBool) { + candidateConfig.SEARXNG_VERIFY_SSL = parsedBool; + hasChanges = true; + } + } else if (originalVerifyBool !== null) { + candidateConfig.SEARXNG_VERIFY_SSL = null; + hasChanges = true; + } + + if (hasChanges) { + newConfig = candidateConfig; + } + break; + } case "LINEAR_CONNECTOR": if (formData.LINEAR_API_KEY !== originalConfig.LINEAR_API_KEY) { @@ -363,6 +510,33 @@ export function useConnectorEditPage(connectorId: number, searchSpaceId: string) editForm.setValue("SERPER_API_KEY", newlySavedConfig.SERPER_API_KEY || ""); } else if (connector.connector_type === "TAVILY_API") { editForm.setValue("TAVILY_API_KEY", newlySavedConfig.TAVILY_API_KEY || ""); + } else if (connector.connector_type === "SEARXNG_API") { + editForm.setValue("SEARXNG_HOST", newlySavedConfig.SEARXNG_HOST || ""); + editForm.setValue("SEARXNG_API_KEY", newlySavedConfig.SEARXNG_API_KEY || ""); + editForm.setValue( + "SEARXNG_ENGINES", + normalizeListInput(newlySavedConfig.SEARXNG_ENGINES).join(", ") + ); + editForm.setValue( + "SEARXNG_CATEGORIES", + normalizeListInput(newlySavedConfig.SEARXNG_CATEGORIES).join(", ") + ); + editForm.setValue( + "SEARXNG_LANGUAGE", + newlySavedConfig.SEARXNG_LANGUAGE || "" + ); + editForm.setValue( + "SEARXNG_SAFESEARCH", + newlySavedConfig.SEARXNG_SAFESEARCH === null || + newlySavedConfig.SEARXNG_SAFESEARCH === undefined + ? "" + : String(newlySavedConfig.SEARXNG_SAFESEARCH) + ); + const verifyValue = normalizeBoolean(newlySavedConfig.SEARXNG_VERIFY_SSL); + editForm.setValue( + "SEARXNG_VERIFY_SSL", + verifyValue === null ? "" : String(verifyValue) + ); } else if (connector.connector_type === "LINEAR_CONNECTOR") { editForm.setValue("LINEAR_API_KEY", newlySavedConfig.LINEAR_API_KEY || ""); } else if (connector.connector_type === "LINKUP_API") { diff --git a/surfsense_web/hooks/use-connectors.ts b/surfsense_web/hooks/use-connectors.ts index 9e9f2dc53..63ee7b372 100644 --- a/surfsense_web/hooks/use-connectors.ts +++ b/surfsense_web/hooks/use-connectors.ts @@ -23,7 +23,7 @@ export const getConnectorTypeDisplay = (type: string): string => { const typeMap: Record = { SERPER_API: "Serper API", TAVILY_API: "Tavily API", - // Add other connector types here as needed + SEARXNG_API: "SearxNG", }; return typeMap[type] || type; }; diff --git a/surfsense_web/hooks/use-llm-configs.ts b/surfsense_web/hooks/use-llm-configs.ts index ccb3a5bc9..73950736d 100644 --- a/surfsense_web/hooks/use-llm-configs.ts +++ b/surfsense_web/hooks/use-llm-configs.ts @@ -10,9 +10,10 @@ export interface LLMConfig { model_name: string; api_key: string; api_base?: string; + language?: string; litellm_params?: Record; created_at: string; - user_id: string; + search_space_id: number; } export interface LLMPreferences { @@ -31,7 +32,9 @@ export interface CreateLLMConfig { model_name: string; api_key: string; api_base?: string; + language?: string; litellm_params?: Record; + search_space_id: number; } export interface UpdateLLMConfig { @@ -44,16 +47,21 @@ export interface UpdateLLMConfig { litellm_params?: Record; } -export function useLLMConfigs() { +export function useLLMConfigs(searchSpaceId: number | null) { const [llmConfigs, setLlmConfigs] = useState([]); const [loading, setLoading] = useState(true); const [error, setError] = useState(null); const fetchLLMConfigs = async () => { + if (!searchSpaceId) { + setLoading(false); + return; + } + try { setLoading(true); const response = await fetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/llm-configs/`, + `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/llm-configs/?search_space_id=${searchSpaceId}`, { headers: { Authorization: `Bearer ${localStorage.getItem("surfsense_bearer_token")}`, @@ -79,7 +87,7 @@ export function useLLMConfigs() { useEffect(() => { fetchLLMConfigs(); - }, []); + }, [searchSpaceId]); const createLLMConfig = async (config: CreateLLMConfig): Promise => { try { @@ -181,16 +189,21 @@ export function useLLMConfigs() { }; } -export function useLLMPreferences() { +export function useLLMPreferences(searchSpaceId: number | null) { const [preferences, setPreferences] = useState({}); const [loading, setLoading] = useState(true); const [error, setError] = useState(null); const fetchPreferences = async () => { + if (!searchSpaceId) { + setLoading(false); + return; + } + try { setLoading(true); const response = await fetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/users/me/llm-preferences`, + `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-spaces/${searchSpaceId}/llm-preferences`, { headers: { Authorization: `Bearer ${localStorage.getItem("surfsense_bearer_token")}`, @@ -216,12 +229,17 @@ export function useLLMPreferences() { useEffect(() => { fetchPreferences(); - }, []); + }, [searchSpaceId]); const updatePreferences = async (newPreferences: Partial): Promise => { + if (!searchSpaceId) { + toast.error("Search space ID is required"); + return false; + } + try { const response = await fetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/users/me/llm-preferences`, + `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-spaces/${searchSpaceId}/llm-preferences`, { method: "PUT", headers: { diff --git a/surfsense_web/hooks/useSearchSourceConnectors.ts b/surfsense_web/hooks/use-search-source-connectors.ts similarity index 73% rename from surfsense_web/hooks/useSearchSourceConnectors.ts rename to surfsense_web/hooks/use-search-source-connectors.ts index 0c59a3bf0..5a45fd761 100644 --- a/surfsense_web/hooks/useSearchSourceConnectors.ts +++ b/surfsense_web/hooks/use-search-source-connectors.ts @@ -7,6 +7,7 @@ export interface SearchSourceConnector { is_indexable: boolean; last_indexed_at: string | null; config: Record; + search_space_id: number; user_id?: string; created_at?: string; } @@ -20,8 +21,10 @@ export interface ConnectorSourceItem { /** * Hook to fetch search source connectors from the API + * @param lazy - If true, connectors won't be fetched on mount + * @param searchSpaceId - Optional search space ID to filter connectors */ -export const useSearchSourceConnectors = (lazy: boolean = false) => { +export const useSearchSourceConnectors = (lazy: boolean = false, searchSpaceId?: number) => { const [connectors, setConnectors] = useState([]); const [isLoading, setIsLoading] = useState(!lazy); // Don't show loading initially for lazy mode const [isLoaded, setIsLoaded] = useState(false); // Memoization flag @@ -53,60 +56,71 @@ export const useSearchSourceConnectors = (lazy: boolean = false) => { }, ]); - const fetchConnectors = useCallback(async () => { - if (isLoaded && lazy) return; // Avoid redundant calls in lazy mode + const fetchConnectors = useCallback( + async (spaceId?: number) => { + if (isLoaded && lazy) return; // Avoid redundant calls in lazy mode - try { - setIsLoading(true); - setError(null); - const token = localStorage.getItem("surfsense_bearer_token"); + try { + setIsLoading(true); + setError(null); + const token = localStorage.getItem("surfsense_bearer_token"); - if (!token) { - throw new Error("No authentication token found"); - } + if (!token) { + throw new Error("No authentication token found"); + } - const response = await fetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-source-connectors/`, - { + // Build URL with optional search_space_id query parameter + const url = new URL( + `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-source-connectors/` + ); + if (spaceId !== undefined) { + url.searchParams.append("search_space_id", spaceId.toString()); + } + + const response = await fetch(url.toString(), { method: "GET", headers: { "Content-Type": "application/json", Authorization: `Bearer ${token}`, }, + }); + + if (!response.ok) { + throw new Error(`Failed to fetch connectors: ${response.statusText}`); } - ); - if (!response.ok) { - throw new Error(`Failed to fetch connectors: ${response.statusText}`); + const data = await response.json(); + setConnectors(data); + setIsLoaded(true); + + // Update connector source items when connectors change + updateConnectorSourceItems(data); + + return data; + } catch (err) { + setError(err instanceof Error ? err : new Error("An unknown error occurred")); + console.error("Error fetching search source connectors:", err); + } finally { + setIsLoading(false); } - - const data = await response.json(); - setConnectors(data); - setIsLoaded(true); - - // Update connector source items when connectors change - updateConnectorSourceItems(data); - - return data; - } catch (err) { - setError(err instanceof Error ? err : new Error("An unknown error occurred")); - console.error("Error fetching search source connectors:", err); - } finally { - setIsLoading(false); - } - }, [isLoaded, lazy]); + }, + [isLoaded, lazy] + ); useEffect(() => { if (!lazy) { - fetchConnectors(); + fetchConnectors(searchSpaceId); } - }, [lazy, fetchConnectors]); + }, [lazy, fetchConnectors, searchSpaceId]); // Function to refresh the connectors list - const refreshConnectors = useCallback(async () => { - setIsLoaded(false); // Reset memoization flag to allow refetch - await fetchConnectors(); - }, [fetchConnectors]); + const refreshConnectors = useCallback( + async (spaceId?: number) => { + setIsLoaded(false); // Reset memoization flag to allow refetch + await fetchConnectors(spaceId !== undefined ? spaceId : searchSpaceId); + }, + [fetchConnectors, searchSpaceId] + ); // Update connector source items when connectors change const updateConnectorSourceItems = (currentConnectors: SearchSourceConnector[]) => { @@ -151,9 +165,12 @@ export const useSearchSourceConnectors = (lazy: boolean = false) => { /** * Create a new search source connector + * @param connectorData - The connector data (excluding search_space_id) + * @param spaceId - The search space ID to associate the connector with */ const createConnector = async ( - connectorData: Omit + connectorData: Omit, + spaceId: number ) => { try { const token = localStorage.getItem("surfsense_bearer_token"); @@ -162,17 +179,20 @@ export const useSearchSourceConnectors = (lazy: boolean = false) => { throw new Error("No authentication token found"); } - const response = await fetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-source-connectors/`, - { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${token}`, - }, - body: JSON.stringify(connectorData), - } + // Add search_space_id as a query parameter + const url = new URL( + `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-source-connectors/` ); + url.searchParams.append("search_space_id", spaceId.toString()); + + const response = await fetch(url.toString(), { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${token}`, + }, + body: JSON.stringify(connectorData), + }); if (!response.ok) { throw new Error(`Failed to create connector: ${response.statusText}`); @@ -194,7 +214,9 @@ export const useSearchSourceConnectors = (lazy: boolean = false) => { */ const updateConnector = async ( connectorId: number, - connectorData: Partial> + connectorData: Partial< + Omit + > ) => { try { const token = localStorage.getItem("surfsense_bearer_token"); diff --git a/surfsense_web/lib/connectors/utils.ts b/surfsense_web/lib/connectors/utils.ts index 798204d94..5b542d32d 100644 --- a/surfsense_web/lib/connectors/utils.ts +++ b/surfsense_web/lib/connectors/utils.ts @@ -3,6 +3,7 @@ export const getConnectorTypeDisplay = (type: string): string => { const typeMap: Record = { SERPER_API: "Serper API", TAVILY_API: "Tavily API", + SEARXNG_API: "SearxNG", SLACK_CONNECTOR: "Slack", NOTION_CONNECTOR: "Notion", GITHUB_CONNECTOR: "GitHub",