From 0eeb0356e29ad4b8934490c5aac83b8b44c7f746 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Sun, 5 Oct 2025 13:53:22 -0700 Subject: [PATCH 01/35] refactor: renameed hooks --- .../dashboard/[search_space_id]/connectors/(manage)/page.tsx | 2 +- .../[search_space_id]/connectors/[connector_id]/edit/page.tsx | 2 +- .../[search_space_id]/connectors/[connector_id]/page.tsx | 2 +- .../connectors/add/airtable-connector/page.tsx | 2 +- .../[search_space_id]/connectors/add/clickup-connector/page.tsx | 2 +- .../connectors/add/confluence-connector/page.tsx | 2 +- .../[search_space_id]/connectors/add/discord-connector/page.tsx | 2 +- .../[search_space_id]/connectors/add/github-connector/page.tsx | 2 +- .../connectors/add/google-calendar-connector/page.tsx | 2 +- .../connectors/add/google-gmail-connector/page.tsx | 2 +- .../[search_space_id]/connectors/add/jira-connector/page.tsx | 2 +- .../[search_space_id]/connectors/add/linear-connector/page.tsx | 2 +- .../[search_space_id]/connectors/add/linkup-api/page.tsx | 2 +- .../[search_space_id]/connectors/add/luma-connector/page.tsx | 2 +- .../[search_space_id]/connectors/add/notion-connector/page.tsx | 2 +- .../[search_space_id]/connectors/add/serper-api/page.tsx | 2 +- .../[search_space_id]/connectors/add/slack-connector/page.tsx | 2 +- .../[search_space_id]/connectors/add/tavily-api/page.tsx | 2 +- .../[search_space_id]/researcher/[[...chat_id]]/page.tsx | 2 +- surfsense_web/components/chat/ChatInputGroup.tsx | 2 +- surfsense_web/hooks/index.ts | 2 +- surfsense_web/hooks/{useChat.ts => use-chat.ts} | 0 .../{useConnectorEditPage.ts => use-connector-edit-page.ts} | 2 +- ...earchSourceConnectors.ts => use-search-source-connectors.ts} | 0 24 files changed, 22 insertions(+), 22 deletions(-) rename surfsense_web/hooks/{useChat.ts => use-chat.ts} (100%) rename surfsense_web/hooks/{useConnectorEditPage.ts => use-connector-edit-page.ts} (99%) rename surfsense_web/hooks/{useSearchSourceConnectors.ts => use-search-source-connectors.ts} (100%) diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx index e7e195759..9786263fa 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx @@ -41,7 +41,7 @@ import { import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { useSearchSourceConnectors } from "@/hooks/useSearchSourceConnectors"; +import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; import { cn } from "@/lib/utils"; // Helper function to format date with time diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/edit/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/edit/page.tsx index 75248e4d0..87844e8c8 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/edit/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/edit/page.tsx @@ -20,7 +20,7 @@ import { } from "@/components/ui/card"; import { Form } from "@/components/ui/form"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { useConnectorEditPage } from "@/hooks/useConnectorEditPage"; +import { useConnectorEditPage } from "@/hooks/use-connector-edit-page"; // Import Utils, Types, Hook, and Components import { getConnectorTypeDisplay } from "@/lib/connectors/utils"; diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/page.tsx index ae8910f95..1e1a58453 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/page.tsx @@ -24,7 +24,7 @@ import { Input } from "@/components/ui/input"; import { type SearchSourceConnector, useSearchSourceConnectors, -} from "@/hooks/useSearchSourceConnectors"; +} from "@/hooks/use-search-source-connectors"; // Define the form schema with Zod const apiConnectorFormSchema = z.object({ diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/airtable-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/airtable-connector/page.tsx index 18163482c..cd81474ba 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/airtable-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/airtable-connector/page.tsx @@ -21,7 +21,7 @@ import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; import { type SearchSourceConnector, useSearchSourceConnectors, -} from "@/hooks/useSearchSourceConnectors"; +} from "@/hooks/use-search-source-connectors"; export default function AirtableConnectorPage() { const router = useRouter(); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/clickup-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/clickup-connector/page.tsx index 0914a1c00..d7ce20cbf 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/clickup-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/clickup-connector/page.tsx @@ -22,7 +22,7 @@ import { import { Input } from "@/components/ui/input"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { useSearchSourceConnectors } from "@/hooks/useSearchSourceConnectors"; +import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; // Define the form schema with Zod const clickupConnectorFormSchema = z.object({ diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/confluence-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/confluence-connector/page.tsx index d135b91cc..e4784d22c 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/confluence-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/confluence-connector/page.tsx @@ -24,7 +24,7 @@ import { Input } from "@/components/ui/input"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { useSearchSourceConnectors } from "@/hooks/useSearchSourceConnectors"; +import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; // Define the form schema with Zod const confluenceConnectorFormSchema = z.object({ diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/discord-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/discord-connector/page.tsx index 2f01e52f4..47366bc01 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/discord-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/discord-connector/page.tsx @@ -37,7 +37,7 @@ import { Input } from "@/components/ui/input"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { useSearchSourceConnectors } from "@/hooks/useSearchSourceConnectors"; +import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; // Define the form schema with Zod const discordConnectorFormSchema = z.object({ diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/github-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/github-connector/page.tsx index 7491b06ef..602cf066f 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/github-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/github-connector/page.tsx @@ -39,7 +39,7 @@ import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; // Assuming useSearchSourceConnectors hook exists and works similarly -import { useSearchSourceConnectors } from "@/hooks/useSearchSourceConnectors"; +import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; // Define the form schema with Zod for GitHub PAT entry step const githubPatFormSchema = z.object({ diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-calendar-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-calendar-connector/page.tsx index 4bb33c3f6..a190af7b3 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-calendar-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-calendar-connector/page.tsx @@ -23,7 +23,7 @@ import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; import { type SearchSourceConnector, useSearchSourceConnectors, -} from "@/hooks/useSearchSourceConnectors"; +} from "@/hooks/use-search-source-connectors"; export default function GoogleCalendarConnectorPage() { const router = useRouter(); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-gmail-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-gmail-connector/page.tsx index 7ce2b645a..573650db9 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-gmail-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-gmail-connector/page.tsx @@ -23,7 +23,7 @@ import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; import { type SearchSourceConnector, useSearchSourceConnectors, -} from "@/hooks/useSearchSourceConnectors"; +} from "@/hooks/use-search-source-connectors"; export default function GoogleGmailConnectorPage() { const router = useRouter(); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/jira-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/jira-connector/page.tsx index 174047a15..ea99c07d6 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/jira-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/jira-connector/page.tsx @@ -37,7 +37,7 @@ import { Input } from "@/components/ui/input"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { useSearchSourceConnectors } from "@/hooks/useSearchSourceConnectors"; +import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; // Define the form schema with Zod const jiraConnectorFormSchema = z.object({ diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/linear-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/linear-connector/page.tsx index d5d940b9d..71cc97ded 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/linear-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/linear-connector/page.tsx @@ -37,7 +37,7 @@ import { Input } from "@/components/ui/input"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { useSearchSourceConnectors } from "@/hooks/useSearchSourceConnectors"; +import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; // Define the form schema with Zod const linearConnectorFormSchema = z.object({ diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/linkup-api/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/linkup-api/page.tsx index 311ea2c3f..f68fee18c 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/linkup-api/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/linkup-api/page.tsx @@ -30,7 +30,7 @@ import { import { Input } from "@/components/ui/input"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { useSearchSourceConnectors } from "@/hooks/useSearchSourceConnectors"; +import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; // Define the form schema with Zod const linkupApiFormSchema = z.object({ diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/luma-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/luma-connector/page.tsx index 864c7fdba..176aadb74 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/luma-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/luma-connector/page.tsx @@ -33,7 +33,7 @@ import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; import { type SearchSourceConnector, useSearchSourceConnectors, -} from "@/hooks/useSearchSourceConnectors"; +} from "@/hooks/use-search-source-connectors"; // Define the form schema with Zod const lumaConnectorFormSchema = z.object({ diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/notion-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/notion-connector/page.tsx index 2a447e611..d9a9bdf52 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/notion-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/notion-connector/page.tsx @@ -37,7 +37,7 @@ import { Input } from "@/components/ui/input"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { useSearchSourceConnectors } from "@/hooks/useSearchSourceConnectors"; +import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; // Define the form schema with Zod const notionConnectorFormSchema = z.object({ diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/serper-api/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/serper-api/page.tsx index 0d38d4314..f594406ed 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/serper-api/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/serper-api/page.tsx @@ -30,7 +30,7 @@ import { import { Input } from "@/components/ui/input"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { useSearchSourceConnectors } from "@/hooks/useSearchSourceConnectors"; +import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; // Define the form schema with Zod const serperApiFormSchema = z.object({ diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/slack-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/slack-connector/page.tsx index 1cf18a89c..e27c96da9 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/slack-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/slack-connector/page.tsx @@ -37,7 +37,7 @@ import { Input } from "@/components/ui/input"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { useSearchSourceConnectors } from "@/hooks/useSearchSourceConnectors"; +import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; // Define the form schema with Zod const slackConnectorFormSchema = z.object({ diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/tavily-api/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/tavily-api/page.tsx index 7fc80b10d..1afb7fbed 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/tavily-api/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/tavily-api/page.tsx @@ -30,7 +30,7 @@ import { import { Input } from "@/components/ui/input"; import { EnumConnectorName } from "@/contracts/enums/connector"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { useSearchSourceConnectors } from "@/hooks/useSearchSourceConnectors"; +import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; // Define the form schema with Zod const tavilyApiFormSchema = z.object({ diff --git a/surfsense_web/app/dashboard/[search_space_id]/researcher/[[...chat_id]]/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/researcher/[[...chat_id]]/page.tsx index a867f5f84..9d3065b0d 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/researcher/[[...chat_id]]/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/researcher/[[...chat_id]]/page.tsx @@ -5,8 +5,8 @@ import { useParams, useRouter } from "next/navigation"; import { useEffect, useMemo } from "react"; import type { ResearchMode } from "@/components/chat"; import ChatInterface from "@/components/chat/ChatInterface"; +import { useChatAPI, useChatState } from "@/hooks/use-chat"; import type { Document } from "@/hooks/use-documents"; -import { useChatAPI, useChatState } from "@/hooks/useChat"; export default function ResearcherPage() { const { search_space_id, chat_id } = useParams(); diff --git a/surfsense_web/components/chat/ChatInputGroup.tsx b/surfsense_web/components/chat/ChatInputGroup.tsx index 97584c230..fdc03e893 100644 --- a/surfsense_web/components/chat/ChatInputGroup.tsx +++ b/surfsense_web/components/chat/ChatInputGroup.tsx @@ -27,7 +27,7 @@ import { import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; import { type Document, useDocuments } from "@/hooks/use-documents"; import { useLLMConfigs, useLLMPreferences } from "@/hooks/use-llm-configs"; -import { useSearchSourceConnectors } from "@/hooks/useSearchSourceConnectors"; +import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; const DocumentSelector = React.memo( ({ diff --git a/surfsense_web/hooks/index.ts b/surfsense_web/hooks/index.ts index 908d2adf5..76d96f6c7 100644 --- a/surfsense_web/hooks/index.ts +++ b/surfsense_web/hooks/index.ts @@ -1,3 +1,3 @@ export * from "./use-document-by-chunk"; export * from "./use-logs"; -export * from "./useSearchSourceConnectors"; +export * from "./use-search-source-connectors"; diff --git a/surfsense_web/hooks/useChat.ts b/surfsense_web/hooks/use-chat.ts similarity index 100% rename from surfsense_web/hooks/useChat.ts rename to surfsense_web/hooks/use-chat.ts diff --git a/surfsense_web/hooks/useConnectorEditPage.ts b/surfsense_web/hooks/use-connector-edit-page.ts similarity index 99% rename from surfsense_web/hooks/useConnectorEditPage.ts rename to surfsense_web/hooks/use-connector-edit-page.ts index fb5b005a9..d31ccf928 100644 --- a/surfsense_web/hooks/useConnectorEditPage.ts +++ b/surfsense_web/hooks/use-connector-edit-page.ts @@ -14,7 +14,7 @@ import { import { type SearchSourceConnector, useSearchSourceConnectors, -} from "@/hooks/useSearchSourceConnectors"; +} from "@/hooks/use-search-source-connectors"; export function useConnectorEditPage(connectorId: number, searchSpaceId: string) { const router = useRouter(); diff --git a/surfsense_web/hooks/useSearchSourceConnectors.ts b/surfsense_web/hooks/use-search-source-connectors.ts similarity index 100% rename from surfsense_web/hooks/useSearchSourceConnectors.ts rename to surfsense_web/hooks/use-search-source-connectors.ts From 9f99b09e0da7b6cfc0ecdb53e09a58c87469697a Mon Sep 17 00:00:00 2001 From: Swayam Date: Mon, 6 Oct 2025 03:00:25 +0530 Subject: [PATCH 02/35] feat: feat: add interface params --- ...fd_add_inference_params_to_searchspaces.py | 38 +++++++++++++++++++ surfsense_backend/app/db.py | 2 + surfsense_backend/app/schemas/search_space.py | 13 ++++++- 3 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 surfsense_backend/alembic/versions/eb0c978da8fd_add_inference_params_to_searchspaces.py diff --git a/surfsense_backend/alembic/versions/eb0c978da8fd_add_inference_params_to_searchspaces.py b/surfsense_backend/alembic/versions/eb0c978da8fd_add_inference_params_to_searchspaces.py new file mode 100644 index 000000000..43ae4676f --- /dev/null +++ b/surfsense_backend/alembic/versions/eb0c978da8fd_add_inference_params_to_searchspaces.py @@ -0,0 +1,38 @@ +"""add inference_params to searchspaces + +Revision ID: eb0c978da8fd +Revises: 22 +Create Date: 2025-10-06 02:54:11.618100 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = 'eb0c978da8fd' +down_revision: Union[str, None] = '22' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f('chucks_search_index'), table_name='chunks', postgresql_using='gin') + op.drop_index(op.f('chucks_vector_index'), table_name='chunks', postgresql_using='hnsw') + op.drop_index(op.f('document_search_index'), table_name='documents', postgresql_using='gin') + op.drop_index(op.f('document_vector_index'), table_name='documents', postgresql_using='hnsw') + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.create_index(op.f('document_vector_index'), 'documents', ['embedding'], unique=False, postgresql_using='hnsw') + op.create_index(op.f('document_search_index'), 'documents', [sa.literal_column("to_tsvector('english'::regconfig, content)")], unique=False, postgresql_using='gin') + op.create_index(op.f('chucks_vector_index'), 'chunks', ['embedding'], unique=False, postgresql_using='hnsw') + op.create_index(op.f('chucks_search_index'), 'chunks', [sa.literal_column("to_tsvector('english'::regconfig, content)")], unique=False, postgresql_using='gin') + # ### end Alembic commands ### diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py index 99be9f277..2fe79f8d1 100644 --- a/surfsense_backend/app/db.py +++ b/surfsense_backend/app/db.py @@ -205,6 +205,8 @@ class SearchSpace(BaseModel, TimestampMixin): name = Column(String(100), nullable=False, index=True) description = Column(String(500), nullable=True) + inference_params = Column(JSON, nullable=True) + user_id = Column( UUID(as_uuid=True), ForeignKey("user.id", ondelete="CASCADE"), nullable=False ) diff --git a/surfsense_backend/app/schemas/search_space.py b/surfsense_backend/app/schemas/search_space.py index 00bfdc0f6..3dd8da4e8 100644 --- a/surfsense_backend/app/schemas/search_space.py +++ b/surfsense_backend/app/schemas/search_space.py @@ -1,14 +1,21 @@ import uuid from datetime import datetime -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel, ConfigDict, Field from .base import IDModel, TimestampModel +class InferenceParams(BaseModel): + temperature: float | None = Field(None, ge=0.0, le=2.0) + max_tokens: int | None = Field(None, ge=0) + top_k: int | None = Field(None, ge=0) + top_p: float | None = Field(None, ge=0.0, le=1.0) + class SearchSpaceBase(BaseModel): name: str description: str | None = None + inference_params: InferenceParams | None = None class SearchSpaceCreate(SearchSpaceBase): @@ -16,7 +23,9 @@ class SearchSpaceCreate(SearchSpaceBase): class SearchSpaceUpdate(SearchSpaceBase): - pass + name: str | None = None + description: str | None = None + inference_params: InferenceParams | None = None class SearchSpaceRead(SearchSpaceBase, IDModel, TimestampModel): From c257235cc3fabc7723e28a05a7b7a4d33f69cb7b Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Sun, 5 Oct 2025 21:47:22 -0700 Subject: [PATCH 03/35] refactor: added chunk id reference for sources --- .../components/chat/ChatCitation.tsx | 229 +--------------- surfsense_web/components/chat/ChatSources.tsx | 77 ++++-- .../components/chat/SourceDetailSheet.tsx | 244 ++++++++++++++++++ 3 files changed, 310 insertions(+), 240 deletions(-) create mode 100644 surfsense_web/components/chat/SourceDetailSheet.tsx diff --git a/surfsense_web/components/chat/ChatCitation.tsx b/surfsense_web/components/chat/ChatCitation.tsx index fe3c2b993..d8c681781 100644 --- a/surfsense_web/components/chat/ChatCitation.tsx +++ b/surfsense_web/components/chat/ChatCitation.tsx @@ -1,233 +1,30 @@ "use client"; -import { ChevronDown, ChevronUp, ExternalLink, FileText, Loader2 } from "lucide-react"; import type React from "react"; -import { useEffect, useRef, useState } from "react"; -import { MarkdownViewer } from "@/components/markdown-viewer"; -import { Button } from "@/components/ui/button"; -import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"; -import { ScrollArea } from "@/components/ui/scroll-area"; -import { - Sheet, - SheetContent, - SheetDescription, - SheetHeader, - SheetTitle, - SheetTrigger, -} from "@/components/ui/sheet"; -import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; -import { useDocumentByChunk } from "@/hooks/use-document-by-chunk"; -import { cn } from "@/lib/utils"; +import { useState } from "react"; +import { SheetTrigger } from "@/components/ui/sheet"; +import { SourceDetailSheet } from "./SourceDetailSheet"; export const CitationDisplay: React.FC<{ index: number; node: any }> = ({ index, node }) => { const chunkId = Number(node?.id); const sourceType = node?.metadata?.source_type; const [isOpen, setIsOpen] = useState(false); - const { document, loading, error, fetchDocumentByChunk, clearDocument } = useDocumentByChunk(); - const chunksContainerRef = useRef(null); - const highlightedChunkRef = useRef(null); - const [summaryOpen, setSummaryOpen] = useState(false); - - // Check if this is a source type that should render directly from node - const isDirectRenderSource = sourceType === "TAVILY_API" || sourceType === "LINKUP_API"; - - const handleOpenChange = async (open: boolean) => { - setIsOpen(open); - if (open && chunkId && !isDirectRenderSource) { - await fetchDocumentByChunk(chunkId); - } else if (!open && !isDirectRenderSource) { - clearDocument(); - } - }; - - useEffect(() => { - // Scroll to highlighted chunk when document loads - if (document && highlightedChunkRef.current && chunksContainerRef.current) { - setTimeout(() => { - highlightedChunkRef.current?.scrollIntoView({ - behavior: "smooth", - block: "start", - }); - }, 100); - } - }, [document]); - - const handleUrlClick = (e: React.MouseEvent, url: string) => { - e.preventDefault(); - e.stopPropagation(); - window.open(url, "_blank", "noopener,noreferrer"); - }; - - const formatDocumentType = (type: string) => { - return type - .split("_") - .map((word) => word.charAt(0) + word.slice(1).toLowerCase()) - .join(" "); - }; return ( - + {index + 1} - - - - {getConnectorIcon(sourceType)} - {document?.title || node?.metadata?.title || node?.metadata?.group_name || "Source"} - - - {document - ? formatDocumentType(document.document_type) - : sourceType && formatDocumentType(sourceType)} - - - - {!isDirectRenderSource && loading && ( -
- -
- )} - - {!isDirectRenderSource && error && ( -
-

{error}

-
- )} - - {/* Direct render for TAVILY_API and LINEAR_API */} - {isDirectRenderSource && ( - -
- {/* External Link */} - {node?.url && ( -
- -
- )} - - {/* Source Information */} -
-

Source Information

-
- {node?.metadata?.title || "Untitled"} -
-
- {node?.text || "No content available"} -
-
-
- - )} - - {/* API-fetched document content */} - {!isDirectRenderSource && document && ( - -
- {/* Document Metadata */} - {document.document_metadata && Object.keys(document.document_metadata).length > 0 && ( -
-

Document Information

-
- {Object.entries(document.document_metadata).map(([key, value]) => ( -
-
- {key.replace(/_/g, " ")}: -
-
{String(value)}
-
- ))} -
-
- )} - - {/* External Link */} - {node?.url && ( -
- -
- )} - - {/* Chunks */} -
-
- {/* Header row: header and button side by side */} -
-

Document Content

- {document.content && ( - - - Summary - {summaryOpen ? ( - - ) : ( - - )} - - - )} -
- {/* Expanded summary content: always full width, below the row */} - {document.content && ( - - -
- -
- - - )} -
- - {document.chunks.map((chunk, idx) => ( -
-
- - Chunk {idx + 1} of {document.chunks.length} - - {chunk.id === chunkId && ( - - Referenced Chunk - - )} -
-
- -
-
- ))} -
-
- - )} - - + ); }; diff --git a/surfsense_web/components/chat/ChatSources.tsx b/surfsense_web/components/chat/ChatSources.tsx index 625a81fa1..5f205d005 100644 --- a/surfsense_web/components/chat/ChatSources.tsx +++ b/surfsense_web/components/chat/ChatSources.tsx @@ -9,12 +9,14 @@ import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/com import { Sheet, SheetContent, SheetHeader, SheetTitle, SheetTrigger } from "@/components/ui/sheet"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; +import { SourceDetailSheet } from "./SourceDetailSheet"; interface Source { id: string; title: string; description: string; url: string; + sourceType: string; } interface SourceGroup { @@ -48,6 +50,9 @@ function getSourceIcon(type: string) { function SourceCard({ source }: { source: Source }) { const hasUrl = source.url && source.url.trim() !== ""; + const chunkId = Number(source.id); + const sourceType = source.sourceType; + const [isOpen, setIsOpen] = useState(false); // Clean up the description for better display const cleanDescription = source.description @@ -55,31 +60,54 @@ function SourceCard({ source }: { source: Source }) { .replace(/\n+/g, " ") .trim(); + const handleUrlClick = (e: React.MouseEvent, url: string) => { + e.preventDefault(); + e.stopPropagation(); + window.open(url, "_blank", "noopener,noreferrer"); + }; + return ( - - -
- - {source.title} - - {hasUrl && ( - - )} -
- - - - {cleanDescription} - - - + + + + +
+ + {source.title} + +
+ + #{chunkId} + + {hasUrl && ( + + )} +
+
+ + + + {cleanDescription} + + + + + ); } @@ -126,6 +154,7 @@ export default function ChatSourcesDisplay({ message }: { message: Message }) { title: node.metadata.title, description: node.text, url: node.url || "", + sourceType: sourceType, })), }); } diff --git a/surfsense_web/components/chat/SourceDetailSheet.tsx b/surfsense_web/components/chat/SourceDetailSheet.tsx new file mode 100644 index 000000000..4f7d129e4 --- /dev/null +++ b/surfsense_web/components/chat/SourceDetailSheet.tsx @@ -0,0 +1,244 @@ +"use client"; + +import { ChevronDown, ChevronUp, ExternalLink, Loader2 } from "lucide-react"; +import type React from "react"; +import { type ReactNode, useEffect, useRef, useState } from "react"; +import { MarkdownViewer } from "@/components/markdown-viewer"; +import { Button } from "@/components/ui/button"; +import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"; +import { ScrollArea } from "@/components/ui/scroll-area"; +import { + Sheet, + SheetContent, + SheetDescription, + SheetHeader, + SheetTitle, +} from "@/components/ui/sheet"; +import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; +import { useDocumentByChunk } from "@/hooks/use-document-by-chunk"; +import { cn } from "@/lib/utils"; + +interface SourceDetailSheetProps { + open: boolean; + onOpenChange: (open: boolean) => void; + chunkId: number; + sourceType: string; + title: string; + description?: string; + url?: string; + children?: ReactNode; +} + +const formatDocumentType = (type: string) => { + return type + .split("_") + .map((word) => word.charAt(0) + word.slice(1).toLowerCase()) + .join(" "); +}; + +export function SourceDetailSheet({ + open, + onOpenChange, + chunkId, + sourceType, + title, + description, + url, + children, +}: SourceDetailSheetProps) { + const { document, loading, error, fetchDocumentByChunk, clearDocument } = useDocumentByChunk(); + const chunksContainerRef = useRef(null); + const highlightedChunkRef = useRef(null); + const [summaryOpen, setSummaryOpen] = useState(false); + + // Check if this is a source type that should render directly from node + const isDirectRenderSource = sourceType === "TAVILY_API" || sourceType === "LINKUP_API"; + + useEffect(() => { + if (open && chunkId && !isDirectRenderSource) { + fetchDocumentByChunk(chunkId); + } else if (!open && !isDirectRenderSource) { + clearDocument(); + } + }, [open, chunkId, isDirectRenderSource, fetchDocumentByChunk, clearDocument]); + + useEffect(() => { + // Scroll to highlighted chunk when document loads + if (document && highlightedChunkRef.current && chunksContainerRef.current) { + setTimeout(() => { + highlightedChunkRef.current?.scrollIntoView({ + behavior: "smooth", + block: "start", + }); + }, 100); + } + }, [document]); + + const handleUrlClick = (e: React.MouseEvent, clickUrl: string) => { + e.preventDefault(); + e.stopPropagation(); + window.open(clickUrl, "_blank", "noopener,noreferrer"); + }; + + return ( + + {children} + + + + {getConnectorIcon(sourceType)} + {document?.title || title} + + + {document + ? formatDocumentType(document.document_type) + : sourceType && formatDocumentType(sourceType)} + + + + {!isDirectRenderSource && loading && ( +
+ +
+ )} + + {!isDirectRenderSource && error && ( +
+

{error}

+
+ )} + + {/* Direct render for TAVILY_API and LINKUP_API */} + {isDirectRenderSource && ( + +
+ {/* External Link */} + {url && ( +
+ +
+ )} + + {/* Source Information */} +
+

Source Information

+
+ {title || "Untitled"} +
+
+ {description || "No content available"} +
+
+
+ + )} + + {/* API-fetched document content */} + {!isDirectRenderSource && document && ( + +
+ {/* Document Metadata */} + {document.document_metadata && Object.keys(document.document_metadata).length > 0 && ( +
+

Document Information

+
+ {Object.entries(document.document_metadata).map(([key, value]) => ( +
+
+ {key.replace(/_/g, " ")}: +
+
{String(value)}
+
+ ))} +
+
+ )} + + {/* External Link */} + {url && ( +
+ +
+ )} + + {/* Chunks */} +
+
+ {/* Header row: header and button side by side */} +
+

Document Content

+ {document.content && ( + + + Summary + {summaryOpen ? ( + + ) : ( + + )} + + + )} +
+ {/* Expanded summary content: always full width, below the row */} + {document.content && ( + + +
+ +
+ + + )} +
+ + {document.chunks.map((chunk, idx) => ( +
+
+ + Chunk {idx + 1} of {document.chunks.length} + + {chunk.id === chunkId && ( + + Referenced Chunk + + )} +
+
+ +
+
+ ))} +
+
+ + )} + + + ); +} From 5bcf7b525a3f3d9966f4d7ea197c785d6417fa17 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Sun, 5 Oct 2025 23:02:54 -0700 Subject: [PATCH 04/35] refactor: move LLM_PROVIDERS to enums and clean up provider definitions --- .../components/onboard/add-provider-step.tsx | 23 +---- .../settings/model-config-manager.tsx | 94 +----------------- .../contracts/enums/llm-providers.ts | 99 +++++++++++++++++++ 3 files changed, 101 insertions(+), 115 deletions(-) create mode 100644 surfsense_web/contracts/enums/llm-providers.ts diff --git a/surfsense_web/components/onboard/add-provider-step.tsx b/surfsense_web/components/onboard/add-provider-step.tsx index e054a8cfe..62f5bc3b1 100644 --- a/surfsense_web/components/onboard/add-provider-step.tsx +++ b/surfsense_web/components/onboard/add-provider-step.tsx @@ -17,30 +17,9 @@ import { SelectTrigger, SelectValue, } from "@/components/ui/select"; +import { LLM_PROVIDERS } from "@/contracts/enums/llm-providers"; import { type CreateLLMConfig, useLLMConfigs } from "@/hooks/use-llm-configs"; -const LLM_PROVIDERS = [ - { value: "OPENAI", label: "OpenAI", example: "gpt-4o, gpt-4, gpt-3.5-turbo" }, - { - value: "ANTHROPIC", - label: "Anthropic", - example: "claude-3-5-sonnet-20241022, claude-3-opus-20240229", - }, - { value: "GROQ", label: "Groq", example: "llama3-70b-8192, mixtral-8x7b-32768" }, - { value: "COHERE", label: "Cohere", example: "command-r-plus, command-r" }, - { value: "HUGGINGFACE", label: "HuggingFace", example: "microsoft/DialoGPT-medium" }, - { value: "AZURE_OPENAI", label: "Azure OpenAI", example: "gpt-4, gpt-35-turbo" }, - { value: "GOOGLE", label: "Google", example: "gemini-pro, gemini-pro-vision" }, - { value: "AWS_BEDROCK", label: "AWS Bedrock", example: "anthropic.claude-v2" }, - { value: "OLLAMA", label: "Ollama", example: "llama2, codellama" }, - { value: "MISTRAL", label: "Mistral", example: "mistral-large-latest, mistral-medium" }, - { value: "TOGETHER_AI", label: "Together AI", example: "togethercomputer/llama-2-70b-chat" }, - { value: "REPLICATE", label: "Replicate", example: "meta/llama-2-70b-chat" }, - { value: "OPENROUTER", label: "OpenRouter", example: "anthropic/claude-opus-4.1, openai/gpt-5" }, - { value: "COMETAPI", label: "CometAPI", example: "gpt-4o, claude-3-5-sonnet-20241022" }, - { value: "CUSTOM", label: "Custom Provider", example: "your-custom-model" }, -]; - interface AddProviderStepProps { onConfigCreated?: () => void; onConfigDeleted?: () => void; diff --git a/surfsense_web/components/settings/model-config-manager.tsx b/surfsense_web/components/settings/model-config-manager.tsx index 709e118ca..28ade3527 100644 --- a/surfsense_web/components/settings/model-config-manager.tsx +++ b/surfsense_web/components/settings/model-config-manager.tsx @@ -37,101 +37,9 @@ import { SelectTrigger, SelectValue, } from "@/components/ui/select"; +import { LLM_PROVIDERS } from "@/contracts/enums/llm-providers"; import { type CreateLLMConfig, type LLMConfig, useLLMConfigs } from "@/hooks/use-llm-configs"; -const LLM_PROVIDERS = [ - { - value: "OPENAI", - label: "OpenAI", - example: "gpt-4o, gpt-4, gpt-3.5-turbo", - description: "Most popular and versatile AI models", - }, - { - value: "ANTHROPIC", - label: "Anthropic", - example: "claude-3-5-sonnet-20241022, claude-3-opus-20240229", - description: "Constitutional AI with strong reasoning", - }, - { - value: "GROQ", - label: "Groq", - example: "llama3-70b-8192, mixtral-8x7b-32768", - description: "Ultra-fast inference speeds", - }, - { - value: "COHERE", - label: "Cohere", - example: "command-r-plus, command-r", - description: "Enterprise-focused language models", - }, - { - value: "HUGGINGFACE", - label: "HuggingFace", - example: "microsoft/DialoGPT-medium", - description: "Open source model hub", - }, - { - value: "AZURE_OPENAI", - label: "Azure OpenAI", - example: "gpt-4, gpt-35-turbo", - description: "Enterprise OpenAI through Azure", - }, - { - value: "GOOGLE", - label: "Google", - example: "gemini-pro, gemini-pro-vision", - description: "Google's Gemini AI models", - }, - { - value: "AWS_BEDROCK", - label: "AWS Bedrock", - example: "anthropic.claude-v2", - description: "AWS managed AI service", - }, - { - value: "OLLAMA", - label: "Ollama", - example: "llama2, codellama", - description: "Run models locally", - }, - { - value: "MISTRAL", - label: "Mistral", - example: "mistral-large-latest, mistral-medium", - description: "European AI excellence", - }, - { - value: "TOGETHER_AI", - label: "Together AI", - example: "togethercomputer/llama-2-70b-chat", - description: "Decentralized AI platform", - }, - { - value: "REPLICATE", - label: "Replicate", - example: "meta/llama-2-70b-chat", - description: "Run models via API", - }, - { - value: "OPENROUTER", - label: "OpenRouter", - example: "anthropic/claude-opus-4.1, openai/gpt-5", - description: "API gateway and LLM marketplace that provides unified access ", - }, - { - value: "COMETAPI", - label: "CometAPI", - example: "gpt-5-mini, claude-sonnet-4-5", - description: "500+ AI models through one unified API", - }, - { - value: "CUSTOM", - label: "Custom Provider", - example: "your-custom-model", - description: "Your own model endpoint", - }, -]; - export function ModelConfigManager() { const { llmConfigs, diff --git a/surfsense_web/contracts/enums/llm-providers.ts b/surfsense_web/contracts/enums/llm-providers.ts new file mode 100644 index 000000000..753276fde --- /dev/null +++ b/surfsense_web/contracts/enums/llm-providers.ts @@ -0,0 +1,99 @@ +export interface LLMProvider { + value: string; + label: string; + example: string; + description: string; +} + +export const LLM_PROVIDERS: LLMProvider[] = [ + { + value: "OPENAI", + label: "OpenAI", + example: "gpt-4o, gpt-4, gpt-3.5-turbo", + description: "Industry-leading GPT models with broad capabilities", + }, + { + value: "ANTHROPIC", + label: "Anthropic", + example: "claude-3-5-sonnet-20241022, claude-3-opus-20240229", + description: "Claude models with strong reasoning and long context windows", + }, + { + value: "GROQ", + label: "Groq", + example: "llama3-70b-8192, mixtral-8x7b-32768", + description: "Lightning-fast inference with custom LPU hardware", + }, + { + value: "COHERE", + label: "Cohere", + example: "command-r-plus, command-r", + description: "Enterprise NLP models optimized for business applications", + }, + { + value: "HUGGINGFACE", + label: "HuggingFace", + example: "microsoft/DialoGPT-medium", + description: "Access thousands of open-source models", + }, + { + value: "AZURE_OPENAI", + label: "Azure OpenAI", + example: "gpt-4, gpt-35-turbo", + description: "OpenAI models with Microsoft Azure enterprise features", + }, + { + value: "GOOGLE", + label: "Google", + example: "gemini-pro, gemini-pro-vision", + description: "Gemini models with multimodal capabilities", + }, + { + value: "AWS_BEDROCK", + label: "AWS Bedrock", + example: "anthropic.claude-v2", + description: "Fully managed foundation models on AWS infrastructure", + }, + { + value: "OLLAMA", + label: "Ollama", + example: "llama2, codellama", + description: "Run open-source models locally on your machine", + }, + { + value: "MISTRAL", + label: "Mistral", + example: "mistral-large-latest, mistral-medium", + description: "High-performance open-source models from Europe", + }, + { + value: "TOGETHER_AI", + label: "Together AI", + example: "togethercomputer/llama-2-70b-chat", + description: "Scalable cloud platform for open-source models", + }, + { + value: "REPLICATE", + label: "Replicate", + example: "meta/llama-2-70b-chat", + description: "Cloud API for running machine learning models", + }, + { + value: "OPENROUTER", + label: "OpenRouter", + example: "anthropic/claude-opus-4.1, openai/gpt-5", + description: "Unified API gateway for multiple LLM providers", + }, + { + value: "COMETAPI", + label: "CometAPI", + example: "gpt-5-mini, claude-sonnet-4-5", + description: "Access 500+ AI models through one unified API", + }, + { + value: "CUSTOM", + label: "Custom Provider", + example: "your-custom-model", + description: "Connect to your own custom model endpoint", + }, +]; From fc76e268a078cb39cb1d7fdd09b4483bbaf60e18 Mon Sep 17 00:00:00 2001 From: Swayam Date: Tue, 7 Oct 2025 01:11:00 +0530 Subject: [PATCH 05/35] revert changes --- surfsense_backend/app/db.py | 2 -- surfsense_backend/app/schemas/search_space.py | 15 +++------------ 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py index 2fe79f8d1..99be9f277 100644 --- a/surfsense_backend/app/db.py +++ b/surfsense_backend/app/db.py @@ -205,8 +205,6 @@ class SearchSpace(BaseModel, TimestampMixin): name = Column(String(100), nullable=False, index=True) description = Column(String(500), nullable=True) - inference_params = Column(JSON, nullable=True) - user_id = Column( UUID(as_uuid=True), ForeignKey("user.id", ondelete="CASCADE"), nullable=False ) diff --git a/surfsense_backend/app/schemas/search_space.py b/surfsense_backend/app/schemas/search_space.py index 3dd8da4e8..49b5cd094 100644 --- a/surfsense_backend/app/schemas/search_space.py +++ b/surfsense_backend/app/schemas/search_space.py @@ -1,21 +1,14 @@ import uuid from datetime import datetime -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict from .base import IDModel, TimestampModel -class InferenceParams(BaseModel): - temperature: float | None = Field(None, ge=0.0, le=2.0) - max_tokens: int | None = Field(None, ge=0) - top_k: int | None = Field(None, ge=0) - top_p: float | None = Field(None, ge=0.0, le=1.0) - class SearchSpaceBase(BaseModel): name: str description: str | None = None - inference_params: InferenceParams | None = None class SearchSpaceCreate(SearchSpaceBase): @@ -23,9 +16,7 @@ class SearchSpaceCreate(SearchSpaceBase): class SearchSpaceUpdate(SearchSpaceBase): - name: str | None = None - description: str | None = None - inference_params: InferenceParams | None = None + pass class SearchSpaceRead(SearchSpaceBase, IDModel, TimestampModel): @@ -33,4 +24,4 @@ class SearchSpaceRead(SearchSpaceBase, IDModel, TimestampModel): created_at: datetime user_id: uuid.UUID - model_config = ConfigDict(from_attributes=True) + model_config = ConfigDict(from_attributes=True) \ No newline at end of file From 61ddf469ed38153255ae5de7c2570b50f26e249c Mon Sep 17 00:00:00 2001 From: samkul-swe Date: Mon, 6 Oct 2025 13:57:47 -0700 Subject: [PATCH 06/35] Not disabling dates anymore for any connector --- .../connectors/(manage)/page.tsx | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx index 9786263fa..4bacb855d 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx @@ -141,28 +141,6 @@ export default function ConnectorsPage() { } }; - const getDisabledEndDates = (date: Date) => { - const connector = connectors.find((c) => c.id === selectedConnectorForIndexing); - - switch (connector?.connector_type) { - case EnumConnectorName.GOOGLE_CALENDAR_CONNECTOR: - return startDate ? date < startDate : false; - default: - return date > today || (startDate ? date < startDate : false); - } - }; - - const getDisabledStartDates = (date: Date) => { - const connector = connectors.find((c) => c.id === selectedConnectorForIndexing); - - switch (connector?.connector_type) { - case EnumConnectorName.GOOGLE_CALENDAR_CONNECTOR: - return endDate ? date > endDate : false; - default: - return date > today || (endDate ? date > endDate : false); - } - }; - return (
@@ -393,7 +370,6 @@ export default function ConnectorsPage() { mode="single" selected={endDate} onSelect={setEndDate} - disabled={getDisabledEndDates} initialFocus /> From b7c68277f6413de3a753644947962ffe18899497 Mon Sep 17 00:00:00 2001 From: Swayam Date: Tue, 7 Oct 2025 04:59:01 +0530 Subject: [PATCH 07/35] feat: add interface params --- ...3873_add_litellm_params_to_searchspaces.py | 40 +++++ .../components/inference-params-editor.tsx | 138 ++++++++++++++++++ .../components/onboard/add-provider-step.tsx | 15 ++ .../settings/model-config-manager.tsx | 11 ++ 4 files changed, 204 insertions(+) create mode 100644 surfsense_backend/alembic/versions/fd3f2e543873_add_litellm_params_to_searchspaces.py create mode 100644 surfsense_web/components/inference-params-editor.tsx diff --git a/surfsense_backend/alembic/versions/fd3f2e543873_add_litellm_params_to_searchspaces.py b/surfsense_backend/alembic/versions/fd3f2e543873_add_litellm_params_to_searchspaces.py new file mode 100644 index 000000000..1758f7602 --- /dev/null +++ b/surfsense_backend/alembic/versions/fd3f2e543873_add_litellm_params_to_searchspaces.py @@ -0,0 +1,40 @@ +"""add litellm_params to searchspaces + +Revision ID: fd3f2e543873 +Revises: eb0c978da8fd +Create Date: 2025-10-07 04:11:50.392338 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = 'fd3f2e543873' +down_revision: Union[str, None] = 'eb0c978da8fd' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f('chucks_search_index'), table_name='chunks', postgresql_using='gin') + op.drop_index(op.f('chucks_vector_index'), table_name='chunks', postgresql_using='hnsw') + op.drop_index(op.f('document_search_index'), table_name='documents', postgresql_using='gin') + op.drop_index(op.f('document_vector_index'), table_name='documents', postgresql_using='hnsw') + op.drop_column('searchspaces', 'inference_params') + # ### end Alembic commands ### + + +def downgrade() -> None: + """Downgrade schema.""" + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('searchspaces', sa.Column('inference_params', postgresql.JSON(astext_type=sa.Text()), autoincrement=False, nullable=True)) + op.create_index(op.f('document_vector_index'), 'documents', ['embedding'], unique=False, postgresql_using='hnsw') + op.create_index(op.f('document_search_index'), 'documents', [sa.literal_column("to_tsvector('english'::regconfig, content)")], unique=False, postgresql_using='gin') + op.create_index(op.f('chucks_vector_index'), 'chunks', ['embedding'], unique=False, postgresql_using='hnsw') + op.create_index(op.f('chucks_search_index'), 'chunks', [sa.literal_column("to_tsvector('english'::regconfig, content)")], unique=False, postgresql_using='gin') + # ### end Alembic commands ### diff --git a/surfsense_web/components/inference-params-editor.tsx b/surfsense_web/components/inference-params-editor.tsx new file mode 100644 index 000000000..df198cfa0 --- /dev/null +++ b/surfsense_web/components/inference-params-editor.tsx @@ -0,0 +1,138 @@ +"use client"; + +import { useState } from "react"; +import { Plus, Trash2 } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; + +interface InferenceParamsEditorProps { + params: Record; + setParams: (newParams: Record) => void; +} + +const PARAM_KEYS = ["temperature", "max_tokens", "top_k", "top_p"] as const; + +export default function InferenceParamsEditor({ + params, + setParams, +}: InferenceParamsEditorProps) { + const [selectedKey, setSelectedKey] = useState(""); + const [value, setValue] = useState(""); + + const handleAdd = () => { + if (!selectedKey || value === "") return; + + if (params[selectedKey]) { + alert(`${selectedKey} already exists`); + return; + } + + const numericValue = Number(value); + + if ((selectedKey === "temperature" || selectedKey === "top_p") && (isNaN(numericValue) || numericValue < 0 || numericValue > 1)) { + alert("Value must be a number between 0 and 1"); + return; + } + + if ((selectedKey === "max_tokens" || selectedKey === "top_k") && (!Number.isInteger(numericValue) || numericValue < 0)) { + alert("Value must be a non-negative integer"); + return; + } + + setParams({ + ...params, + [selectedKey]: isNaN(numericValue) ? value : numericValue, + }); + + setSelectedKey(""); + setValue(""); + }; + + const handleDelete = (key: string) => { + const newParams = { ...params }; + delete newParams[key]; + setParams(newParams); + }; + + return ( +
+
+
+ + +
+ +
+ + setValue(e.target.value)} + className="w-full" + /> +
+ + +
+ +
+ + {Object.keys(params).length > 0 && ( +
+ + + + + + + + + + {Object.entries(params).map(([key, val]) => ( + + + + + + ))} + +
KeyValueActions
{key}{val.toString()} + +
+
+ )} +
+ ); +} diff --git a/surfsense_web/components/onboard/add-provider-step.tsx b/surfsense_web/components/onboard/add-provider-step.tsx index 62f5bc3b1..f582000a3 100644 --- a/surfsense_web/components/onboard/add-provider-step.tsx +++ b/surfsense_web/components/onboard/add-provider-step.tsx @@ -20,6 +20,8 @@ import { import { LLM_PROVIDERS } from "@/contracts/enums/llm-providers"; import { type CreateLLMConfig, useLLMConfigs } from "@/hooks/use-llm-configs"; +import InferenceParamsEditor from "../inference-params-editor"; + interface AddProviderStepProps { onConfigCreated?: () => void; onConfigDeleted?: () => void; @@ -72,6 +74,10 @@ export function AddProviderStep({ onConfigCreated, onConfigDeleted }: AddProvide const selectedProvider = LLM_PROVIDERS.find((p) => p.value === formData.provider); + const handleParamsChange = (newParams: Record) => { + setFormData((prev) => ({ ...prev, litellm_params: newParams })); + }; + return (
{/* Info Alert */} @@ -239,6 +245,15 @@ export function AddProviderStep({ onConfigCreated, onConfigDeleted }: AddProvide />
+ {/* Optional Inference Parameters */} +
+ +
+ +
+ {/* Optional Inference Parameters */} +
+ + setFormData((prev) => ({ ...prev, litellm_params: newParams })) + } + /> +
+
- +
diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/page.tsx index 1e1a58453..41a8492fd 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/page.tsx @@ -83,7 +83,7 @@ export default function EditConnectorPage() { const searchSpaceId = params.search_space_id as string; const connectorId = parseInt(params.connector_id as string, 10); - const { connectors, updateConnector } = useSearchSourceConnectors(); + const { connectors, updateConnector } = useSearchSourceConnectors(false, parseInt(searchSpaceId)); const [connector, setConnector] = useState(null); const [isLoading, setIsLoading] = useState(true); const [isSubmitting, setIsSubmitting] = useState(false); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/airtable-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/airtable-connector/page.tsx index cd81474ba..2d82877b3 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/airtable-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/airtable-connector/page.tsx @@ -30,10 +30,10 @@ export default function AirtableConnectorPage() { const [isConnecting, setIsConnecting] = useState(false); const [doesConnectorExist, setDoesConnectorExist] = useState(false); - const { fetchConnectors } = useSearchSourceConnectors(); + const { fetchConnectors } = useSearchSourceConnectors(true, parseInt(searchSpaceId)); useEffect(() => { - fetchConnectors().then((data) => { + fetchConnectors(parseInt(searchSpaceId)).then((data) => { const connector = data.find( (c: SearchSourceConnector) => c.connector_type === EnumConnectorName.AIRTABLE_CONNECTOR ); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/clickup-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/clickup-connector/page.tsx index d7ce20cbf..cc86b1680 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/clickup-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/clickup-connector/page.tsx @@ -69,7 +69,7 @@ export default function ClickUpConnectorPage() { last_indexed_at: null, }; - await createConnector(connectorData); + await createConnector(connectorData, parseInt(searchSpaceId)); toast.success("ClickUp connector created successfully!"); router.push(`/dashboard/${searchSpaceId}/connectors`); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/confluence-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/confluence-connector/page.tsx index e4784d22c..361afd5ff 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/confluence-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/confluence-connector/page.tsx @@ -77,17 +77,20 @@ export default function ConfluenceConnectorPage() { const onSubmit = async (values: ConfluenceConnectorFormValues) => { setIsSubmitting(true); try { - await createConnector({ - name: values.name, - connector_type: EnumConnectorName.CONFLUENCE_CONNECTOR, - config: { - CONFLUENCE_BASE_URL: values.base_url, - CONFLUENCE_EMAIL: values.email, - CONFLUENCE_API_TOKEN: values.api_token, + await createConnector( + { + name: values.name, + connector_type: EnumConnectorName.CONFLUENCE_CONNECTOR, + config: { + CONFLUENCE_BASE_URL: values.base_url, + CONFLUENCE_EMAIL: values.email, + CONFLUENCE_API_TOKEN: values.api_token, + }, + is_indexable: true, + last_indexed_at: null, }, - is_indexable: true, - last_indexed_at: null, - }); + parseInt(searchSpaceId) + ); toast.success("Confluence connector created successfully!"); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/discord-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/discord-connector/page.tsx index 47366bc01..573190945 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/discord-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/discord-connector/page.tsx @@ -73,15 +73,18 @@ export default function DiscordConnectorPage() { const onSubmit = async (values: DiscordConnectorFormValues) => { setIsSubmitting(true); try { - await createConnector({ - name: values.name, - connector_type: EnumConnectorName.DISCORD_CONNECTOR, - config: { - DISCORD_BOT_TOKEN: values.bot_token, + await createConnector( + { + name: values.name, + connector_type: EnumConnectorName.DISCORD_CONNECTOR, + config: { + DISCORD_BOT_TOKEN: values.bot_token, + }, + is_indexable: true, + last_indexed_at: null, }, - is_indexable: true, - last_indexed_at: null, - }); + parseInt(searchSpaceId) + ); toast.success("Discord connector created successfully!"); router.push(`/dashboard/${searchSpaceId}/connectors`); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/github-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/github-connector/page.tsx index 602cf066f..e5a6d24f7 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/github-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/github-connector/page.tsx @@ -148,16 +148,19 @@ export default function GithubConnectorPage() { setIsCreatingConnector(true); try { - await createConnector({ - name: connectorName, // Use the stored name - connector_type: EnumConnectorName.GITHUB_CONNECTOR, - config: { - GITHUB_PAT: validatedPat, // Use the stored validated PAT - repo_full_names: selectedRepos, // Add the selected repo names + await createConnector( + { + name: connectorName, // Use the stored name + connector_type: EnumConnectorName.GITHUB_CONNECTOR, + config: { + GITHUB_PAT: validatedPat, // Use the stored validated PAT + repo_full_names: selectedRepos, // Add the selected repo names + }, + is_indexable: true, + last_indexed_at: null, }, - is_indexable: true, - last_indexed_at: null, - }); + parseInt(searchSpaceId) + ); toast.success("GitHub connector created successfully!"); router.push(`/dashboard/${searchSpaceId}/connectors`); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-calendar-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-calendar-connector/page.tsx index a190af7b3..2fdc95671 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-calendar-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-calendar-connector/page.tsx @@ -32,10 +32,10 @@ export default function GoogleCalendarConnectorPage() { const [isConnecting, setIsConnecting] = useState(false); const [doesConnectorExist, setDoesConnectorExist] = useState(false); - const { fetchConnectors } = useSearchSourceConnectors(); + const { fetchConnectors } = useSearchSourceConnectors(true, parseInt(searchSpaceId)); useEffect(() => { - fetchConnectors().then((data) => { + fetchConnectors(parseInt(searchSpaceId)).then((data) => { const connector = data.find( (c: SearchSourceConnector) => c.connector_type === EnumConnectorName.GOOGLE_CALENDAR_CONNECTOR diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-gmail-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-gmail-connector/page.tsx index 573650db9..c1354d03e 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-gmail-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-gmail-connector/page.tsx @@ -32,10 +32,10 @@ export default function GoogleGmailConnectorPage() { const [isConnecting, setIsConnecting] = useState(false); const [doesConnectorExist, setDoesConnectorExist] = useState(false); - const { fetchConnectors } = useSearchSourceConnectors(); + const { fetchConnectors } = useSearchSourceConnectors(true, parseInt(searchSpaceId)); useEffect(() => { - fetchConnectors().then((data) => { + fetchConnectors(parseInt(searchSpaceId)).then((data) => { const connector = data.find( (c: SearchSourceConnector) => c.connector_type === EnumConnectorName.GOOGLE_GMAIL_CONNECTOR ); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/jira-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/jira-connector/page.tsx index ea99c07d6..6d3e92e42 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/jira-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/jira-connector/page.tsx @@ -90,17 +90,20 @@ export default function JiraConnectorPage() { const onSubmit = async (values: JiraConnectorFormValues) => { setIsSubmitting(true); try { - await createConnector({ - name: values.name, - connector_type: EnumConnectorName.JIRA_CONNECTOR, - config: { - JIRA_BASE_URL: values.base_url, - JIRA_EMAIL: values.email, - JIRA_API_TOKEN: values.api_token, + await createConnector( + { + name: values.name, + connector_type: EnumConnectorName.JIRA_CONNECTOR, + config: { + JIRA_BASE_URL: values.base_url, + JIRA_EMAIL: values.email, + JIRA_API_TOKEN: values.api_token, + }, + is_indexable: true, + last_indexed_at: null, }, - is_indexable: true, - last_indexed_at: null, - }); + parseInt(searchSpaceId) + ); toast.success("Jira connector created successfully!"); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/linear-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/linear-connector/page.tsx index 71cc97ded..b1d2a51e6 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/linear-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/linear-connector/page.tsx @@ -77,15 +77,18 @@ export default function LinearConnectorPage() { const onSubmit = async (values: LinearConnectorFormValues) => { setIsSubmitting(true); try { - await createConnector({ - name: values.name, - connector_type: EnumConnectorName.LINEAR_CONNECTOR, - config: { - LINEAR_API_KEY: values.api_key, + await createConnector( + { + name: values.name, + connector_type: EnumConnectorName.LINEAR_CONNECTOR, + config: { + LINEAR_API_KEY: values.api_key, + }, + is_indexable: true, + last_indexed_at: null, }, - is_indexable: true, - last_indexed_at: null, - }); + parseInt(searchSpaceId) + ); toast.success("Linear connector created successfully!"); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/linkup-api/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/linkup-api/page.tsx index f68fee18c..a4467fb1d 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/linkup-api/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/linkup-api/page.tsx @@ -65,15 +65,18 @@ export default function LinkupApiPage() { const onSubmit = async (values: LinkupApiFormValues) => { setIsSubmitting(true); try { - await createConnector({ - name: values.name, - connector_type: EnumConnectorName.LINKUP_API, - config: { - LINKUP_API_KEY: values.api_key, + await createConnector( + { + name: values.name, + connector_type: EnumConnectorName.LINKUP_API, + config: { + LINKUP_API_KEY: values.api_key, + }, + is_indexable: false, + last_indexed_at: null, }, - is_indexable: false, - last_indexed_at: null, - }); + parseInt(searchSpaceId) + ); toast.success("Linkup API connector created successfully!"); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/luma-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/luma-connector/page.tsx index 176aadb74..f35c09770 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/luma-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/luma-connector/page.tsx @@ -55,7 +55,10 @@ export default function LumaConnectorPage() { const [isSubmitting, setIsSubmitting] = useState(false); const [doesConnectorExist, setDoesConnectorExist] = useState(false); - const { fetchConnectors, createConnector } = useSearchSourceConnectors(); + const { fetchConnectors, createConnector } = useSearchSourceConnectors( + true, + parseInt(searchSpaceId) + ); // Initialize the form const form = useForm({ @@ -67,7 +70,7 @@ export default function LumaConnectorPage() { }); useEffect(() => { - fetchConnectors().then((data) => { + fetchConnectors(parseInt(searchSpaceId)).then((data) => { const connector = data.find( (c: SearchSourceConnector) => c.connector_type === EnumConnectorName.LUMA_CONNECTOR ); @@ -81,15 +84,18 @@ export default function LumaConnectorPage() { const onSubmit = async (values: LumaConnectorFormValues) => { setIsSubmitting(true); try { - await createConnector({ - name: values.name, - connector_type: EnumConnectorName.LUMA_CONNECTOR, - config: { - LUMA_API_KEY: values.api_key, + await createConnector( + { + name: values.name, + connector_type: EnumConnectorName.LUMA_CONNECTOR, + config: { + LUMA_API_KEY: values.api_key, + }, + is_indexable: true, + last_indexed_at: null, }, - is_indexable: true, - last_indexed_at: null, - }); + parseInt(searchSpaceId) + ); toast.success("Luma connector created successfully!"); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/notion-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/notion-connector/page.tsx index d9a9bdf52..5e0a4bbb3 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/notion-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/notion-connector/page.tsx @@ -72,15 +72,18 @@ export default function NotionConnectorPage() { const onSubmit = async (values: NotionConnectorFormValues) => { setIsSubmitting(true); try { - await createConnector({ - name: values.name, - connector_type: EnumConnectorName.NOTION_CONNECTOR, - config: { - NOTION_INTEGRATION_TOKEN: values.integration_token, + await createConnector( + { + name: values.name, + connector_type: EnumConnectorName.NOTION_CONNECTOR, + config: { + NOTION_INTEGRATION_TOKEN: values.integration_token, + }, + is_indexable: true, + last_indexed_at: null, }, - is_indexable: true, - last_indexed_at: null, - }); + parseInt(searchSpaceId) + ); toast.success("Notion connector created successfully!"); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/serper-api/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/serper-api/page.tsx index f594406ed..10e5da6f7 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/serper-api/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/serper-api/page.tsx @@ -65,15 +65,18 @@ export default function SerperApiPage() { const onSubmit = async (values: SerperApiFormValues) => { setIsSubmitting(true); try { - await createConnector({ - name: values.name, - connector_type: EnumConnectorName.SERPER_API, - config: { - SERPER_API_KEY: values.api_key, + await createConnector( + { + name: values.name, + connector_type: EnumConnectorName.SERPER_API, + config: { + SERPER_API_KEY: values.api_key, + }, + is_indexable: false, + last_indexed_at: null, }, - is_indexable: false, - last_indexed_at: null, - }); + parseInt(searchSpaceId) + ); toast.success("Serper API connector created successfully!"); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/slack-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/slack-connector/page.tsx index e27c96da9..e22d69d6d 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/slack-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/slack-connector/page.tsx @@ -72,15 +72,18 @@ export default function SlackConnectorPage() { const onSubmit = async (values: SlackConnectorFormValues) => { setIsSubmitting(true); try { - await createConnector({ - name: values.name, - connector_type: EnumConnectorName.SLACK_CONNECTOR, - config: { - SLACK_BOT_TOKEN: values.bot_token, + await createConnector( + { + name: values.name, + connector_type: EnumConnectorName.SLACK_CONNECTOR, + config: { + SLACK_BOT_TOKEN: values.bot_token, + }, + is_indexable: true, + last_indexed_at: null, }, - is_indexable: true, - last_indexed_at: null, - }); + parseInt(searchSpaceId) + ); toast.success("Slack connector created successfully!"); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/tavily-api/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/tavily-api/page.tsx index 1afb7fbed..f744241b2 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/tavily-api/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/tavily-api/page.tsx @@ -65,15 +65,18 @@ export default function TavilyApiPage() { const onSubmit = async (values: TavilyApiFormValues) => { setIsSubmitting(true); try { - await createConnector({ - name: values.name, - connector_type: EnumConnectorName.TAVILY_API, - config: { - TAVILY_API_KEY: values.api_key, + await createConnector( + { + name: values.name, + connector_type: EnumConnectorName.TAVILY_API, + config: { + TAVILY_API_KEY: values.api_key, + }, + is_indexable: false, + last_indexed_at: null, }, - is_indexable: false, - last_indexed_at: null, - }); + parseInt(searchSpaceId) + ); toast.success("Tavily API connector created successfully!"); diff --git a/surfsense_web/app/dashboard/[search_space_id]/podcasts/podcasts-client.tsx b/surfsense_web/app/dashboard/[search_space_id]/podcasts/podcasts-client.tsx index 7dd95f04f..1f3fd275f 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/podcasts/podcasts-client.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/podcasts/podcasts-client.tsx @@ -1,7 +1,6 @@ "use client"; import { format } from "date-fns"; -import { AnimatePresence, motion, type Variants } from "framer-motion"; import { Calendar, MoreHorizontal, @@ -16,6 +15,7 @@ import { VolumeX, X, } from "lucide-react"; +import { AnimatePresence, motion, type Variants } from "motion/react"; import Image from "next/image"; import { useEffect, useRef, useState } from "react"; import { toast } from "sonner"; diff --git a/surfsense_web/components/chat/ChatInputGroup.tsx b/surfsense_web/components/chat/ChatInputGroup.tsx index fdc03e893..f57691e4d 100644 --- a/surfsense_web/components/chat/ChatInputGroup.tsx +++ b/surfsense_web/components/chat/ChatInputGroup.tsx @@ -124,19 +124,20 @@ const ConnectorSelector = React.memo( onSelectionChange?: (connectorTypes: string[]) => void; selectedConnectors?: string[]; }) => { + const { search_space_id } = useParams(); const [isOpen, setIsOpen] = useState(false); const { connectorSourceItems, isLoading, isLoaded, fetchConnectors } = - useSearchSourceConnectors(true); + useSearchSourceConnectors(true, Number(search_space_id)); const handleOpenChange = useCallback( (open: boolean) => { setIsOpen(open); if (open && !isLoaded) { - fetchConnectors(); + fetchConnectors(Number(search_space_id)); } }, - [fetchConnectors, isLoaded] + [fetchConnectors, isLoaded, search_space_id] ); const handleConnectorToggle = useCallback( diff --git a/surfsense_web/hooks/use-connector-edit-page.ts b/surfsense_web/hooks/use-connector-edit-page.ts index d31ccf928..fe90db3c8 100644 --- a/surfsense_web/hooks/use-connector-edit-page.ts +++ b/surfsense_web/hooks/use-connector-edit-page.ts @@ -18,7 +18,11 @@ import { export function useConnectorEditPage(connectorId: number, searchSpaceId: string) { const router = useRouter(); - const { connectors, updateConnector, isLoading: connectorsLoading } = useSearchSourceConnectors(); + const { + connectors, + updateConnector, + isLoading: connectorsLoading, + } = useSearchSourceConnectors(false, parseInt(searchSpaceId)); // State managed by the hook const [connector, setConnector] = useState(null); diff --git a/surfsense_web/hooks/use-search-source-connectors.ts b/surfsense_web/hooks/use-search-source-connectors.ts index 0c59a3bf0..5a45fd761 100644 --- a/surfsense_web/hooks/use-search-source-connectors.ts +++ b/surfsense_web/hooks/use-search-source-connectors.ts @@ -7,6 +7,7 @@ export interface SearchSourceConnector { is_indexable: boolean; last_indexed_at: string | null; config: Record; + search_space_id: number; user_id?: string; created_at?: string; } @@ -20,8 +21,10 @@ export interface ConnectorSourceItem { /** * Hook to fetch search source connectors from the API + * @param lazy - If true, connectors won't be fetched on mount + * @param searchSpaceId - Optional search space ID to filter connectors */ -export const useSearchSourceConnectors = (lazy: boolean = false) => { +export const useSearchSourceConnectors = (lazy: boolean = false, searchSpaceId?: number) => { const [connectors, setConnectors] = useState([]); const [isLoading, setIsLoading] = useState(!lazy); // Don't show loading initially for lazy mode const [isLoaded, setIsLoaded] = useState(false); // Memoization flag @@ -53,60 +56,71 @@ export const useSearchSourceConnectors = (lazy: boolean = false) => { }, ]); - const fetchConnectors = useCallback(async () => { - if (isLoaded && lazy) return; // Avoid redundant calls in lazy mode + const fetchConnectors = useCallback( + async (spaceId?: number) => { + if (isLoaded && lazy) return; // Avoid redundant calls in lazy mode - try { - setIsLoading(true); - setError(null); - const token = localStorage.getItem("surfsense_bearer_token"); + try { + setIsLoading(true); + setError(null); + const token = localStorage.getItem("surfsense_bearer_token"); - if (!token) { - throw new Error("No authentication token found"); - } + if (!token) { + throw new Error("No authentication token found"); + } - const response = await fetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-source-connectors/`, - { + // Build URL with optional search_space_id query parameter + const url = new URL( + `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-source-connectors/` + ); + if (spaceId !== undefined) { + url.searchParams.append("search_space_id", spaceId.toString()); + } + + const response = await fetch(url.toString(), { method: "GET", headers: { "Content-Type": "application/json", Authorization: `Bearer ${token}`, }, + }); + + if (!response.ok) { + throw new Error(`Failed to fetch connectors: ${response.statusText}`); } - ); - if (!response.ok) { - throw new Error(`Failed to fetch connectors: ${response.statusText}`); + const data = await response.json(); + setConnectors(data); + setIsLoaded(true); + + // Update connector source items when connectors change + updateConnectorSourceItems(data); + + return data; + } catch (err) { + setError(err instanceof Error ? err : new Error("An unknown error occurred")); + console.error("Error fetching search source connectors:", err); + } finally { + setIsLoading(false); } - - const data = await response.json(); - setConnectors(data); - setIsLoaded(true); - - // Update connector source items when connectors change - updateConnectorSourceItems(data); - - return data; - } catch (err) { - setError(err instanceof Error ? err : new Error("An unknown error occurred")); - console.error("Error fetching search source connectors:", err); - } finally { - setIsLoading(false); - } - }, [isLoaded, lazy]); + }, + [isLoaded, lazy] + ); useEffect(() => { if (!lazy) { - fetchConnectors(); + fetchConnectors(searchSpaceId); } - }, [lazy, fetchConnectors]); + }, [lazy, fetchConnectors, searchSpaceId]); // Function to refresh the connectors list - const refreshConnectors = useCallback(async () => { - setIsLoaded(false); // Reset memoization flag to allow refetch - await fetchConnectors(); - }, [fetchConnectors]); + const refreshConnectors = useCallback( + async (spaceId?: number) => { + setIsLoaded(false); // Reset memoization flag to allow refetch + await fetchConnectors(spaceId !== undefined ? spaceId : searchSpaceId); + }, + [fetchConnectors, searchSpaceId] + ); // Update connector source items when connectors change const updateConnectorSourceItems = (currentConnectors: SearchSourceConnector[]) => { @@ -151,9 +165,12 @@ export const useSearchSourceConnectors = (lazy: boolean = false) => { /** * Create a new search source connector + * @param connectorData - The connector data (excluding search_space_id) + * @param spaceId - The search space ID to associate the connector with */ const createConnector = async ( - connectorData: Omit + connectorData: Omit, + spaceId: number ) => { try { const token = localStorage.getItem("surfsense_bearer_token"); @@ -162,17 +179,20 @@ export const useSearchSourceConnectors = (lazy: boolean = false) => { throw new Error("No authentication token found"); } - const response = await fetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-source-connectors/`, - { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${token}`, - }, - body: JSON.stringify(connectorData), - } + // Add search_space_id as a query parameter + const url = new URL( + `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-source-connectors/` ); + url.searchParams.append("search_space_id", spaceId.toString()); + + const response = await fetch(url.toString(), { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${token}`, + }, + body: JSON.stringify(connectorData), + }); if (!response.ok) { throw new Error(`Failed to create connector: ${response.statusText}`); @@ -194,7 +214,9 @@ export const useSearchSourceConnectors = (lazy: boolean = false) => { */ const updateConnector = async ( connectorId: number, - connectorData: Partial> + connectorData: Partial< + Omit + > ) => { try { const token = localStorage.getItem("surfsense_bearer_token"); From b5c1157468616cedb510cc5e7854b144feb6a527 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Wed, 8 Oct 2025 21:45:21 -0700 Subject: [PATCH 10/35] fix: removed checks that were not checked - fuck this hacktoberfest --- surfsense_backend/app/utils/validators.py | 28 +++++++++++------------ 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/surfsense_backend/app/utils/validators.py b/surfsense_backend/app/utils/validators.py index 32b406b9d..7677d54f2 100644 --- a/surfsense_backend/app/utils/validators.py +++ b/surfsense_backend/app/utils/validators.py @@ -524,20 +524,20 @@ def validate_connector_config(connector_type: str | Any, config: dict[str, Any]) "required": ["CLICKUP_API_TOKEN"], "validators": {} }, - "GOOGLE_CALENDAR_CONNECTOR": { - "required": ["token", "refresh_token", "token_uri", "client_id", "expiry", "scopes", "client_secret"], - "validators": {}, - "allow_none_or_empty": False # Special flag for Google connectors - }, - "GOOGLE_GMAIL_CONNECTOR": { - "required": ["token", "refresh_token", "token_uri", "client_id", "expiry", "scopes", "client_secret"], - "validators": {}, - "allow_none_or_empty": False - }, - "AIRTABLE_CONNECTOR": { - "required": ["AIRTABLE_API_KEY", "AIRTABLE_BASE_ID"], - "validators": {} - }, + # "GOOGLE_CALENDAR_CONNECTOR": { + # "required": ["token", "refresh_token", "token_uri", "client_id", "expiry", "scopes", "client_secret"], + # "validators": {}, + # "allow_none_or_empty": False # Special flag for Google connectors + # }, + # "GOOGLE_GMAIL_CONNECTOR": { + # "required": ["token", "refresh_token", "token_uri", "client_id", "expiry", "scopes", "client_secret"], + # "validators": {}, + # "allow_none_or_empty": False + # }, + # "AIRTABLE_CONNECTOR": { + # "required": ["AIRTABLE_API_KEY", "AIRTABLE_BASE_ID"], + # "validators": {} + # }, "LUMA_CONNECTOR": { "required": ["LUMA_API_KEY"], "validators": {} From 633ea3ac0f5365d46603a1fba226e3b729fd7785 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Fri, 10 Oct 2025 00:50:29 -0700 Subject: [PATCH 11/35] feat: moved LLMConfigs from User to SearchSpaces - RBAC soon?? - Updated various services and routes to handle search space-specific LLM preferences. - Modified frontend components to pass search space ID for LLM configuration management. - Removed onboarding page and settings page as part of the refactor. --- ...25_migrate_llm_configs_to_search_spaces.py | 352 +++++++++++++++++ .../app/agents/podcaster/configuration.py | 1 + .../app/agents/podcaster/nodes.py | 5 +- .../app/agents/researcher/nodes.py | 11 +- .../app/agents/researcher/qna_agent/nodes.py | 5 +- .../researcher/sub_section_writer/nodes.py | 5 +- surfsense_backend/app/db.py | 107 ++--- surfsense_backend/app/routes/chats_routes.py | 31 +- .../app/routes/llm_config_routes.py | 227 ++++++++--- surfsense_backend/app/schemas/llm_config.py | 7 +- surfsense_backend/app/schemas/search_space.py | 2 +- surfsense_backend/app/services/llm_service.py | 69 ++-- .../app/services/query_service.py | 6 +- .../connector_indexers/airtable_indexer.py | 4 +- .../connector_indexers/clickup_indexer.py | 4 +- .../connector_indexers/confluence_indexer.py | 4 +- .../connector_indexers/discord_indexer.py | 4 +- .../connector_indexers/github_indexer.py | 4 +- .../google_calendar_indexer.py | 4 +- .../google_gmail_indexer.py | 4 +- .../tasks/connector_indexers/jira_indexer.py | 4 +- .../connector_indexers/linear_indexer.py | 4 +- .../tasks/connector_indexers/luma_indexer.py | 4 +- .../connector_indexers/notion_indexer.py | 4 +- .../extension_processor.py | 6 +- .../document_processors/file_processors.py | 18 +- .../document_processors/markdown_processor.py | 6 +- .../tasks/document_processors/url_crawler.py | 6 +- .../document_processors/youtube_processor.py | 6 +- surfsense_backend/app/tasks/podcast_tasks.py | 1 + surfsense_backend/app/utils/validators.py | 370 ++++++++---------- .../[search_space_id]/client-layout.tsx | 78 +++- .../dashboard/[search_space_id]/layout.tsx | 6 + .../[search_space_id]}/onboard/page.tsx | 39 +- .../[search_space_id]}/settings/page.tsx | 16 +- surfsense_web/app/dashboard/layout.tsx | 56 +-- surfsense_web/components/UserDropdown.tsx | 4 - .../components/chat/ChatInputGroup.tsx | 7 +- .../components/onboard/add-provider-step.tsx | 12 +- .../components/onboard/assign-roles-step.tsx | 7 +- .../components/onboard/completion-step.tsx | 10 +- .../components/settings/llm-role-manager.tsx | 10 +- .../settings/model-config-manager.tsx | 31 +- surfsense_web/hooks/use-llm-configs.ts | 32 +- 44 files changed, 1075 insertions(+), 518 deletions(-) create mode 100644 surfsense_backend/alembic/versions/25_migrate_llm_configs_to_search_spaces.py rename surfsense_web/app/{ => dashboard/[search_space_id]}/onboard/page.tsx (87%) rename surfsense_web/app/{ => dashboard/[search_space_id]}/settings/page.tsx (82%) diff --git a/surfsense_backend/alembic/versions/25_migrate_llm_configs_to_search_spaces.py b/surfsense_backend/alembic/versions/25_migrate_llm_configs_to_search_spaces.py new file mode 100644 index 000000000..116a3c687 --- /dev/null +++ b/surfsense_backend/alembic/versions/25_migrate_llm_configs_to_search_spaces.py @@ -0,0 +1,352 @@ +"""Migrate LLM configs to search spaces and add user preferences + +Revision ID: 25 +Revises: 24 +Create Date: 2025-01-10 14:00:00.000000 + +Changes: +1. Migrate llm_configs from user association to search_space association +2. Create user_search_space_preferences table for per-user LLM preferences +3. Migrate existing user LLM preferences to user_search_space_preferences +4. Remove LLM preference columns from user table +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "25" +down_revision: str | None = "24" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """ + Upgrade schema to support collaborative search spaces with per-user preferences. + + Migration steps: + 1. Add search_space_id to llm_configs + 2. Migrate existing llm_configs to first search space of their user + 3. Replace user_id with search_space_id in llm_configs + 4. Create user_search_space_preferences table + 5. Migrate user LLM preferences to user_search_space_preferences + 6. Remove LLM preference columns from user table + """ + + from sqlalchemy import inspect + + conn = op.get_bind() + inspector = inspect(conn) + + # Get existing columns + llm_config_columns = [col["name"] for col in inspector.get_columns("llm_configs")] + user_columns = [col["name"] for col in inspector.get_columns("user")] + + # ===== STEP 1: Add search_space_id to llm_configs ===== + if "search_space_id" not in llm_config_columns: + op.add_column( + "llm_configs", + sa.Column("search_space_id", sa.Integer(), nullable=True), + ) + + # ===== STEP 2: Populate search_space_id with user's first search space ===== + # This ensures existing LLM configs are assigned to a valid search space + op.execute( + """ + UPDATE llm_configs lc + SET search_space_id = ( + SELECT id + FROM searchspaces ss + WHERE ss.user_id = lc.user_id + ORDER BY ss.created_at ASC + LIMIT 1 + ) + WHERE search_space_id IS NULL AND user_id IS NOT NULL + """ + ) + + # ===== STEP 3: Make search_space_id NOT NULL and add FK constraint ===== + op.alter_column( + "llm_configs", + "search_space_id", + nullable=False, + ) + + # Add foreign key constraint + foreign_keys = [fk["name"] for fk in inspector.get_foreign_keys("llm_configs")] + if "fk_llm_configs_search_space_id" not in foreign_keys: + op.create_foreign_key( + "fk_llm_configs_search_space_id", + "llm_configs", + "searchspaces", + ["search_space_id"], + ["id"], + ondelete="CASCADE", + ) + + # Drop old user_id foreign key if it exists + if "fk_llm_configs_user_id_user" in foreign_keys: + op.drop_constraint( + "fk_llm_configs_user_id_user", + "llm_configs", + type_="foreignkey", + ) + + # Remove user_id column + if "user_id" in llm_config_columns: + op.drop_column("llm_configs", "user_id") + + # ===== STEP 4: Create user_search_space_preferences table ===== + op.execute( + """ + DO $$ + BEGIN + IF NOT EXISTS ( + SELECT FROM information_schema.tables + WHERE table_name = 'user_search_space_preferences' + ) THEN + CREATE TABLE user_search_space_preferences ( + id SERIAL PRIMARY KEY, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), + user_id UUID NOT NULL REFERENCES "user"(id) ON DELETE CASCADE, + search_space_id INTEGER NOT NULL REFERENCES searchspaces(id) ON DELETE CASCADE, + long_context_llm_id INTEGER REFERENCES llm_configs(id) ON DELETE SET NULL, + fast_llm_id INTEGER REFERENCES llm_configs(id) ON DELETE SET NULL, + strategic_llm_id INTEGER REFERENCES llm_configs(id) ON DELETE SET NULL, + CONSTRAINT uq_user_searchspace UNIQUE (user_id, search_space_id) + ); + END IF; + END$$; + """ + ) + + # Create indexes + op.execute( + """ + DO $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_indexes + WHERE tablename = 'user_search_space_preferences' + AND indexname = 'ix_user_search_space_preferences_id' + ) THEN + CREATE INDEX ix_user_search_space_preferences_id + ON user_search_space_preferences(id); + END IF; + + IF NOT EXISTS ( + SELECT 1 FROM pg_indexes + WHERE tablename = 'user_search_space_preferences' + AND indexname = 'ix_user_search_space_preferences_created_at' + ) THEN + CREATE INDEX ix_user_search_space_preferences_created_at + ON user_search_space_preferences(created_at); + END IF; + END$$; + """ + ) + + # ===== STEP 5: Migrate user LLM preferences to user_search_space_preferences ===== + # For each user, create preferences for each of their search spaces + if all( + col in user_columns + for col in ["long_context_llm_id", "fast_llm_id", "strategic_llm_id"] + ): + op.execute( + """ + INSERT INTO user_search_space_preferences + (user_id, search_space_id, long_context_llm_id, fast_llm_id, strategic_llm_id, created_at) + SELECT + u.id as user_id, + ss.id as search_space_id, + u.long_context_llm_id, + u.fast_llm_id, + u.strategic_llm_id, + NOW() as created_at + FROM "user" u + CROSS JOIN searchspaces ss + WHERE ss.user_id = u.id + ON CONFLICT (user_id, search_space_id) DO NOTHING + """ + ) + + # ===== STEP 6: Remove LLM preference columns from user table ===== + # Get fresh list of foreign keys after previous operations + user_foreign_keys = [fk["name"] for fk in inspector.get_foreign_keys("user")] + + # Drop foreign key constraints if they exist + if "fk_user_long_context_llm_id_llm_configs" in user_foreign_keys: + op.drop_constraint( + "fk_user_long_context_llm_id_llm_configs", + "user", + type_="foreignkey", + ) + + if "fk_user_fast_llm_id_llm_configs" in user_foreign_keys: + op.drop_constraint( + "fk_user_fast_llm_id_llm_configs", + "user", + type_="foreignkey", + ) + + if "fk_user_strategic_llm_id_llm_configs" in user_foreign_keys: + op.drop_constraint( + "fk_user_strategic_llm_id_llm_configs", + "user", + type_="foreignkey", + ) + + # Drop columns from user table + if "long_context_llm_id" in user_columns: + op.drop_column("user", "long_context_llm_id") + + if "fast_llm_id" in user_columns: + op.drop_column("user", "fast_llm_id") + + if "strategic_llm_id" in user_columns: + op.drop_column("user", "strategic_llm_id") + + +def downgrade() -> None: + """ + Downgrade schema back to user-owned LLM configs. + + WARNING: This downgrade will result in data loss: + - LLM configs will be moved back to user ownership (first occurrence kept) + - Per-search-space user preferences will be consolidated to user level + - Additional LLM configs in search spaces beyond the first will be deleted + """ + + from sqlalchemy import inspect + + conn = op.get_bind() + inspector = inspect(conn) + + # Get existing columns and constraints + llm_config_columns = [col["name"] for col in inspector.get_columns("llm_configs")] + user_columns = [col["name"] for col in inspector.get_columns("user")] + + # ===== STEP 1: Add LLM preference columns back to user table ===== + if "long_context_llm_id" not in user_columns: + op.add_column( + "user", + sa.Column("long_context_llm_id", sa.Integer(), nullable=True), + ) + + if "fast_llm_id" not in user_columns: + op.add_column( + "user", + sa.Column("fast_llm_id", sa.Integer(), nullable=True), + ) + + if "strategic_llm_id" not in user_columns: + op.add_column( + "user", + sa.Column("strategic_llm_id", sa.Integer(), nullable=True), + ) + + # ===== STEP 2: Migrate preferences back to user table ===== + # Take the first preference for each user + op.execute( + """ + UPDATE "user" u + SET + long_context_llm_id = ussp.long_context_llm_id, + fast_llm_id = ussp.fast_llm_id, + strategic_llm_id = ussp.strategic_llm_id + FROM ( + SELECT DISTINCT ON (user_id) + user_id, + long_context_llm_id, + fast_llm_id, + strategic_llm_id + FROM user_search_space_preferences + ORDER BY user_id, created_at ASC + ) ussp + WHERE u.id = ussp.user_id + """ + ) + + # ===== STEP 3: Add foreign key constraints back to user table ===== + op.create_foreign_key( + "fk_user_long_context_llm_id_llm_configs", + "user", + "llm_configs", + ["long_context_llm_id"], + ["id"], + ondelete="SET NULL", + ) + + op.create_foreign_key( + "fk_user_fast_llm_id_llm_configs", + "user", + "llm_configs", + ["fast_llm_id"], + ["id"], + ondelete="SET NULL", + ) + + op.create_foreign_key( + "fk_user_strategic_llm_id_llm_configs", + "user", + "llm_configs", + ["strategic_llm_id"], + ["id"], + ondelete="SET NULL", + ) + + # ===== STEP 4: Drop user_search_space_preferences table ===== + op.execute("DROP TABLE IF EXISTS user_search_space_preferences CASCADE") + + # ===== STEP 5: Add user_id back to llm_configs ===== + if "user_id" not in llm_config_columns: + op.add_column( + "llm_configs", + sa.Column("user_id", postgresql.UUID(), nullable=True), + ) + + # Populate user_id from search_space + op.execute( + """ + UPDATE llm_configs lc + SET user_id = ss.user_id + FROM searchspaces ss + WHERE lc.search_space_id = ss.id + """ + ) + + # Make user_id NOT NULL + op.alter_column( + "llm_configs", + "user_id", + nullable=False, + ) + + # Add foreign key constraint for user_id + op.create_foreign_key( + "fk_llm_configs_user_id_user", + "llm_configs", + "user", + ["user_id"], + ["id"], + ondelete="CASCADE", + ) + + # ===== STEP 6: Remove search_space_id from llm_configs ===== + # Drop foreign key constraint + foreign_keys = [fk["name"] for fk in inspector.get_foreign_keys("llm_configs")] + if "fk_llm_configs_search_space_id" in foreign_keys: + op.drop_constraint( + "fk_llm_configs_search_space_id", + "llm_configs", + type_="foreignkey", + ) + + # Drop search_space_id column + if "search_space_id" in llm_config_columns: + op.drop_column("llm_configs", "search_space_id") diff --git a/surfsense_backend/app/agents/podcaster/configuration.py b/surfsense_backend/app/agents/podcaster/configuration.py index c4c5f9e9c..453f12676 100644 --- a/surfsense_backend/app/agents/podcaster/configuration.py +++ b/surfsense_backend/app/agents/podcaster/configuration.py @@ -17,6 +17,7 @@ class Configuration: # and when you invoke the graph podcast_title: str user_id: str + search_space_id: int @classmethod def from_runnable_config( diff --git a/surfsense_backend/app/agents/podcaster/nodes.py b/surfsense_backend/app/agents/podcaster/nodes.py index 891928d90..bce9882d6 100644 --- a/surfsense_backend/app/agents/podcaster/nodes.py +++ b/surfsense_backend/app/agents/podcaster/nodes.py @@ -28,11 +28,12 @@ async def create_podcast_transcript( # Get configuration from runnable config configuration = Configuration.from_runnable_config(config) user_id = configuration.user_id + search_space_id = configuration.search_space_id # Get user's long context LLM - llm = await get_user_long_context_llm(state.db_session, user_id) + llm = await get_user_long_context_llm(state.db_session, user_id, search_space_id) if not llm: - error_message = f"No long context LLM configured for user {user_id}" + error_message = f"No long context LLM configured for user {user_id} in search space {search_space_id}" print(error_message) raise RuntimeError(error_message) diff --git a/surfsense_backend/app/agents/researcher/nodes.py b/surfsense_backend/app/agents/researcher/nodes.py index 5cd314d6d..0835fb861 100644 --- a/surfsense_backend/app/agents/researcher/nodes.py +++ b/surfsense_backend/app/agents/researcher/nodes.py @@ -577,6 +577,7 @@ async def write_answer_outline( user_query = configuration.user_query num_sections = configuration.num_sections user_id = configuration.user_id + search_space_id = configuration.search_space_id writer( { @@ -587,9 +588,9 @@ async def write_answer_outline( ) # Get user's strategic LLM - llm = await get_user_strategic_llm(state.db_session, user_id) + llm = await get_user_strategic_llm(state.db_session, user_id, search_space_id) if not llm: - error_message = f"No strategic LLM configured for user {user_id}" + error_message = f"No strategic LLM configured for user {user_id} in search space {search_space_id}" writer({"yield_value": streaming_service.format_error(error_message)}) raise RuntimeError(error_message) @@ -1854,6 +1855,7 @@ async def reformulate_user_query( user_query=user_query, session=state.db_session, user_id=configuration.user_id, + search_space_id=configuration.search_space_id, chat_history_str=chat_history_str, ) @@ -2093,6 +2095,7 @@ async def generate_further_questions( configuration = Configuration.from_runnable_config(config) chat_history = state.chat_history user_id = configuration.user_id + search_space_id = configuration.search_space_id streaming_service = state.streaming_service # Get reranked documents from the state (will be populated by sub-agents) @@ -2107,9 +2110,9 @@ async def generate_further_questions( ) # Get user's fast LLM - llm = await get_user_fast_llm(state.db_session, user_id) + llm = await get_user_fast_llm(state.db_session, user_id, search_space_id) if not llm: - error_message = f"No fast LLM configured for user {user_id}" + error_message = f"No fast LLM configured for user {user_id} in search space {search_space_id}" print(error_message) writer({"yield_value": streaming_service.format_error(error_message)}) diff --git a/surfsense_backend/app/agents/researcher/qna_agent/nodes.py b/surfsense_backend/app/agents/researcher/qna_agent/nodes.py index 4e01bbb58..fd6861efb 100644 --- a/surfsense_backend/app/agents/researcher/qna_agent/nodes.py +++ b/surfsense_backend/app/agents/researcher/qna_agent/nodes.py @@ -101,11 +101,12 @@ async def answer_question(state: State, config: RunnableConfig) -> dict[str, Any documents = state.reranked_documents user_query = configuration.user_query user_id = configuration.user_id + search_space_id = configuration.search_space_id # Get user's fast LLM - llm = await get_user_fast_llm(state.db_session, user_id) + llm = await get_user_fast_llm(state.db_session, user_id, search_space_id) if not llm: - error_message = f"No fast LLM configured for user {user_id}" + error_message = f"No fast LLM configured for user {user_id} in search space {search_space_id}" print(error_message) raise RuntimeError(error_message) diff --git a/surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py b/surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py index 91a8bf84e..153cafac5 100644 --- a/surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py +++ b/surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py @@ -107,11 +107,12 @@ async def write_sub_section(state: State, config: RunnableConfig) -> dict[str, A configuration = Configuration.from_runnable_config(config) documents = state.reranked_documents user_id = configuration.user_id + search_space_id = configuration.search_space_id # Get user's fast LLM - llm = await get_user_fast_llm(state.db_session, user_id) + llm = await get_user_fast_llm(state.db_session, user_id, search_space_id) if not llm: - error_message = f"No fast LLM configured for user {user_id}" + error_message = f"No fast LLM configured for user {user_id} in search space {search_space_id}" print(error_message) raise RuntimeError(error_message) diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py index 7af9c0661..eb33145cf 100644 --- a/surfsense_backend/app/db.py +++ b/surfsense_backend/app/db.py @@ -240,6 +240,17 @@ class SearchSpace(BaseModel, TimestampMixin): order_by="SearchSourceConnector.id", cascade="all, delete-orphan", ) + llm_configs = relationship( + "LLMConfig", + back_populates="search_space", + order_by="LLMConfig.id", + cascade="all, delete-orphan", + ) + user_preferences = relationship( + "UserSearchSpacePreference", + back_populates="search_space", + cascade="all, delete-orphan", + ) class SearchSourceConnector(BaseModel, TimestampMixin): @@ -288,10 +299,54 @@ class LLMConfig(BaseModel, TimestampMixin): # For any other parameters that litellm supports litellm_params = Column(JSON, nullable=True, default={}) + search_space_id = Column( + Integer, ForeignKey("searchspaces.id", ondelete="CASCADE"), nullable=False + ) + search_space = relationship("SearchSpace", back_populates="llm_configs") + + +class UserSearchSpacePreference(BaseModel, TimestampMixin): + __tablename__ = "user_search_space_preferences" + __table_args__ = ( + UniqueConstraint( + "user_id", + "search_space_id", + name="uq_user_searchspace", + ), + ) + user_id = Column( UUID(as_uuid=True), ForeignKey("user.id", ondelete="CASCADE"), nullable=False ) - user = relationship("User", back_populates="llm_configs", foreign_keys=[user_id]) + search_space_id = Column( + Integer, ForeignKey("searchspaces.id", ondelete="CASCADE"), nullable=False + ) + + # User-specific LLM preferences for this search space + long_context_llm_id = Column( + Integer, ForeignKey("llm_configs.id", ondelete="SET NULL"), nullable=True + ) + fast_llm_id = Column( + Integer, ForeignKey("llm_configs.id", ondelete="SET NULL"), nullable=True + ) + strategic_llm_id = Column( + Integer, ForeignKey("llm_configs.id", ondelete="SET NULL"), nullable=True + ) + + # Future RBAC fields can be added here + # role = Column(String(50), nullable=True) # e.g., 'owner', 'editor', 'viewer' + # permissions = Column(JSON, nullable=True) + + user = relationship("User", back_populates="search_space_preferences") + search_space = relationship("SearchSpace", back_populates="user_preferences") + + long_context_llm = relationship( + "LLMConfig", foreign_keys=[long_context_llm_id], post_update=True + ) + fast_llm = relationship("LLMConfig", foreign_keys=[fast_llm_id], post_update=True) + strategic_llm = relationship( + "LLMConfig", foreign_keys=[strategic_llm_id], post_update=True + ) class Log(BaseModel, TimestampMixin): @@ -321,64 +376,22 @@ if config.AUTH_TYPE == "GOOGLE": "OAuthAccount", lazy="joined" ) search_spaces = relationship("SearchSpace", back_populates="user") - llm_configs = relationship( - "LLMConfig", + search_space_preferences = relationship( + "UserSearchSpacePreference", back_populates="user", - foreign_keys="LLMConfig.user_id", cascade="all, delete-orphan", ) - long_context_llm_id = Column( - Integer, ForeignKey("llm_configs.id", ondelete="SET NULL"), nullable=True - ) - fast_llm_id = Column( - Integer, ForeignKey("llm_configs.id", ondelete="SET NULL"), nullable=True - ) - strategic_llm_id = Column( - Integer, ForeignKey("llm_configs.id", ondelete="SET NULL"), nullable=True - ) - - long_context_llm = relationship( - "LLMConfig", foreign_keys=[long_context_llm_id], post_update=True - ) - fast_llm = relationship( - "LLMConfig", foreign_keys=[fast_llm_id], post_update=True - ) - strategic_llm = relationship( - "LLMConfig", foreign_keys=[strategic_llm_id], post_update=True - ) - else: class User(SQLAlchemyBaseUserTableUUID, Base): search_spaces = relationship("SearchSpace", back_populates="user") - llm_configs = relationship( - "LLMConfig", + search_space_preferences = relationship( + "UserSearchSpacePreference", back_populates="user", - foreign_keys="LLMConfig.user_id", cascade="all, delete-orphan", ) - long_context_llm_id = Column( - Integer, ForeignKey("llm_configs.id", ondelete="SET NULL"), nullable=True - ) - fast_llm_id = Column( - Integer, ForeignKey("llm_configs.id", ondelete="SET NULL"), nullable=True - ) - strategic_llm_id = Column( - Integer, ForeignKey("llm_configs.id", ondelete="SET NULL"), nullable=True - ) - - long_context_llm = relationship( - "LLMConfig", foreign_keys=[long_context_llm_id], post_update=True - ) - fast_llm = relationship( - "LLMConfig", foreign_keys=[fast_llm_id], post_update=True - ) - strategic_llm = relationship( - "LLMConfig", foreign_keys=[strategic_llm_id], post_update=True - ) - engine = create_async_engine(DATABASE_URL) async_session_maker = async_sessionmaker(engine, expire_on_commit=False) diff --git a/surfsense_backend/app/routes/chats_routes.py b/surfsense_backend/app/routes/chats_routes.py index 1dcda505c..e4d02686f 100644 --- a/surfsense_backend/app/routes/chats_routes.py +++ b/surfsense_backend/app/routes/chats_routes.py @@ -17,19 +17,17 @@ from app.tasks.stream_connector_search_results import stream_connector_search_re from app.users import current_active_user from app.utils.check_ownership import check_ownership from app.utils.validators import ( - validate_search_space_id, - validate_document_ids, validate_connectors, + validate_document_ids, + validate_messages, validate_research_mode, validate_search_mode, - validate_messages, + validate_search_space_id, ) router = APIRouter() - - @router.post("/chat") async def handle_chat_data( request: AISDKChatRequest, @@ -38,20 +36,22 @@ async def handle_chat_data( ): # Validate and sanitize all input data messages = validate_messages(request.messages) - + if messages[-1]["role"] != "user": raise HTTPException( status_code=400, detail="Last message must be a user message" ) user_query = messages[-1]["content"] - + # Extract and validate data from request request_data = request.data or {} search_space_id = validate_search_space_id(request_data.get("search_space_id")) research_mode = validate_research_mode(request_data.get("research_mode")) selected_connectors = validate_connectors(request_data.get("selected_connectors")) - document_ids_to_add_in_context = validate_document_ids(request_data.get("document_ids_to_add_in_context")) + document_ids_to_add_in_context = validate_document_ids( + request_data.get("document_ids_to_add_in_context") + ) search_mode_str = validate_search_mode(request_data.get("search_mode")) # Check if the search space belongs to the current user @@ -132,21 +132,16 @@ async def read_chats( # Validate pagination parameters if skip < 0: raise HTTPException( - status_code=400, - detail="skip must be a non-negative integer" + status_code=400, detail="skip must be a non-negative integer" ) - + if limit <= 0 or limit > 1000: # Reasonable upper limit - raise HTTPException( - status_code=400, - detail="limit must be between 1 and 1000" - ) - + raise HTTPException(status_code=400, detail="limit must be between 1 and 1000") + # Validate search_space_id if provided if search_space_id is not None and search_space_id <= 0: raise HTTPException( - status_code=400, - detail="search_space_id must be a positive integer" + status_code=400, detail="search_space_id must be a positive integer" ) try: # Select specific fields excluding messages diff --git a/surfsense_backend/app/routes/llm_config_routes.py b/surfsense_backend/app/routes/llm_config_routes.py index ce76dc9bc..63d540d2c 100644 --- a/surfsense_backend/app/routes/llm_config_routes.py +++ b/surfsense_backend/app/routes/llm_config_routes.py @@ -2,15 +2,72 @@ from fastapi import APIRouter, Depends, HTTPException from pydantic import BaseModel from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select +from sqlalchemy.orm import selectinload -from app.db import LLMConfig, User, get_async_session +from app.db import ( + LLMConfig, + SearchSpace, + User, + UserSearchSpacePreference, + get_async_session, +) from app.schemas import LLMConfigCreate, LLMConfigRead, LLMConfigUpdate from app.users import current_active_user -from app.utils.check_ownership import check_ownership router = APIRouter() +# Helper function to check search space access +async def check_search_space_access( + session: AsyncSession, search_space_id: int, user: User +) -> SearchSpace: + """Verify that the user has access to the search space""" + result = await session.execute( + select(SearchSpace).filter( + SearchSpace.id == search_space_id, SearchSpace.user_id == user.id + ) + ) + search_space = result.scalars().first() + if not search_space: + raise HTTPException( + status_code=404, + detail="Search space not found or you don't have permission to access it", + ) + return search_space + + +# Helper function to get or create user search space preference +async def get_or_create_user_preference( + session: AsyncSession, user_id, search_space_id: int +) -> UserSearchSpacePreference: + """Get or create user preference for a search space""" + result = await session.execute( + select(UserSearchSpacePreference) + .filter( + UserSearchSpacePreference.user_id == user_id, + UserSearchSpacePreference.search_space_id == search_space_id, + ) + .options( + selectinload(UserSearchSpacePreference.long_context_llm), + selectinload(UserSearchSpacePreference.fast_llm), + selectinload(UserSearchSpacePreference.strategic_llm), + ) + ) + preference = result.scalars().first() + + if not preference: + # Create new preference entry + preference = UserSearchSpacePreference( + user_id=user_id, + search_space_id=search_space_id, + ) + session.add(preference) + await session.commit() + await session.refresh(preference) + + return preference + + class LLMPreferencesUpdate(BaseModel): """Schema for updating user LLM preferences""" @@ -36,9 +93,12 @@ async def create_llm_config( session: AsyncSession = Depends(get_async_session), user: User = Depends(current_active_user), ): - """Create a new LLM configuration for the authenticated user""" + """Create a new LLM configuration for a search space""" try: - db_llm_config = LLMConfig(**llm_config.model_dump(), user_id=user.id) + # Verify user has access to the search space + await check_search_space_access(session, llm_config.search_space_id, user) + + db_llm_config = LLMConfig(**llm_config.model_dump()) session.add(db_llm_config) await session.commit() await session.refresh(db_llm_config) @@ -54,20 +114,26 @@ async def create_llm_config( @router.get("/llm-configs/", response_model=list[LLMConfigRead]) async def read_llm_configs( + search_space_id: int, skip: int = 0, limit: int = 200, session: AsyncSession = Depends(get_async_session), user: User = Depends(current_active_user), ): - """Get all LLM configurations for the authenticated user""" + """Get all LLM configurations for a search space""" try: + # Verify user has access to the search space + await check_search_space_access(session, search_space_id, user) + result = await session.execute( select(LLMConfig) - .filter(LLMConfig.user_id == user.id) + .filter(LLMConfig.search_space_id == search_space_id) .offset(skip) .limit(limit) ) return result.scalars().all() + except HTTPException: + raise except Exception as e: raise HTTPException( status_code=500, detail=f"Failed to fetch LLM configurations: {e!s}" @@ -82,7 +148,18 @@ async def read_llm_config( ): """Get a specific LLM configuration by ID""" try: - llm_config = await check_ownership(session, LLMConfig, llm_config_id, user) + # Get the LLM config + result = await session.execute( + select(LLMConfig).filter(LLMConfig.id == llm_config_id) + ) + llm_config = result.scalars().first() + + if not llm_config: + raise HTTPException(status_code=404, detail="LLM configuration not found") + + # Verify user has access to the search space + await check_search_space_access(session, llm_config.search_space_id, user) + return llm_config except HTTPException: raise @@ -101,7 +178,18 @@ async def update_llm_config( ): """Update an existing LLM configuration""" try: - db_llm_config = await check_ownership(session, LLMConfig, llm_config_id, user) + # Get the LLM config + result = await session.execute( + select(LLMConfig).filter(LLMConfig.id == llm_config_id) + ) + db_llm_config = result.scalars().first() + + if not db_llm_config: + raise HTTPException(status_code=404, detail="LLM configuration not found") + + # Verify user has access to the search space + await check_search_space_access(session, db_llm_config.search_space_id, user) + update_data = llm_config_update.model_dump(exclude_unset=True) for key, value in update_data.items(): @@ -127,7 +215,18 @@ async def delete_llm_config( ): """Delete an LLM configuration""" try: - db_llm_config = await check_ownership(session, LLMConfig, llm_config_id, user) + # Get the LLM config + result = await session.execute( + select(LLMConfig).filter(LLMConfig.id == llm_config_id) + ) + db_llm_config = result.scalars().first() + + if not db_llm_config: + raise HTTPException(status_code=404, detail="LLM configuration not found") + + # Verify user has access to the search space + await check_search_space_access(session, db_llm_config.search_space_id, user) + await session.delete(db_llm_config) await session.commit() return {"message": "LLM configuration deleted successfully"} @@ -143,99 +242,101 @@ async def delete_llm_config( # User LLM Preferences endpoints -@router.get("/users/me/llm-preferences", response_model=LLMPreferencesRead) +@router.get( + "/search-spaces/{search_space_id}/llm-preferences", + response_model=LLMPreferencesRead, +) async def get_user_llm_preferences( + search_space_id: int, session: AsyncSession = Depends(get_async_session), user: User = Depends(current_active_user), ): - """Get the current user's LLM preferences""" + """Get the current user's LLM preferences for a specific search space""" try: - # Refresh user to get latest relationships - await session.refresh(user) + # Verify user has access to the search space + await check_search_space_access(session, search_space_id, user) - result = { - "long_context_llm_id": user.long_context_llm_id, - "fast_llm_id": user.fast_llm_id, - "strategic_llm_id": user.strategic_llm_id, - "long_context_llm": None, - "fast_llm": None, - "strategic_llm": None, + # Get or create user preference for this search space + preference = await get_or_create_user_preference( + session, user.id, search_space_id + ) + + return { + "long_context_llm_id": preference.long_context_llm_id, + "fast_llm_id": preference.fast_llm_id, + "strategic_llm_id": preference.strategic_llm_id, + "long_context_llm": preference.long_context_llm, + "fast_llm": preference.fast_llm, + "strategic_llm": preference.strategic_llm, } - - # Fetch the actual LLM configs if they exist - if user.long_context_llm_id: - long_context_llm = await session.execute( - select(LLMConfig).filter( - LLMConfig.id == user.long_context_llm_id, - LLMConfig.user_id == user.id, - ) - ) - llm_config = long_context_llm.scalars().first() - if llm_config: - result["long_context_llm"] = llm_config - - if user.fast_llm_id: - fast_llm = await session.execute( - select(LLMConfig).filter( - LLMConfig.id == user.fast_llm_id, LLMConfig.user_id == user.id - ) - ) - llm_config = fast_llm.scalars().first() - if llm_config: - result["fast_llm"] = llm_config - - if user.strategic_llm_id: - strategic_llm = await session.execute( - select(LLMConfig).filter( - LLMConfig.id == user.strategic_llm_id, LLMConfig.user_id == user.id - ) - ) - llm_config = strategic_llm.scalars().first() - if llm_config: - result["strategic_llm"] = llm_config - - return result + except HTTPException: + raise except Exception as e: raise HTTPException( status_code=500, detail=f"Failed to fetch LLM preferences: {e!s}" ) from e -@router.put("/users/me/llm-preferences", response_model=LLMPreferencesRead) +@router.put( + "/search-spaces/{search_space_id}/llm-preferences", + response_model=LLMPreferencesRead, +) async def update_user_llm_preferences( + search_space_id: int, preferences: LLMPreferencesUpdate, session: AsyncSession = Depends(get_async_session), user: User = Depends(current_active_user), ): - """Update the current user's LLM preferences""" + """Update the current user's LLM preferences for a specific search space""" try: - # Validate that all provided LLM config IDs belong to the user + # Verify user has access to the search space + await check_search_space_access(session, search_space_id, user) + + # Get or create user preference for this search space + preference = await get_or_create_user_preference( + session, user.id, search_space_id + ) + + # Validate that all provided LLM config IDs belong to the search space update_data = preferences.model_dump(exclude_unset=True) for _key, llm_config_id in update_data.items(): if llm_config_id is not None: - # Verify ownership of the LLM config + # Verify the LLM config belongs to the search space result = await session.execute( select(LLMConfig).filter( - LLMConfig.id == llm_config_id, LLMConfig.user_id == user.id + LLMConfig.id == llm_config_id, + LLMConfig.search_space_id == search_space_id, ) ) llm_config = result.scalars().first() if not llm_config: raise HTTPException( status_code=404, - detail=f"LLM configuration {llm_config_id} not found or you don't have permission to access it", + detail=f"LLM configuration {llm_config_id} not found in this search space", ) # Update user preferences for key, value in update_data.items(): - setattr(user, key, value) + setattr(preference, key, value) await session.commit() - await session.refresh(user) + await session.refresh(preference) + + # Reload relationships + await session.refresh( + preference, ["long_context_llm", "fast_llm", "strategic_llm"] + ) # Return updated preferences - return await get_user_llm_preferences(session, user) + return { + "long_context_llm_id": preference.long_context_llm_id, + "fast_llm_id": preference.fast_llm_id, + "strategic_llm_id": preference.strategic_llm_id, + "long_context_llm": preference.long_context_llm, + "fast_llm": preference.fast_llm, + "strategic_llm": preference.strategic_llm, + } except HTTPException: raise except Exception as e: diff --git a/surfsense_backend/app/schemas/llm_config.py b/surfsense_backend/app/schemas/llm_config.py index c3c003397..8beb65347 100644 --- a/surfsense_backend/app/schemas/llm_config.py +++ b/surfsense_backend/app/schemas/llm_config.py @@ -1,4 +1,3 @@ -import uuid from datetime import datetime from typing import Any @@ -30,7 +29,9 @@ class LLMConfigBase(BaseModel): class LLMConfigCreate(LLMConfigBase): - pass + search_space_id: int = Field( + ..., description="Search space ID to associate the LLM config with" + ) class LLMConfigUpdate(BaseModel): @@ -56,6 +57,6 @@ class LLMConfigUpdate(BaseModel): class LLMConfigRead(LLMConfigBase, IDModel, TimestampModel): id: int created_at: datetime - user_id: uuid.UUID + search_space_id: int model_config = ConfigDict(from_attributes=True) diff --git a/surfsense_backend/app/schemas/search_space.py b/surfsense_backend/app/schemas/search_space.py index 49b5cd094..00bfdc0f6 100644 --- a/surfsense_backend/app/schemas/search_space.py +++ b/surfsense_backend/app/schemas/search_space.py @@ -24,4 +24,4 @@ class SearchSpaceRead(SearchSpaceBase, IDModel, TimestampModel): created_at: datetime user_id: uuid.UUID - model_config = ConfigDict(from_attributes=True) \ No newline at end of file + model_config = ConfigDict(from_attributes=True) diff --git a/surfsense_backend/app/services/llm_service.py b/surfsense_backend/app/services/llm_service.py index 9135e49dc..d9299549c 100644 --- a/surfsense_backend/app/services/llm_service.py +++ b/surfsense_backend/app/services/llm_service.py @@ -1,10 +1,14 @@ import logging +import litellm from langchain_litellm import ChatLiteLLM from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select -from app.db import LLMConfig, User +from app.db import LLMConfig, UserSearchSpacePreference + +# Configure litellm to automatically drop unsupported parameters +litellm.drop_params = True logger = logging.getLogger(__name__) @@ -16,54 +20,67 @@ class LLMRole: async def get_user_llm_instance( - session: AsyncSession, user_id: str, role: str + session: AsyncSession, user_id: str, search_space_id: int, role: str ) -> ChatLiteLLM | None: """ - Get a ChatLiteLLM instance for a specific user and role. + Get a ChatLiteLLM instance for a specific user, search space, and role. Args: session: Database session user_id: User ID + search_space_id: Search Space ID role: LLM role ('long_context', 'fast', or 'strategic') Returns: ChatLiteLLM instance or None if not found """ try: - # Get user with their LLM preferences - result = await session.execute(select(User).where(User.id == user_id)) - user = result.scalars().first() + # Get user's LLM preferences for this search space + result = await session.execute( + select(UserSearchSpacePreference).where( + UserSearchSpacePreference.user_id == user_id, + UserSearchSpacePreference.search_space_id == search_space_id, + ) + ) + preference = result.scalars().first() - if not user: - logger.error(f"User {user_id} not found") + if not preference: + logger.error( + f"No LLM preferences found for user {user_id} in search space {search_space_id}" + ) return None # Get the appropriate LLM config ID based on role llm_config_id = None if role == LLMRole.LONG_CONTEXT: - llm_config_id = user.long_context_llm_id + llm_config_id = preference.long_context_llm_id elif role == LLMRole.FAST: - llm_config_id = user.fast_llm_id + llm_config_id = preference.fast_llm_id elif role == LLMRole.STRATEGIC: - llm_config_id = user.strategic_llm_id + llm_config_id = preference.strategic_llm_id else: logger.error(f"Invalid LLM role: {role}") return None if not llm_config_id: - logger.error(f"No {role} LLM configured for user {user_id}") + logger.error( + f"No {role} LLM configured for user {user_id} in search space {search_space_id}" + ) return None # Get the LLM configuration result = await session.execute( select(LLMConfig).where( - LLMConfig.id == llm_config_id, LLMConfig.user_id == user_id + LLMConfig.id == llm_config_id, + LLMConfig.search_space_id == search_space_id, ) ) llm_config = result.scalars().first() if not llm_config: - logger.error(f"LLM config {llm_config_id} not found for user {user_id}") + logger.error( + f"LLM config {llm_config_id} not found in search space {search_space_id}" + ) return None # Build the model string for litellm @@ -113,19 +130,25 @@ async def get_user_llm_instance( async def get_user_long_context_llm( - session: AsyncSession, user_id: str + session: AsyncSession, user_id: str, search_space_id: int ) -> ChatLiteLLM | None: - """Get user's long context LLM instance.""" - return await get_user_llm_instance(session, user_id, LLMRole.LONG_CONTEXT) + """Get user's long context LLM instance for a specific search space.""" + return await get_user_llm_instance( + session, user_id, search_space_id, LLMRole.LONG_CONTEXT + ) -async def get_user_fast_llm(session: AsyncSession, user_id: str) -> ChatLiteLLM | None: - """Get user's fast LLM instance.""" - return await get_user_llm_instance(session, user_id, LLMRole.FAST) +async def get_user_fast_llm( + session: AsyncSession, user_id: str, search_space_id: int +) -> ChatLiteLLM | None: + """Get user's fast LLM instance for a specific search space.""" + return await get_user_llm_instance(session, user_id, search_space_id, LLMRole.FAST) async def get_user_strategic_llm( - session: AsyncSession, user_id: str + session: AsyncSession, user_id: str, search_space_id: int ) -> ChatLiteLLM | None: - """Get user's strategic LLM instance.""" - return await get_user_llm_instance(session, user_id, LLMRole.STRATEGIC) + """Get user's strategic LLM instance for a specific search space.""" + return await get_user_llm_instance( + session, user_id, search_space_id, LLMRole.STRATEGIC + ) diff --git a/surfsense_backend/app/services/query_service.py b/surfsense_backend/app/services/query_service.py index 4a4bc59be..d2759ab27 100644 --- a/surfsense_backend/app/services/query_service.py +++ b/surfsense_backend/app/services/query_service.py @@ -17,6 +17,7 @@ class QueryService: user_query: str, session: AsyncSession, user_id: str, + search_space_id: int, chat_history_str: str | None = None, ) -> str: """ @@ -27,6 +28,7 @@ class QueryService: user_query: The original user query session: Database session for accessing user LLM configs user_id: User ID to get their specific LLM configuration + search_space_id: Search Space ID to get user's LLM preferences chat_history_str: Optional chat history string Returns: @@ -37,10 +39,10 @@ class QueryService: try: # Get the user's strategic LLM instance - llm = await get_user_strategic_llm(session, user_id) + llm = await get_user_strategic_llm(session, user_id, search_space_id) if not llm: print( - f"Warning: No strategic LLM configured for user {user_id}. Using original query." + f"Warning: No strategic LLM configured for user {user_id} in search space {search_space_id}. Using original query." ) return user_query diff --git a/surfsense_backend/app/tasks/connector_indexers/airtable_indexer.py b/surfsense_backend/app/tasks/connector_indexers/airtable_indexer.py index 7ba2f2d44..0cc21bb47 100644 --- a/surfsense_backend/app/tasks/connector_indexers/airtable_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/airtable_indexer.py @@ -260,7 +260,9 @@ async def index_airtable_records( continue # Generate document summary - user_llm = await get_user_long_context_llm(session, user_id) + user_llm = await get_user_long_context_llm( + session, user_id, search_space_id + ) if user_llm: document_metadata = { diff --git a/surfsense_backend/app/tasks/connector_indexers/clickup_indexer.py b/surfsense_backend/app/tasks/connector_indexers/clickup_indexer.py index 3120fcbc7..5ee7342fa 100644 --- a/surfsense_backend/app/tasks/connector_indexers/clickup_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/clickup_indexer.py @@ -222,7 +222,9 @@ async def index_clickup_tasks( continue # Generate summary with metadata - user_llm = await get_user_long_context_llm(session, user_id) + user_llm = await get_user_long_context_llm( + session, user_id, search_space_id + ) if user_llm: document_metadata = { diff --git a/surfsense_backend/app/tasks/connector_indexers/confluence_indexer.py b/surfsense_backend/app/tasks/connector_indexers/confluence_indexer.py index 625992922..28cb3b1f4 100644 --- a/surfsense_backend/app/tasks/connector_indexers/confluence_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/confluence_indexer.py @@ -233,7 +233,9 @@ async def index_confluence_pages( continue # Generate summary with metadata - user_llm = await get_user_long_context_llm(session, user_id) + user_llm = await get_user_long_context_llm( + session, user_id, search_space_id + ) comment_count = len(comments) if user_llm: diff --git a/surfsense_backend/app/tasks/connector_indexers/discord_indexer.py b/surfsense_backend/app/tasks/connector_indexers/discord_indexer.py index c538f12f7..08c995f64 100644 --- a/surfsense_backend/app/tasks/connector_indexers/discord_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/discord_indexer.py @@ -325,7 +325,9 @@ async def index_discord_messages( continue # Get user's long context LLM - user_llm = await get_user_long_context_llm(session, user_id) + user_llm = await get_user_long_context_llm( + session, user_id, search_space_id + ) if not user_llm: logger.error( f"No long context LLM configured for user {user_id}" diff --git a/surfsense_backend/app/tasks/connector_indexers/github_indexer.py b/surfsense_backend/app/tasks/connector_indexers/github_indexer.py index ba01e3979..9cc0c0993 100644 --- a/surfsense_backend/app/tasks/connector_indexers/github_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/github_indexer.py @@ -213,7 +213,9 @@ async def index_github_repos( continue # Generate summary with metadata - user_llm = await get_user_long_context_llm(session, user_id) + user_llm = await get_user_long_context_llm( + session, user_id, search_space_id + ) if user_llm: # Extract file extension from file path file_extension = ( diff --git a/surfsense_backend/app/tasks/connector_indexers/google_calendar_indexer.py b/surfsense_backend/app/tasks/connector_indexers/google_calendar_indexer.py index 4d4794284..be5169612 100644 --- a/surfsense_backend/app/tasks/connector_indexers/google_calendar_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/google_calendar_indexer.py @@ -266,7 +266,9 @@ async def index_google_calendar_events( continue # Generate summary with metadata - user_llm = await get_user_long_context_llm(session, user_id) + user_llm = await get_user_long_context_llm( + session, user_id, search_space_id + ) if user_llm: document_metadata = { diff --git a/surfsense_backend/app/tasks/connector_indexers/google_gmail_indexer.py b/surfsense_backend/app/tasks/connector_indexers/google_gmail_indexer.py index 38fec29a9..872e19d03 100644 --- a/surfsense_backend/app/tasks/connector_indexers/google_gmail_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/google_gmail_indexer.py @@ -210,7 +210,9 @@ async def index_google_gmail_messages( continue # Generate summary with metadata - user_llm = await get_user_long_context_llm(session, user_id) + user_llm = await get_user_long_context_llm( + session, user_id, search_space_id + ) if user_llm: document_metadata = { diff --git a/surfsense_backend/app/tasks/connector_indexers/jira_indexer.py b/surfsense_backend/app/tasks/connector_indexers/jira_indexer.py index c199faed0..e9d556954 100644 --- a/surfsense_backend/app/tasks/connector_indexers/jira_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/jira_indexer.py @@ -216,7 +216,9 @@ async def index_jira_issues( continue # Generate summary with metadata - user_llm = await get_user_long_context_llm(session, user_id) + user_llm = await get_user_long_context_llm( + session, user_id, search_space_id + ) comment_count = len(formatted_issue.get("comments", [])) if user_llm: diff --git a/surfsense_backend/app/tasks/connector_indexers/linear_indexer.py b/surfsense_backend/app/tasks/connector_indexers/linear_indexer.py index 6ca145357..aca1e2040 100644 --- a/surfsense_backend/app/tasks/connector_indexers/linear_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/linear_indexer.py @@ -228,7 +228,9 @@ async def index_linear_issues( continue # Generate summary with metadata - user_llm = await get_user_long_context_llm(session, user_id) + user_llm = await get_user_long_context_llm( + session, user_id, search_space_id + ) state = formatted_issue.get("state", "Unknown") description = formatted_issue.get("description", "") comment_count = len(formatted_issue.get("comments", [])) diff --git a/surfsense_backend/app/tasks/connector_indexers/luma_indexer.py b/surfsense_backend/app/tasks/connector_indexers/luma_indexer.py index d7b6d3058..3d8970654 100644 --- a/surfsense_backend/app/tasks/connector_indexers/luma_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/luma_indexer.py @@ -270,7 +270,9 @@ async def index_luma_events( continue # Generate summary with metadata - user_llm = await get_user_long_context_llm(session, user_id) + user_llm = await get_user_long_context_llm( + session, user_id, search_space_id + ) if user_llm: document_metadata = { diff --git a/surfsense_backend/app/tasks/connector_indexers/notion_indexer.py b/surfsense_backend/app/tasks/connector_indexers/notion_indexer.py index a6c8853a3..b290f86da 100644 --- a/surfsense_backend/app/tasks/connector_indexers/notion_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/notion_indexer.py @@ -299,7 +299,9 @@ async def index_notion_pages( continue # Get user's long context LLM - user_llm = await get_user_long_context_llm(session, user_id) + user_llm = await get_user_long_context_llm( + session, user_id, search_space_id + ) if not user_llm: logger.error(f"No long context LLM configured for user {user_id}") skipped_pages.append(f"{page_title} (no LLM configured)") diff --git a/surfsense_backend/app/tasks/document_processors/extension_processor.py b/surfsense_backend/app/tasks/document_processors/extension_processor.py index 8f8433148..ed25b8fbd 100644 --- a/surfsense_backend/app/tasks/document_processors/extension_processor.py +++ b/surfsense_backend/app/tasks/document_processors/extension_processor.py @@ -104,9 +104,11 @@ async def add_extension_received_document( return existing_document # Get user's long context LLM - user_llm = await get_user_long_context_llm(session, user_id) + user_llm = await get_user_long_context_llm(session, user_id, search_space_id) if not user_llm: - raise RuntimeError(f"No long context LLM configured for user {user_id}") + raise RuntimeError( + f"No long context LLM configured for user {user_id} in search space {search_space_id}" + ) # Generate summary with metadata document_metadata = { diff --git a/surfsense_backend/app/tasks/document_processors/file_processors.py b/surfsense_backend/app/tasks/document_processors/file_processors.py index 3803f4b2b..573b2c28c 100644 --- a/surfsense_backend/app/tasks/document_processors/file_processors.py +++ b/surfsense_backend/app/tasks/document_processors/file_processors.py @@ -60,9 +60,11 @@ async def add_received_file_document_using_unstructured( # TODO: Check if file_markdown exceeds token limit of embedding model # Get user's long context LLM - user_llm = await get_user_long_context_llm(session, user_id) + user_llm = await get_user_long_context_llm(session, user_id, search_space_id) if not user_llm: - raise RuntimeError(f"No long context LLM configured for user {user_id}") + raise RuntimeError( + f"No long context LLM configured for user {user_id} in search space {search_space_id}" + ) # Generate summary with metadata document_metadata = { @@ -140,9 +142,11 @@ async def add_received_file_document_using_llamacloud( return existing_document # Get user's long context LLM - user_llm = await get_user_long_context_llm(session, user_id) + user_llm = await get_user_long_context_llm(session, user_id, search_space_id) if not user_llm: - raise RuntimeError(f"No long context LLM configured for user {user_id}") + raise RuntimeError( + f"No long context LLM configured for user {user_id} in search space {search_space_id}" + ) # Generate summary with metadata document_metadata = { @@ -221,9 +225,11 @@ async def add_received_file_document_using_docling( return existing_document # Get user's long context LLM - user_llm = await get_user_long_context_llm(session, user_id) + user_llm = await get_user_long_context_llm(session, user_id, search_space_id) if not user_llm: - raise RuntimeError(f"No long context LLM configured for user {user_id}") + raise RuntimeError( + f"No long context LLM configured for user {user_id} in search space {search_space_id}" + ) # Generate summary using chunked processing for large documents from app.services.docling_service import create_docling_service diff --git a/surfsense_backend/app/tasks/document_processors/markdown_processor.py b/surfsense_backend/app/tasks/document_processors/markdown_processor.py index 493b046af..fa3c79d81 100644 --- a/surfsense_backend/app/tasks/document_processors/markdown_processor.py +++ b/surfsense_backend/app/tasks/document_processors/markdown_processor.py @@ -75,9 +75,11 @@ async def add_received_markdown_file_document( return existing_document # Get user's long context LLM - user_llm = await get_user_long_context_llm(session, user_id) + user_llm = await get_user_long_context_llm(session, user_id, search_space_id) if not user_llm: - raise RuntimeError(f"No long context LLM configured for user {user_id}") + raise RuntimeError( + f"No long context LLM configured for user {user_id} in search space {search_space_id}" + ) # Generate summary with metadata document_metadata = { diff --git a/surfsense_backend/app/tasks/document_processors/url_crawler.py b/surfsense_backend/app/tasks/document_processors/url_crawler.py index eddcda388..682086112 100644 --- a/surfsense_backend/app/tasks/document_processors/url_crawler.py +++ b/surfsense_backend/app/tasks/document_processors/url_crawler.py @@ -161,9 +161,11 @@ async def add_crawled_url_document( ) # Get user's long context LLM - user_llm = await get_user_long_context_llm(session, user_id) + user_llm = await get_user_long_context_llm(session, user_id, search_space_id) if not user_llm: - raise RuntimeError(f"No long context LLM configured for user {user_id}") + raise RuntimeError( + f"No long context LLM configured for user {user_id} in search space {search_space_id}" + ) # Generate summary await task_logger.log_task_progress( diff --git a/surfsense_backend/app/tasks/document_processors/youtube_processor.py b/surfsense_backend/app/tasks/document_processors/youtube_processor.py index 37981f6ae..a28a7f186 100644 --- a/surfsense_backend/app/tasks/document_processors/youtube_processor.py +++ b/surfsense_backend/app/tasks/document_processors/youtube_processor.py @@ -234,9 +234,11 @@ async def add_youtube_video_document( ) # Get user's long context LLM - user_llm = await get_user_long_context_llm(session, user_id) + user_llm = await get_user_long_context_llm(session, user_id, search_space_id) if not user_llm: - raise RuntimeError(f"No long context LLM configured for user {user_id}") + raise RuntimeError( + f"No long context LLM configured for user {user_id} in search space {search_space_id}" + ) # Generate summary await task_logger.log_task_progress( diff --git a/surfsense_backend/app/tasks/podcast_tasks.py b/surfsense_backend/app/tasks/podcast_tasks.py index 312ae5bc3..e5f828ef2 100644 --- a/surfsense_backend/app/tasks/podcast_tasks.py +++ b/surfsense_backend/app/tasks/podcast_tasks.py @@ -98,6 +98,7 @@ async def generate_chat_podcast( "configurable": { "podcast_title": "SurfSense", "user_id": str(user_id), + "search_space_id": search_space_id, } } # Initialize state with database session and streaming service diff --git a/surfsense_backend/app/utils/validators.py b/surfsense_backend/app/utils/validators.py index 7677d54f2..437d23b55 100644 --- a/surfsense_backend/app/utils/validators.py +++ b/surfsense_backend/app/utils/validators.py @@ -16,89 +16,80 @@ from fastapi import HTTPException def validate_search_space_id(search_space_id: Any) -> int: """ Validate and convert search_space_id to integer. - + Args: search_space_id: The search space ID to validate - + Returns: int: Validated search space ID - + Raises: HTTPException: If validation fails """ if search_space_id is None: - raise HTTPException( - status_code=400, - detail="search_space_id is required" - ) - + raise HTTPException(status_code=400, detail="search_space_id is required") + if isinstance(search_space_id, bool): raise HTTPException( - status_code=400, - detail="search_space_id must be an integer, not a boolean" + status_code=400, detail="search_space_id must be an integer, not a boolean" ) - + if isinstance(search_space_id, int): if search_space_id <= 0: raise HTTPException( - status_code=400, - detail="search_space_id must be a positive integer" - + status_code=400, detail="search_space_id must be a positive integer" ) return search_space_id - + if isinstance(search_space_id, str): # Check if it's a valid integer string if not search_space_id.strip(): raise HTTPException( - status_code=400, - detail="search_space_id cannot be empty" + status_code=400, detail="search_space_id cannot be empty" ) - + # Check for valid integer format (no leading zeros, no decimal points) - if not re.match(r'^[1-9]\d*$', search_space_id.strip()): + if not re.match(r"^[1-9]\d*$", search_space_id.strip()): raise HTTPException( status_code=400, - detail="search_space_id must be a valid positive integer" + detail="search_space_id must be a valid positive integer", ) - + value = int(search_space_id.strip()) # Regex already guarantees value > 0, but check retained for clarity if value <= 0: raise HTTPException( - status_code=400, - detail="search_space_id must be a positive integer" + status_code=400, detail="search_space_id must be a positive integer" ) return value - + raise HTTPException( status_code=400, - detail="search_space_id must be an integer or string representation of an integer" + detail="search_space_id must be an integer or string representation of an integer", ) def validate_document_ids(document_ids: Any) -> list[int]: """ Validate and convert document_ids to list of integers. - + Args: document_ids: The document IDs to validate - + Returns: List[int]: Validated list of document IDs - + Raises: HTTPException: If validation fails """ if document_ids is None: return [] - + if not isinstance(document_ids, list): raise HTTPException( - status_code=400, - detail="document_ids_to_add_in_context must be a list" + status_code=400, detail="document_ids_to_add_in_context must be a list" ) - + validated_ids = [] for i, doc_id in enumerate(document_ids): if isinstance(doc_id, bool): @@ -111,119 +102,110 @@ def validate_document_ids(document_ids: Any) -> list[int]: if doc_id <= 0: raise HTTPException( status_code=400, - detail=f"document_ids_to_add_in_context[{i}] must be a positive integer" + detail=f"document_ids_to_add_in_context[{i}] must be a positive integer", ) validated_ids.append(doc_id) elif isinstance(doc_id, str): if not doc_id.strip(): raise HTTPException( status_code=400, - detail=f"document_ids_to_add_in_context[{i}] cannot be empty" + detail=f"document_ids_to_add_in_context[{i}] cannot be empty", ) - - if not re.match(r'^[1-9]\d*$', doc_id.strip()): + + if not re.match(r"^[1-9]\d*$", doc_id.strip()): raise HTTPException( status_code=400, - detail=f"document_ids_to_add_in_context[{i}] must be a valid positive integer" + detail=f"document_ids_to_add_in_context[{i}] must be a valid positive integer", ) - + value = int(doc_id.strip()) # Regex already guarantees value > 0 if value <= 0: raise HTTPException( status_code=400, - detail=f"document_ids_to_add_in_context[{i}] must be a positive integer" + detail=f"document_ids_to_add_in_context[{i}] must be a positive integer", ) validated_ids.append(value) else: raise HTTPException( status_code=400, - detail=f"document_ids_to_add_in_context[{i}] must be an integer or string representation of an integer" + detail=f"document_ids_to_add_in_context[{i}] must be an integer or string representation of an integer", ) - + return validated_ids def validate_connectors(connectors: Any) -> list[str]: """ Validate selected_connectors list. - + Args: connectors: The connectors to validate - + Returns: List[str]: Validated list of connector names - + Raises: HTTPException: If validation fails """ if connectors is None: return [] - + if not isinstance(connectors, list): raise HTTPException( - status_code=400, - detail="selected_connectors must be a list" + status_code=400, detail="selected_connectors must be a list" ) - + validated_connectors = [] for i, connector in enumerate(connectors): if not isinstance(connector, str): raise HTTPException( - status_code=400, - detail=f"selected_connectors[{i}] must be a string" + status_code=400, detail=f"selected_connectors[{i}] must be a string" ) - + if not connector.strip(): raise HTTPException( - status_code=400, - detail=f"selected_connectors[{i}] cannot be empty" + status_code=400, detail=f"selected_connectors[{i}] cannot be empty" ) - + trimmed = connector.strip() - if not re.fullmatch(r'[\w\-_]+', trimmed): + if not re.fullmatch(r"[\w\-_]+", trimmed): raise HTTPException( status_code=400, - detail=f"selected_connectors[{i}] contains invalid characters" + detail=f"selected_connectors[{i}] contains invalid characters", ) validated_connectors.append(trimmed) - + return validated_connectors def validate_research_mode(research_mode: Any) -> str: """ Validate research_mode parameter. - + Args: research_mode: The research mode to validate - + Returns: str: Validated research mode - + Raises: HTTPException: If validation fails """ if research_mode is None: return "QNA" # Default value - + if not isinstance(research_mode, str): - raise HTTPException( - status_code=400, - detail="research_mode must be a string" - ) + raise HTTPException(status_code=400, detail="research_mode must be a string") normalized_mode = research_mode.strip().upper() if not normalized_mode: - raise HTTPException( - status_code=400, - detail="research_mode cannot be empty" - ) + raise HTTPException(status_code=400, detail="research_mode cannot be empty") valid_modes = ["REPORT_GENERAL", "REPORT_DEEP", "REPORT_DEEPER", "QNA"] if normalized_mode not in valid_modes: raise HTTPException( status_code=400, - detail=f"research_mode must be one of: {', '.join(valid_modes)}" + detail=f"research_mode must be one of: {', '.join(valid_modes)}", ) return normalized_mode @@ -231,36 +213,30 @@ def validate_research_mode(research_mode: Any) -> str: def validate_search_mode(search_mode: Any) -> str: """ Validate search_mode parameter. - + Args: search_mode: The search mode to validate - + Returns: str: Validated search mode - + Raises: HTTPException: If validation fails """ if search_mode is None: return "CHUNKS" # Default value - + if not isinstance(search_mode, str): - raise HTTPException( - status_code=400, - detail="search_mode must be a string" - ) + raise HTTPException(status_code=400, detail="search_mode must be a string") normalized_mode = search_mode.strip().upper() if not normalized_mode: - raise HTTPException( - status_code=400, - detail="search_mode cannot be empty" - ) + raise HTTPException(status_code=400, detail="search_mode cannot be empty") valid_modes = ["CHUNKS", "DOCUMENTS"] if normalized_mode not in valid_modes: raise HTTPException( status_code=400, - detail=f"search_mode must be one of: {', '.join(valid_modes)}" + detail=f"search_mode must be one of: {', '.join(valid_modes)}", ) return normalized_mode @@ -268,185 +244,155 @@ def validate_search_mode(search_mode: Any) -> str: def validate_messages(messages: Any) -> list[dict]: """ Validate messages structure. - + Args: messages: The messages to validate - + Returns: List[dict]: Validated messages - + Raises: HTTPException: If validation fails """ if not isinstance(messages, list): - raise HTTPException( - status_code=400, - detail="messages must be a list" - ) - + raise HTTPException(status_code=400, detail="messages must be a list") + if not messages: - raise HTTPException( - status_code=400, - detail="messages cannot be empty" - ) - + raise HTTPException(status_code=400, detail="messages cannot be empty") + validated_messages = [] for i, message in enumerate(messages): if not isinstance(message, dict): raise HTTPException( - status_code=400, - detail=f"messages[{i}] must be a dictionary" + status_code=400, detail=f"messages[{i}] must be a dictionary" ) - + if "role" not in message: raise HTTPException( - status_code=400, - detail=f"messages[{i}] must have a 'role' field" + status_code=400, detail=f"messages[{i}] must have a 'role' field" ) - + if "content" not in message: raise HTTPException( - status_code=400, - detail=f"messages[{i}] must have a 'content' field" + status_code=400, detail=f"messages[{i}] must have a 'content' field" ) - + role = message["role"] if not isinstance(role, str) or role not in ["user", "assistant", "system"]: raise HTTPException( status_code=400, - detail=f"messages[{i}].role must be 'user', 'assistant', or 'system'" + detail=f"messages[{i}].role must be 'user', 'assistant', or 'system'", ) - + content = message["content"] if not isinstance(content, str): raise HTTPException( - status_code=400, - detail=f"messages[{i}].content must be a string" + status_code=400, detail=f"messages[{i}].content must be a string" ) - + if not content.strip(): raise HTTPException( - status_code=400, - detail=f"messages[{i}].content cannot be empty" + status_code=400, detail=f"messages[{i}].content cannot be empty" ) - + # Trim content and enforce max length (10,000 chars) sanitized_content = content.strip() if len(sanitized_content) > 10000: # Reasonable limit raise HTTPException( status_code=400, - detail=f"messages[{i}].content is too long (max 10000 characters)" + detail=f"messages[{i}].content is too long (max 10000 characters)", ) - - validated_messages.append({ - "role": role, - "content": sanitized_content - }) - + + validated_messages.append({"role": role, "content": sanitized_content}) + return validated_messages def validate_email(email: str) -> str: """ Validate email address using pyvalidators library. - + Args: email: The email address to validate - + Returns: str: Validated email address - + Raises: HTTPException: If validation fails """ if not email or not email.strip(): - raise HTTPException( - status_code=400, - detail="Email address is required" - ) - + raise HTTPException(status_code=400, detail="Email address is required") + email = email.strip() - + if not validators.email(email): - raise HTTPException( - status_code=400, - detail="Invalid email address format" - ) - + raise HTTPException(status_code=400, detail="Invalid email address format") + return email def validate_url(url: str) -> str: """ Validate URL using pyvalidators library. - + Args: url: The URL to validate - + Returns: str: Validated URL - + Raises: HTTPException: If validation fails """ if not url or not url.strip(): - raise HTTPException( - status_code=400, - detail="URL is required" - ) - + raise HTTPException(status_code=400, detail="URL is required") + url = url.strip() - + if not validators.url(url): - raise HTTPException( - status_code=400, - detail="Invalid URL format" - ) - + raise HTTPException(status_code=400, detail="Invalid URL format") + return url def validate_uuid(uuid_string: str) -> str: """ Validate UUID using pyvalidators library. - + Args: uuid_string: The UUID string to validate - + Returns: str: Validated UUID string - + Raises: HTTPException: If validation fails """ if not uuid_string or not uuid_string.strip(): - raise HTTPException( - status_code=400, - detail="UUID is required" - ) - + raise HTTPException(status_code=400, detail="UUID is required") + uuid_string = uuid_string.strip() - + if not validators.uuid(uuid_string): - raise HTTPException( - status_code=400, - detail="Invalid UUID format" - ) - + raise HTTPException(status_code=400, detail="Invalid UUID format") + return uuid_string -def validate_connector_config(connector_type: str | Any, config: dict[str, Any]) -> dict[str, Any]: +def validate_connector_config( + connector_type: str | Any, config: dict[str, Any] +) -> dict[str, Any]: """ Validate connector configuration based on connector type. - + Args: connector_type: The type of connector (string or enum) config: The configuration dictionary to validate - + Returns: dict: Validated configuration - + Raises: ValueError: If validation fails """ @@ -454,76 +400,69 @@ def validate_connector_config(connector_type: str | Any, config: dict[str, Any]) raise ValueError("config must be a dictionary of connector settings") # Convert enum to string if needed - connector_type_str = str(connector_type).split('.')[-1] if hasattr(connector_type, 'value') else str(connector_type) - + connector_type_str = ( + str(connector_type).split(".")[-1] + if hasattr(connector_type, "value") + else str(connector_type) + ) + # Validation function helpers def validate_email_field(key: str, connector_name: str) -> None: if not validators.email(config.get(key, "")): raise ValueError(f"Invalid email format for {connector_name} connector") - + def validate_url_field(key: str, connector_name: str) -> None: if not validators.url(config.get(key, "")): raise ValueError(f"Invalid base URL format for {connector_name} connector") - + def validate_list_field(key: str, field_name: str) -> None: value = config.get(key) if not isinstance(value, list) or not value: raise ValueError(f"{field_name} must be a non-empty list of strings") - + # Lookup table for connector validation rules connector_rules = { - "SERPER_API": { - "required": ["SERPER_API_KEY"], - "validators": {} - }, - "TAVILY_API": { - "required": ["TAVILY_API_KEY"], - "validators": {} - }, - "LINKUP_API": { - "required": ["LINKUP_API_KEY"], - "validators": {} - }, - "SLACK_CONNECTOR": { - "required": ["SLACK_BOT_TOKEN"], - "validators": {} - }, + "SERPER_API": {"required": ["SERPER_API_KEY"], "validators": {}}, + "TAVILY_API": {"required": ["TAVILY_API_KEY"], "validators": {}}, + "LINKUP_API": {"required": ["LINKUP_API_KEY"], "validators": {}}, + "SLACK_CONNECTOR": {"required": ["SLACK_BOT_TOKEN"], "validators": {}}, "NOTION_CONNECTOR": { "required": ["NOTION_INTEGRATION_TOKEN"], - "validators": {} + "validators": {}, }, "GITHUB_CONNECTOR": { "required": ["GITHUB_PAT", "repo_full_names"], "validators": { - "repo_full_names": lambda: validate_list_field("repo_full_names", "repo_full_names") - } - }, - "LINEAR_CONNECTOR": { - "required": ["LINEAR_API_KEY"], - "validators": {} - }, - "DISCORD_CONNECTOR": { - "required": ["DISCORD_BOT_TOKEN"], - "validators": {} + "repo_full_names": lambda: validate_list_field( + "repo_full_names", "repo_full_names" + ) + }, }, + "LINEAR_CONNECTOR": {"required": ["LINEAR_API_KEY"], "validators": {}}, + "DISCORD_CONNECTOR": {"required": ["DISCORD_BOT_TOKEN"], "validators": {}}, "JIRA_CONNECTOR": { "required": ["JIRA_EMAIL", "JIRA_API_TOKEN", "JIRA_BASE_URL"], "validators": { "JIRA_EMAIL": lambda: validate_email_field("JIRA_EMAIL", "JIRA"), - "JIRA_BASE_URL": lambda: validate_url_field("JIRA_BASE_URL", "JIRA") - } + "JIRA_BASE_URL": lambda: validate_url_field("JIRA_BASE_URL", "JIRA"), + }, }, "CONFLUENCE_CONNECTOR": { - "required": ["CONFLUENCE_BASE_URL", "CONFLUENCE_EMAIL", "CONFLUENCE_API_TOKEN"], + "required": [ + "CONFLUENCE_BASE_URL", + "CONFLUENCE_EMAIL", + "CONFLUENCE_API_TOKEN", + ], "validators": { - "CONFLUENCE_EMAIL": lambda: validate_email_field("CONFLUENCE_EMAIL", "Confluence"), - "CONFLUENCE_BASE_URL": lambda: validate_url_field("CONFLUENCE_BASE_URL", "Confluence") - } - }, - "CLICKUP_CONNECTOR": { - "required": ["CLICKUP_API_TOKEN"], - "validators": {} + "CONFLUENCE_EMAIL": lambda: validate_email_field( + "CONFLUENCE_EMAIL", "Confluence" + ), + "CONFLUENCE_BASE_URL": lambda: validate_url_field( + "CONFLUENCE_BASE_URL", "Confluence" + ), + }, }, + "CLICKUP_CONNECTOR": {"required": ["CLICKUP_API_TOKEN"], "validators": {}}, # "GOOGLE_CALENDAR_CONNECTOR": { # "required": ["token", "refresh_token", "token_uri", "client_id", "expiry", "scopes", "client_secret"], # "validators": {}, @@ -538,26 +477,23 @@ def validate_connector_config(connector_type: str | Any, config: dict[str, Any]) # "required": ["AIRTABLE_API_KEY", "AIRTABLE_BASE_ID"], # "validators": {} # }, - "LUMA_CONNECTOR": { - "required": ["LUMA_API_KEY"], - "validators": {} - } + "LUMA_CONNECTOR": {"required": ["LUMA_API_KEY"], "validators": {}}, } - + rules = connector_rules.get(connector_type_str) if not rules: return config # Unknown connector type, pass through - + # Validate required keys match exactly if set(config.keys()) != set(rules["required"]): raise ValueError( f"For {connector_type_str} connector type, config must only contain these keys: {rules['required']}" ) - + # Apply custom validators first (these check format before emptiness) for validator_func in rules["validators"].values(): validator_func() - + # Validate each field is not empty for key in rules["required"]: # Special handling for Google connectors that don't allow None or empty strings @@ -568,5 +504,5 @@ def validate_connector_config(connector_type: str | Any, config: dict[str, Any]) # Standard check: field must have a truthy value if not config.get(key): raise ValueError(f"{key} cannot be empty") - + return config diff --git a/surfsense_web/app/dashboard/[search_space_id]/client-layout.tsx b/surfsense_web/app/dashboard/[search_space_id]/client-layout.tsx index 6610cb046..f3c1531a8 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/client-layout.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/client-layout.tsx @@ -1,12 +1,16 @@ "use client"; +import { Loader2 } from "lucide-react"; +import { usePathname, useRouter } from "next/navigation"; import type React from "react"; -import { useState } from "react"; +import { useEffect, useState } from "react"; import { DashboardBreadcrumb } from "@/components/dashboard-breadcrumb"; import { AppSidebarProvider } from "@/components/sidebar/AppSidebarProvider"; import { ThemeTogglerComponent } from "@/components/theme/theme-toggle"; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; import { Separator } from "@/components/ui/separator"; import { SidebarInset, SidebarProvider, SidebarTrigger } from "@/components/ui/sidebar"; +import { useLLMPreferences } from "@/hooks/use-llm-configs"; export function DashboardClientLayout({ children, @@ -19,6 +23,16 @@ export function DashboardClientLayout({ navSecondary: any[]; navMain: any[]; }) { + const router = useRouter(); + const pathname = usePathname(); + const searchSpaceIdNum = Number(searchSpaceId); + + const { loading, error, isOnboardingComplete } = useLLMPreferences(searchSpaceIdNum); + const [hasCheckedOnboarding, setHasCheckedOnboarding] = useState(false); + + // Skip onboarding check if we're already on the onboarding page + const isOnboardingPage = pathname?.includes("/onboard"); + const [open, setOpen] = useState(() => { try { const match = document.cookie.match(/(?:^|; )sidebar_state=([^;]+)/); @@ -29,6 +43,68 @@ export function DashboardClientLayout({ return true; }); + useEffect(() => { + // Skip check if already on onboarding page + if (isOnboardingPage) { + setHasCheckedOnboarding(true); + return; + } + + // Only check once after preferences have loaded + if (!loading && !hasCheckedOnboarding) { + const onboardingComplete = isOnboardingComplete(); + + if (!onboardingComplete) { + router.push(`/dashboard/${searchSpaceId}/onboard`); + } + + setHasCheckedOnboarding(true); + } + }, [ + loading, + isOnboardingComplete, + isOnboardingPage, + router, + searchSpaceId, + hasCheckedOnboarding, + ]); + + // Show loading screen while checking onboarding status (only on first load) + if (!hasCheckedOnboarding && loading && !isOnboardingPage) { + return ( +
+ + + Loading Configuration + Checking your LLM preferences... + + + + + +
+ ); + } + + // Show error screen if there's an error loading preferences (but not on onboarding page) + if (error && !hasCheckedOnboarding && !isOnboardingPage) { + return ( +
+ + + + Configuration Error + + Failed to load your LLM configuration + + +

{error}

+ + +
+ ); + } + return ( {/* Use AppSidebarProvider which fetches user, search space, and recent chats */} diff --git a/surfsense_web/app/dashboard/[search_space_id]/layout.tsx b/surfsense_web/app/dashboard/[search_space_id]/layout.tsx index d0e04fe68..b012484aa 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/layout.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/layout.tsx @@ -33,6 +33,12 @@ export default function DashboardLayout({ icon: "SquareTerminal", items: [], }, + { + title: "Manage LLMs", + url: `/dashboard/${search_space_id}/settings`, + icon: "Settings2", + items: [], + }, { title: "Documents", diff --git a/surfsense_web/app/onboard/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/onboard/page.tsx similarity index 87% rename from surfsense_web/app/onboard/page.tsx rename to surfsense_web/app/dashboard/[search_space_id]/onboard/page.tsx index 387bf736a..0ff0fb205 100644 --- a/surfsense_web/app/onboard/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/onboard/page.tsx @@ -2,7 +2,7 @@ import { ArrowLeft, ArrowRight, Bot, CheckCircle, Sparkles } from "lucide-react"; import { AnimatePresence, motion } from "motion/react"; -import { useRouter } from "next/navigation"; +import { useParams, useRouter } from "next/navigation"; import { useEffect, useState } from "react"; import { Logo } from "@/components/Logo"; import { AddProviderStep } from "@/components/onboard/add-provider-step"; @@ -17,13 +17,16 @@ const TOTAL_STEPS = 3; const OnboardPage = () => { const router = useRouter(); - const { llmConfigs, loading: configsLoading, refreshConfigs } = useLLMConfigs(); + const params = useParams(); + const searchSpaceId = Number(params.search_space_id); + + const { llmConfigs, loading: configsLoading, refreshConfigs } = useLLMConfigs(searchSpaceId); const { preferences, loading: preferencesLoading, isOnboardingComplete, refreshPreferences, - } = useLLMPreferences(); + } = useLLMPreferences(searchSpaceId); const [currentStep, setCurrentStep] = useState(1); const [hasUserProgressed, setHasUserProgressed] = useState(false); @@ -44,11 +47,23 @@ const OnboardPage = () => { }, [currentStep]); // Redirect to dashboard if onboarding is already complete and user hasn't progressed (fresh page load) + // But only check once to avoid redirect loops useEffect(() => { - if (!preferencesLoading && isOnboardingComplete() && !hasUserProgressed) { - router.push("/dashboard"); + if (!preferencesLoading && !configsLoading && isOnboardingComplete() && !hasUserProgressed) { + // Small delay to ensure the check is stable + const timer = setTimeout(() => { + router.push(`/dashboard/${searchSpaceId}`); + }, 100); + return () => clearTimeout(timer); } - }, [preferencesLoading, isOnboardingComplete, hasUserProgressed, router]); + }, [ + preferencesLoading, + configsLoading, + isOnboardingComplete, + hasUserProgressed, + router, + searchSpaceId, + ]); const progress = (currentStep / TOTAL_STEPS) * 100; @@ -80,7 +95,7 @@ const OnboardPage = () => { }; const handleComplete = () => { - router.push("/dashboard"); + router.push(`/dashboard/${searchSpaceId}/documents`); }; if (configsLoading || preferencesLoading) { @@ -184,12 +199,18 @@ const OnboardPage = () => { > {currentStep === 1 && ( )} - {currentStep === 2 && } - {currentStep === 3 && } + {currentStep === 2 && ( + + )} + {currentStep === 3 && } diff --git a/surfsense_web/app/settings/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/settings/page.tsx similarity index 82% rename from surfsense_web/app/settings/page.tsx rename to surfsense_web/app/dashboard/[search_space_id]/settings/page.tsx index 18278be94..9eba74617 100644 --- a/surfsense_web/app/settings/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/settings/page.tsx @@ -1,14 +1,16 @@ "use client"; -import { ArrowLeft, Bot, Brain, Settings } from "lucide-react"; // Import ArrowLeft icon -import { useRouter } from "next/navigation"; // Add this import +import { ArrowLeft, Bot, Brain, Settings } from "lucide-react"; +import { useParams, useRouter } from "next/navigation"; import { LLMRoleManager } from "@/components/settings/llm-role-manager"; import { ModelConfigManager } from "@/components/settings/model-config-manager"; import { Separator } from "@/components/ui/separator"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; export default function SettingsPage() { - const router = useRouter(); // Initialize router + const router = useRouter(); + const params = useParams(); + const searchSpaceId = Number(params.search_space_id); return (
@@ -19,7 +21,7 @@ export default function SettingsPage() {
{/* Back Button */}
@@ -57,11 +59,11 @@ export default function SettingsPage() {
- + - +
diff --git a/surfsense_web/app/dashboard/layout.tsx b/surfsense_web/app/dashboard/layout.tsx index 41a269d41..55482d299 100644 --- a/surfsense_web/app/dashboard/layout.tsx +++ b/surfsense_web/app/dashboard/layout.tsx @@ -4,7 +4,6 @@ import { Loader2 } from "lucide-react"; import { useRouter } from "next/navigation"; import { useEffect, useState } from "react"; import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; -import { useLLMPreferences } from "@/hooks/use-llm-configs"; interface DashboardLayoutProps { children: React.ReactNode; @@ -12,7 +11,6 @@ interface DashboardLayoutProps { export default function DashboardLayout({ children }: DashboardLayoutProps) { const router = useRouter(); - const { loading, error, isOnboardingComplete } = useLLMPreferences(); const [isCheckingAuth, setIsCheckingAuth] = useState(true); useEffect(() => { @@ -25,23 +23,14 @@ export default function DashboardLayout({ children }: DashboardLayoutProps) { setIsCheckingAuth(false); }, [router]); - useEffect(() => { - // Wait for preferences to load, then check if onboarding is complete - if (!loading && !error && !isCheckingAuth) { - if (!isOnboardingComplete()) { - router.push("/onboard"); - } - } - }, [loading, error, isCheckingAuth, isOnboardingComplete, router]); - - // Show loading screen while checking authentication or loading preferences - if (isCheckingAuth || loading) { + // Show loading screen while checking authentication + if (isCheckingAuth) { return (
Loading Dashboard - Checking your configuration... + Checking authentication... @@ -51,42 +40,5 @@ export default function DashboardLayout({ children }: DashboardLayoutProps) { ); } - // Show error screen if there's an error loading preferences - if (error) { - return ( -
- - - - Configuration Error - - Failed to load your LLM configuration - - -

{error}

- - -
- ); - } - - // Only render children if onboarding is complete - if (isOnboardingComplete()) { - return <>{children}; - } - - // This should not be reached due to redirect, but just in case - return ( -
- - - Redirecting... - Taking you to complete your setup - - - - - -
- ); + return <>{children}; } diff --git a/surfsense_web/components/UserDropdown.tsx b/surfsense_web/components/UserDropdown.tsx index 5d15b152b..230bf0554 100644 --- a/surfsense_web/components/UserDropdown.tsx +++ b/surfsense_web/components/UserDropdown.tsx @@ -66,10 +66,6 @@ export function UserDropdown({ - router.push(`/settings`)}> - - Settings - Log out diff --git a/surfsense_web/components/chat/ChatInputGroup.tsx b/surfsense_web/components/chat/ChatInputGroup.tsx index f57691e4d..c3877c108 100644 --- a/surfsense_web/components/chat/ChatInputGroup.tsx +++ b/surfsense_web/components/chat/ChatInputGroup.tsx @@ -332,8 +332,11 @@ const ResearchModeSelector = React.memo( ResearchModeSelector.displayName = "ResearchModeSelector"; const LLMSelector = React.memo(() => { - const { llmConfigs, loading: llmLoading, error } = useLLMConfigs(); - const { preferences, updatePreferences, loading: preferencesLoading } = useLLMPreferences(); + const { search_space_id } = useParams(); + const searchSpaceId = Number(search_space_id); + + const { llmConfigs, loading: llmLoading, error } = useLLMConfigs(searchSpaceId); + const { preferences, updatePreferences, loading: preferencesLoading } = useLLMPreferences(searchSpaceId); const isLoading = llmLoading || preferencesLoading; diff --git a/surfsense_web/components/onboard/add-provider-step.tsx b/surfsense_web/components/onboard/add-provider-step.tsx index f582000a3..9b70c8d7f 100644 --- a/surfsense_web/components/onboard/add-provider-step.tsx +++ b/surfsense_web/components/onboard/add-provider-step.tsx @@ -23,12 +23,17 @@ import { type CreateLLMConfig, useLLMConfigs } from "@/hooks/use-llm-configs"; import InferenceParamsEditor from "../inference-params-editor"; interface AddProviderStepProps { + searchSpaceId: number; onConfigCreated?: () => void; onConfigDeleted?: () => void; } -export function AddProviderStep({ onConfigCreated, onConfigDeleted }: AddProviderStepProps) { - const { llmConfigs, createLLMConfig, deleteLLMConfig } = useLLMConfigs(); +export function AddProviderStep({ + searchSpaceId, + onConfigCreated, + onConfigDeleted, +}: AddProviderStepProps) { + const { llmConfigs, createLLMConfig, deleteLLMConfig } = useLLMConfigs(searchSpaceId); const [isAddingNew, setIsAddingNew] = useState(false); const [formData, setFormData] = useState({ name: "", @@ -38,6 +43,7 @@ export function AddProviderStep({ onConfigCreated, onConfigDeleted }: AddProvide api_key: "", api_base: "", litellm_params: {}, + search_space_id: searchSpaceId, }); const [isSubmitting, setIsSubmitting] = useState(false); @@ -65,6 +71,7 @@ export function AddProviderStep({ onConfigCreated, onConfigDeleted }: AddProvide api_key: "", api_base: "", litellm_params: {}, + search_space_id: searchSpaceId, }); setIsAddingNew(false); // Notify parent component that a config was created @@ -253,7 +260,6 @@ export function AddProviderStep({ onConfigCreated, onConfigDeleted }: AddProvide />
-
@@ -578,6 +580,7 @@ export function ModelConfigManager() { api_key: "", api_base: "", litellm_params: {}, + search_space_id: searchSpaceId, }); }} disabled={isSubmitting} diff --git a/surfsense_web/hooks/use-llm-configs.ts b/surfsense_web/hooks/use-llm-configs.ts index ccb3a5bc9..adf49d634 100644 --- a/surfsense_web/hooks/use-llm-configs.ts +++ b/surfsense_web/hooks/use-llm-configs.ts @@ -12,7 +12,7 @@ export interface LLMConfig { api_base?: string; litellm_params?: Record; created_at: string; - user_id: string; + search_space_id: number; } export interface LLMPreferences { @@ -32,6 +32,7 @@ export interface CreateLLMConfig { api_key: string; api_base?: string; litellm_params?: Record; + search_space_id: number; } export interface UpdateLLMConfig { @@ -44,16 +45,21 @@ export interface UpdateLLMConfig { litellm_params?: Record; } -export function useLLMConfigs() { +export function useLLMConfigs(searchSpaceId: number | null) { const [llmConfigs, setLlmConfigs] = useState([]); const [loading, setLoading] = useState(true); const [error, setError] = useState(null); const fetchLLMConfigs = async () => { + if (!searchSpaceId) { + setLoading(false); + return; + } + try { setLoading(true); const response = await fetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/llm-configs/`, + `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/llm-configs/?search_space_id=${searchSpaceId}`, { headers: { Authorization: `Bearer ${localStorage.getItem("surfsense_bearer_token")}`, @@ -79,7 +85,7 @@ export function useLLMConfigs() { useEffect(() => { fetchLLMConfigs(); - }, []); + }, [searchSpaceId]); const createLLMConfig = async (config: CreateLLMConfig): Promise => { try { @@ -181,16 +187,21 @@ export function useLLMConfigs() { }; } -export function useLLMPreferences() { +export function useLLMPreferences(searchSpaceId: number | null) { const [preferences, setPreferences] = useState({}); const [loading, setLoading] = useState(true); const [error, setError] = useState(null); const fetchPreferences = async () => { + if (!searchSpaceId) { + setLoading(false); + return; + } + try { setLoading(true); const response = await fetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/users/me/llm-preferences`, + `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-spaces/${searchSpaceId}/llm-preferences`, { headers: { Authorization: `Bearer ${localStorage.getItem("surfsense_bearer_token")}`, @@ -216,12 +227,17 @@ export function useLLMPreferences() { useEffect(() => { fetchPreferences(); - }, []); + }, [searchSpaceId]); const updatePreferences = async (newPreferences: Partial): Promise => { + if (!searchSpaceId) { + toast.error("Search space ID is required"); + return false; + } + try { const response = await fetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/users/me/llm-preferences`, + `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-spaces/${searchSpaceId}/llm-preferences`, { method: "PUT", headers: { From bee6b7c2cc98b4b44a0173ed45fd80876d534bf6 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Fri, 10 Oct 2025 01:24:27 -0700 Subject: [PATCH 12/35] fix: error handling and validation in LumaConnectorPage --- .../connectors/add/luma-connector/page.tsx | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/luma-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/luma-connector/page.tsx index f35c09770..ca9b716d5 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/luma-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/luma-connector/page.tsx @@ -70,15 +70,21 @@ export default function LumaConnectorPage() { }); useEffect(() => { - fetchConnectors(parseInt(searchSpaceId)).then((data) => { - const connector = data.find( - (c: SearchSourceConnector) => c.connector_type === EnumConnectorName.LUMA_CONNECTOR - ); - if (connector) { - setDoesConnectorExist(true); - } - }); - }, [fetchConnectors]); + fetchConnectors(parseInt(searchSpaceId)) + .then((data) => { + if (data && Array.isArray(data)) { + const connector = data.find( + (c: SearchSourceConnector) => c.connector_type === EnumConnectorName.LUMA_CONNECTOR + ); + if (connector) { + setDoesConnectorExist(true); + } + } + }) + .catch((error) => { + console.error("Error fetching connectors:", error); + }); + }, [fetchConnectors, searchSpaceId]); // Handle form submission const onSubmit = async (values: LumaConnectorFormValues) => { From 045537aa796970fffdefef54d2b3498dc28c149d Mon Sep 17 00:00:00 2001 From: Tarun Date: Sun, 12 Oct 2025 13:13:42 +0530 Subject: [PATCH 13/35] feat: add language support across configurations and prompts --- .../app/agents/researcher/configuration.py | 2 + .../app/agents/researcher/nodes.py | 4 +- .../app/agents/researcher/prompts.py | 7 +- .../researcher/qna_agent/configuration.py | 1 + .../app/agents/researcher/qna_agent/nodes.py | 8 +-- .../agents/researcher/qna_agent/prompts.py | 18 +++-- .../researcher/sub_section_writer/prompts.py | 18 +++-- surfsense_backend/app/db.py | 2 + surfsense_backend/app/routes/chats_routes.py | 46 ++++++++++++- .../app/routes/llm_config_routes.py | 15 +++- surfsense_backend/app/schemas/llm_config.py | 6 ++ .../tasks/stream_connector_search_results.py | 3 + .../components/onboard/add-provider-step.tsx | 27 +++++++- .../settings/model-config-manager.tsx | 32 +++++++++ surfsense_web/contracts/enums/languages.ts | 69 +++++++++++++++++++ surfsense_web/hooks/use-llm-configs.ts | 2 + 16 files changed, 242 insertions(+), 18 deletions(-) create mode 100644 surfsense_web/contracts/enums/languages.ts diff --git a/surfsense_backend/app/agents/researcher/configuration.py b/surfsense_backend/app/agents/researcher/configuration.py index 3e81a59c0..e8ee856de 100644 --- a/surfsense_backend/app/agents/researcher/configuration.py +++ b/surfsense_backend/app/agents/researcher/configuration.py @@ -37,6 +37,8 @@ class Configuration: search_mode: SearchMode research_mode: ResearchMode document_ids_to_add_in_context: list[int] + language: str | None = None + @classmethod def from_runnable_config( diff --git a/surfsense_backend/app/agents/researcher/nodes.py b/surfsense_backend/app/agents/researcher/nodes.py index 0835fb861..fe869c265 100644 --- a/surfsense_backend/app/agents/researcher/nodes.py +++ b/surfsense_backend/app/agents/researcher/nodes.py @@ -578,6 +578,7 @@ async def write_answer_outline( num_sections = configuration.num_sections user_id = configuration.user_id search_space_id = configuration.search_space_id + language = configuration.language # Get language from configuration writer( { @@ -628,7 +629,7 @@ async def write_answer_outline( # Create messages for the LLM messages = [ - SystemMessage(content=get_answer_outline_system_prompt()), + SystemMessage(content=get_answer_outline_system_prompt(language=language)), HumanMessage(content=human_message_content), ] @@ -2000,6 +2001,7 @@ async def handle_qna_workflow( "relevant_documents": all_documents, # Use combined documents "user_id": configuration.user_id, "search_space_id": configuration.search_space_id, + "language": configuration.language, } } diff --git a/surfsense_backend/app/agents/researcher/prompts.py b/surfsense_backend/app/agents/researcher/prompts.py index 44b218913..b7265602a 100644 --- a/surfsense_backend/app/agents/researcher/prompts.py +++ b/surfsense_backend/app/agents/researcher/prompts.py @@ -1,9 +1,14 @@ import datetime -def get_answer_outline_system_prompt(): +def get_answer_outline_system_prompt(language: str | None = None) -> str: + language_instruction = "" + if language: + language_instruction = f"\n\nIMPORTANT: Please respond in {language} language. All your responses, explanations, and analysis should be written in {language}." + return f""" Today's date: {datetime.datetime.now().strftime("%Y-%m-%d")} +{language_instruction} You are an expert research assistant specializing in structuring information. Your task is to create a detailed and logical research outline based on the user's query. This outline will serve as the blueprint for generating a comprehensive research report. diff --git a/surfsense_backend/app/agents/researcher/qna_agent/configuration.py b/surfsense_backend/app/agents/researcher/qna_agent/configuration.py index 5a4529e0d..ea107a575 100644 --- a/surfsense_backend/app/agents/researcher/qna_agent/configuration.py +++ b/surfsense_backend/app/agents/researcher/qna_agent/configuration.py @@ -20,6 +20,7 @@ class Configuration: ] # Documents provided directly to the agent for answering user_id: str # User identifier search_space_id: int # Search space identifier + language: str | None = None # Language for responses @classmethod def from_runnable_config( diff --git a/surfsense_backend/app/agents/researcher/qna_agent/nodes.py b/surfsense_backend/app/agents/researcher/qna_agent/nodes.py index fd6861efb..20374b706 100644 --- a/surfsense_backend/app/agents/researcher/qna_agent/nodes.py +++ b/surfsense_backend/app/agents/researcher/qna_agent/nodes.py @@ -102,7 +102,7 @@ async def answer_question(state: State, config: RunnableConfig) -> dict[str, Any user_query = configuration.user_query user_id = configuration.user_id search_space_id = configuration.search_space_id - + language = configuration.language # Get user's fast LLM llm = await get_user_fast_llm(state.db_session, user_id, search_space_id) if not llm: @@ -127,7 +127,7 @@ async def answer_question(state: State, config: RunnableConfig) -> dict[str, Any """ # Use initial system prompt for token calculation - initial_system_prompt = get_qna_citation_system_prompt(chat_history_str) + initial_system_prompt = get_qna_citation_system_prompt(chat_history_str, language) base_messages = [ SystemMessage(content=initial_system_prompt), HumanMessage(content=base_human_message_template), @@ -146,9 +146,9 @@ async def answer_question(state: State, config: RunnableConfig) -> dict[str, Any # Choose system prompt based on final document availability system_prompt = ( - get_qna_citation_system_prompt(chat_history_str) + get_qna_citation_system_prompt(chat_history_str, language) if has_documents - else get_qna_no_documents_system_prompt(chat_history_str) + else get_qna_no_documents_system_prompt(chat_history_str, language) ) # Generate documents section diff --git a/surfsense_backend/app/agents/researcher/qna_agent/prompts.py b/surfsense_backend/app/agents/researcher/qna_agent/prompts.py index 212788804..deb5dd59f 100644 --- a/surfsense_backend/app/agents/researcher/qna_agent/prompts.py +++ b/surfsense_backend/app/agents/researcher/qna_agent/prompts.py @@ -1,7 +1,7 @@ import datetime -def get_qna_citation_system_prompt(chat_history: str | None = None): +def get_qna_citation_system_prompt(chat_history: str | None = None, language: str | None = None): chat_history_section = ( f""" @@ -15,10 +15,15 @@ NO CHAT HISTORY PROVIDED """ ) + + # Add language instruction if specified + language_instruction = "" + if language: + language_instruction = f"\n\nIMPORTANT: Please respond in {language} language. All your responses, explanations, and analysis should be written in {language}." return f""" Today's date: {datetime.datetime.now().strftime("%Y-%m-%d")} -You are SurfSense, an advanced AI research assistant that provides detailed, well-researched answers to user questions by synthesizing information from multiple personal knowledge sources. +You are SurfSense, an advanced AI research assistant that provides detailed, well-researched answers to user questions by synthesizing information from multiple personal knowledge sources.{language_instruction} {chat_history_section} - EXTENSION: "Web content saved via SurfSense browser extension" (personal browsing history) @@ -149,7 +154,7 @@ Make sure your response: """ -def get_qna_no_documents_system_prompt(chat_history: str | None = None): +def get_qna_no_documents_system_prompt(chat_history: str | None = None, language: str | None = None): chat_history_section = ( f""" @@ -163,10 +168,15 @@ NO CHAT HISTORY PROVIDED """ ) + + # Add language instruction if specified + language_instruction = "" + if language: + language_instruction = f"\n\nIMPORTANT: Please respond in {language} language. All your responses, explanations, and analysis should be written in {language}." return f""" Today's date: {datetime.datetime.now().strftime("%Y-%m-%d")} -You are SurfSense, an advanced AI research assistant that provides helpful, detailed answers to user questions in a conversational manner. +You are SurfSense, an advanced AI research assistant that provides helpful, detailed answers to user questions in a conversational manner.{language_instruction} {chat_history_section} The user has asked a question but there are no specific documents from their personal knowledge base available to answer it. You should provide a helpful response based on: diff --git a/surfsense_backend/app/agents/researcher/sub_section_writer/prompts.py b/surfsense_backend/app/agents/researcher/sub_section_writer/prompts.py index c3d487671..3954d47e5 100644 --- a/surfsense_backend/app/agents/researcher/sub_section_writer/prompts.py +++ b/surfsense_backend/app/agents/researcher/sub_section_writer/prompts.py @@ -1,7 +1,7 @@ import datetime -def get_citation_system_prompt(chat_history: str | None = None): +def get_citation_system_prompt(chat_history: str | None = None, language: str | None = None): chat_history_section = ( f""" @@ -15,10 +15,15 @@ NO CHAT HISTORY PROVIDED """ ) + + # Add language instruction if specified + language_instruction = "" + if language: + language_instruction = f"\n\nIMPORTANT: Please respond in {language} language. All your responses, explanations, and analysis should be written in {language}." return f""" Today's date: {datetime.datetime.now().strftime("%Y-%m-%d")} -You are SurfSense, an advanced AI research assistant that synthesizes information from multiple knowledge sources to provide comprehensive, well-cited answers to user queries. +You are SurfSense, an advanced AI research assistant that synthesizes information from multiple knowledge sources to provide comprehensive, well-cited answers to user queries.{language_instruction} {chat_history_section} - EXTENSION: "Web content saved via SurfSense browser extension" (personal browsing history) @@ -156,7 +161,7 @@ Make sure your response: """ -def get_no_documents_system_prompt(chat_history: str | None = None): +def get_no_documents_system_prompt(chat_history: str | None = None, language: str | None = None): chat_history_section = ( f""" @@ -170,10 +175,15 @@ NO CHAT HISTORY PROVIDED """ ) + + # Add language instruction if specified + language_instruction = "" + if language: + language_instruction = f"\n\nIMPORTANT: Please respond in {language} language. All your responses, explanations, and analysis should be written in {language}." return f""" Today's date: {datetime.datetime.now().strftime("%Y-%m-%d")} -You are SurfSense, an advanced AI research assistant that helps users create well-structured content for their documents and research. +You are SurfSense, an advanced AI research assistant that helps users create well-structured content for their documents and research.{language_instruction} {chat_history_section} You are writing content for a specific sub-section of a document. No specific documents from the user's personal knowledge base are available, so you should create content based on: diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py index eb33145cf..e476ed8e5 100644 --- a/surfsense_backend/app/db.py +++ b/surfsense_backend/app/db.py @@ -296,6 +296,8 @@ class LLMConfig(BaseModel, TimestampMixin): api_key = Column(String, nullable=False) api_base = Column(String(500), nullable=True) + language = Column(String(50), nullable=True, default="English") + # For any other parameters that litellm supports litellm_params = Column(JSON, nullable=True, default={}) diff --git a/surfsense_backend/app/routes/chats_routes.py b/surfsense_backend/app/routes/chats_routes.py index e4d02686f..d30bf5451 100644 --- a/surfsense_backend/app/routes/chats_routes.py +++ b/surfsense_backend/app/routes/chats_routes.py @@ -4,8 +4,10 @@ from langchain.schema import AIMessage, HumanMessage from sqlalchemy.exc import IntegrityError, OperationalError from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select +from sqlalchemy.orm import selectinload -from app.db import Chat, SearchSpace, User, get_async_session + +from app.db import Chat, SearchSpace, User, UserSearchSpacePreference, get_async_session from app.schemas import ( AISDKChatRequest, ChatCreate, @@ -53,21 +55,60 @@ async def handle_chat_data( request_data.get("document_ids_to_add_in_context") ) search_mode_str = validate_search_mode(request_data.get("search_mode")) + # print("RESQUEST DATA:", request_data) + # print("SELECTED CONNECTORS:", selected_connectors) # Check if the search space belongs to the current user try: await check_ownership(session, SearchSpace, search_space_id, user) + language_result = await session.execute( + select(UserSearchSpacePreference) + .options( + selectinload(UserSearchSpacePreference.search_space).selectinload(SearchSpace.llm_configs), + selectinload(UserSearchSpacePreference.long_context_llm), + selectinload(UserSearchSpacePreference.fast_llm), + selectinload(UserSearchSpacePreference.strategic_llm) + ) + .filter( + UserSearchSpacePreference.search_space_id == search_space_id, + UserSearchSpacePreference.user_id == user.id + ) + ) + user_preference = language_result.scalars().first() + print("UserSearchSpacePreference:", user_preference) + + language = None + if user_preference and user_preference.search_space and user_preference.search_space.llm_configs: + llm_configs = user_preference.search_space.llm_configs + # print(f"Found {len(llm_configs)} LLM Configs") + # for i, config in enumerate(llm_configs): + # print(f" Config {i+1}: name={config.name}, provider={config.provider}, language={getattr(config, 'language', None)}") + + + for preferred_llm in [user_preference.fast_llm, user_preference.long_context_llm, user_preference.strategic_llm]: + if preferred_llm and getattr(preferred_llm, 'language', None): + language = preferred_llm.language + # print(f"Using language from preferred LLM: {preferred_llm.name} -> {language}") + break + + # no preferred llM has language use first available LLM config + if not language: + first_llm_config = llm_configs[0] + language = getattr(first_llm_config, 'language', None) + # print(f"Using language from first LLM config: {first_llm_config.name} -> {language}") + except HTTPException: raise HTTPException( status_code=403, detail="You don't have access to this search space" ) from None - + # print("Language selected:", language) langchain_chat_history = [] for message in messages[:-1]: if message["role"] == "user": langchain_chat_history.append(HumanMessage(content=message["content"])) elif message["role"] == "assistant": langchain_chat_history.append(AIMessage(content=message["content"])) + response = StreamingResponse( stream_connector_search_results( @@ -80,6 +121,7 @@ async def handle_chat_data( langchain_chat_history, search_mode_str, document_ids_to_add_in_context, + language, ) ) diff --git a/surfsense_backend/app/routes/llm_config_routes.py b/surfsense_backend/app/routes/llm_config_routes.py index 63d540d2c..896f7be41 100644 --- a/surfsense_backend/app/routes/llm_config_routes.py +++ b/surfsense_backend/app/routes/llm_config_routes.py @@ -299,7 +299,10 @@ async def update_user_llm_preferences( # Validate that all provided LLM config IDs belong to the search space update_data = preferences.model_dump(exclude_unset=True) - + + # Store language from configs to validate consistency + languages = set() + for _key, llm_config_id in update_data.items(): if llm_config_id is not None: # Verify the LLM config belongs to the search space @@ -315,6 +318,16 @@ async def update_user_llm_preferences( status_code=404, detail=f"LLM configuration {llm_config_id} not found in this search space", ) + + # Collect language for consistency check + languages.add(llm_config.language) + + # Check if all selected LLM configs have the same language + if len(languages) > 1: + raise HTTPException( + status_code=400, + detail="All selected LLM configurations must have the same language setting", + ) # Update user preferences for key, value in update_data.items(): diff --git a/surfsense_backend/app/schemas/llm_config.py b/surfsense_backend/app/schemas/llm_config.py index 8beb65347..285c15665 100644 --- a/surfsense_backend/app/schemas/llm_config.py +++ b/surfsense_backend/app/schemas/llm_config.py @@ -26,6 +26,9 @@ class LLMConfigBase(BaseModel): litellm_params: dict[str, Any] | None = Field( default=None, description="Additional LiteLLM parameters" ) + language: str | None = Field( + default="English", max_length=50, description="Language for the LLM" + ) class LLMConfigCreate(LLMConfigBase): @@ -49,6 +52,9 @@ class LLMConfigUpdate(BaseModel): api_base: str | None = Field( None, max_length=500, description="Optional API base URL" ) + language: str | None = Field( + None, max_length=50, description="Language for the LLM" + ) litellm_params: dict[str, Any] | None = Field( None, description="Additional LiteLLM parameters" ) diff --git a/surfsense_backend/app/tasks/stream_connector_search_results.py b/surfsense_backend/app/tasks/stream_connector_search_results.py index ead6a89e7..dd1ae4ce5 100644 --- a/surfsense_backend/app/tasks/stream_connector_search_results.py +++ b/surfsense_backend/app/tasks/stream_connector_search_results.py @@ -20,6 +20,7 @@ async def stream_connector_search_results( langchain_chat_history: list[Any], search_mode_str: str, document_ids_to_add_in_context: list[int], + language: str | None = None, ) -> AsyncGenerator[str, None]: """ Stream connector search results to the client @@ -66,8 +67,10 @@ async def stream_connector_search_results( "search_mode": search_mode, "research_mode": research_mode, "document_ids_to_add_in_context": document_ids_to_add_in_context, + "language": language, # Add language to the configuration } } + # print(f"Researcher configuration: {config['configurable']}") # Debug print # Initialize state with database session and streaming service initial_state = State( db_session=session, diff --git a/surfsense_web/components/onboard/add-provider-step.tsx b/surfsense_web/components/onboard/add-provider-step.tsx index 9b70c8d7f..6517fe0a1 100644 --- a/surfsense_web/components/onboard/add-provider-step.tsx +++ b/surfsense_web/components/onboard/add-provider-step.tsx @@ -18,6 +18,7 @@ import { SelectValue, } from "@/components/ui/select"; import { LLM_PROVIDERS } from "@/contracts/enums/llm-providers"; +import { LANGUAGES } from "@/contracts/enums/languages"; import { type CreateLLMConfig, useLLMConfigs } from "@/hooks/use-llm-configs"; import InferenceParamsEditor from "../inference-params-editor"; @@ -42,6 +43,7 @@ export function AddProviderStep({ model_name: "", api_key: "", api_base: "", + language: "English", litellm_params: {}, search_space_id: searchSpaceId, }); @@ -70,6 +72,7 @@ export function AddProviderStep({ model_name: "", api_key: "", api_base: "", + language: "English", litellm_params: {}, search_space_id: searchSpaceId, }); @@ -119,6 +122,7 @@ export function AddProviderStep({

Model: {config.model_name} + {config.language && ` • Language: ${config.language}`} {config.api_base && ` • Base: ${config.api_base}`}

@@ -169,7 +173,7 @@ export function AddProviderStep({
-
+
+ + {/* language */} +
+ + +
+
{formData.provider === "CUSTOM" && ( diff --git a/surfsense_web/components/settings/model-config-manager.tsx b/surfsense_web/components/settings/model-config-manager.tsx index 7384337b6..a236f1965 100644 --- a/surfsense_web/components/settings/model-config-manager.tsx +++ b/surfsense_web/components/settings/model-config-manager.tsx @@ -38,6 +38,7 @@ import { SelectValue, } from "@/components/ui/select"; import { LLM_PROVIDERS } from "@/contracts/enums/llm-providers"; +import { LANGUAGES } from "@/contracts/enums/languages"; import { type CreateLLMConfig, type LLMConfig, useLLMConfigs } from "@/hooks/use-llm-configs"; import InferenceParamsEditor from "../inference-params-editor"; @@ -65,6 +66,7 @@ export function ModelConfigManager({ searchSpaceId }: ModelConfigManagerProps) { model_name: "", api_key: "", api_base: "", + language: "English", litellm_params: {}, search_space_id: searchSpaceId, }); @@ -80,6 +82,7 @@ export function ModelConfigManager({ searchSpaceId }: ModelConfigManagerProps) { model_name: editingConfig.model_name, api_key: editingConfig.api_key, api_base: editingConfig.api_base || "", + language: editingConfig.language || "Enlgish", litellm_params: editingConfig.litellm_params || {}, search_space_id: searchSpaceId, }); @@ -118,6 +121,7 @@ export function ModelConfigManager({ searchSpaceId }: ModelConfigManagerProps) { model_name: "", api_key: "", api_base: "", + language: "English", litellm_params: {}, search_space_id: searchSpaceId, }); @@ -323,6 +327,13 @@ export function ModelConfigManager({ searchSpaceId }: ModelConfigManagerProps) {

{config.model_name}

+ {config.language && ( +
+ + {config.language} + +
+ )}
@@ -432,6 +443,7 @@ export function ModelConfigManager({ searchSpaceId }: ModelConfigManagerProps) { model_name: "", api_key: "", api_base: "", + language: "", litellm_params: {}, search_space_id: searchSpaceId, }); @@ -524,6 +536,25 @@ export function ModelConfigManager({ searchSpaceId }: ModelConfigManagerProps) { )} +
+ + +
+
; created_at: string; search_space_id: number; @@ -31,6 +32,7 @@ export interface CreateLLMConfig { model_name: string; api_key: string; api_base?: string; + language?: string; litellm_params?: Record; search_space_id: number; } From 807f4055f93398db3642202d80beb691de5d80de Mon Sep 17 00:00:00 2001 From: Tarun Date: Sun, 12 Oct 2025 13:44:45 +0530 Subject: [PATCH 14/35] Spelling mistake --- surfsense_web/components/settings/model-config-manager.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/surfsense_web/components/settings/model-config-manager.tsx b/surfsense_web/components/settings/model-config-manager.tsx index a236f1965..5719b01d1 100644 --- a/surfsense_web/components/settings/model-config-manager.tsx +++ b/surfsense_web/components/settings/model-config-manager.tsx @@ -82,7 +82,7 @@ export function ModelConfigManager({ searchSpaceId }: ModelConfigManagerProps) { model_name: editingConfig.model_name, api_key: editingConfig.api_key, api_base: editingConfig.api_base || "", - language: editingConfig.language || "Enlgish", + language: editingConfig.language || "English", litellm_params: editingConfig.litellm_params || {}, search_space_id: searchSpaceId, }); From 917cf4f398fae80ea8f014af9c5585d321cb6627 Mon Sep 17 00:00:00 2001 From: Differ <1467673018@qq.com> Date: Sun, 12 Oct 2025 19:10:46 +0800 Subject: [PATCH 15/35] feat: add Chinese LLM providers support with auto-fill API Base URL - Add support for DeepSeek, Qwen (Alibaba), Kimi (Moonshot), and GLM (Zhipu) - Implement auto-fill API Base URL when selecting Chinese LLM providers - Add smart validation and warnings for missing API endpoints - Fix session state management in task logging service - Add comprehensive Chinese setup documentation - Add database migration for new LLM provider enums Closes #383 --- docs/chinese-llm-setup.md | 323 ++++++++++++++++++ surfsense_backend/alembic/env.py | 6 + .../versions/26_add_chinese_llm_providers.py | 118 +++++++ surfsense_backend/app/db.py | 9 + .../app/routes/documents_routes.py | 1 + surfsense_backend/app/services/llm_service.py | 9 +- .../app/services/task_logging_service.py | 31 ++ .../settings/model-config-manager.tsx | 43 ++- .../contracts/enums/llm-providers.ts | 30 ++ 9 files changed, 565 insertions(+), 5 deletions(-) create mode 100644 docs/chinese-llm-setup.md create mode 100644 surfsense_backend/alembic/versions/26_add_chinese_llm_providers.py diff --git a/docs/chinese-llm-setup.md b/docs/chinese-llm-setup.md new file mode 100644 index 000000000..2a184608f --- /dev/null +++ b/docs/chinese-llm-setup.md @@ -0,0 +1,323 @@ +# 国产 LLM 配置指南 | Chinese LLM Setup Guide + +本指南将帮助你在 SurfSense 中配置和使用国产大语言模型。 + +This guide helps you configure and use Chinese LLM providers in SurfSense. + +--- + +## 📋 支持的提供商 | Supported Providers + +SurfSense 现已支持以下国产 LLM： + +- ✅ **DeepSeek** - 国产高性能 AI 模型 +- ✅ **阿里通义千问 (Alibaba Qwen)** - 阿里云通义千问大模型 +- ✅ **月之暗面 Kimi (Moonshot)** - 月之暗面 Kimi 大模型 +- ✅ **智谱 AI GLM (Zhipu)** - 智谱 AI GLM 系列模型 + +--- + +## 🚀 快速开始 | Quick Start + +### 通用配置步骤 | General Configuration Steps + +1. 登录 SurfSense Dashboard +2. 进入 **Settings** → **API Keys** (或 **LLM Configurations**) +3. 点击 **Add New Configuration** +4. 从 **Provider** 下拉菜单中选择你的国产 LLM 提供商 +5. 填写必填字段（见下方各提供商详细配置） +6. 点击 **Save** + +--- + +## 1️⃣ DeepSeek 配置 | DeepSeek Configuration + +### 获取 API Key + +1. 访问 [DeepSeek 开放平台](https://platform.deepseek.com/) +2. 注册并登录账号 +3. 进入 **API Keys** 页面 +4. 点击 **Create New API Key** +5. 复制生成的 API Key (格式: `sk-xxx`) + +### 在 SurfSense 中配置 + +| 字段 | 值 | 说明 | +|------|-----|------| +| **Configuration Name** | `DeepSeek Chat` | 配置名称（自定义） | +| **Provider** | `DEEPSEEK` | 选择 DeepSeek | +| **Model Name** | `deepseek-chat` | 推荐模型
其他选项: `deepseek-coder` | +| **API Key** | `sk-xxx...` | 你的 DeepSeek API Key | +| **API Base URL** | `https://api.deepseek.com` | DeepSeek API 地址 | +| **Parameters** | _(留空)_ | 使用默认参数 | + +### 示例配置 + +``` +Configuration Name: DeepSeek Chat +Provider: DEEPSEEK +Model Name: deepseek-chat +API Key: sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +API Base URL: https://api.deepseek.com +``` + +### 可用模型 + +- **deepseek-chat**: 通用对话模型（推荐） +- **deepseek-coder**: 代码专用模型 + +### 定价 +- 请访问 [DeepSeek 定价页面](https://platform.deepseek.com/pricing) 查看最新价格 + +--- + +## 2️⃣ 阿里通义千问 (Alibaba Qwen) 配置 + +### 获取 API Key + +1. 访问 [阿里云百炼平台](https://dashscope.aliyun.com/) +2. 登录阿里云账号 +3. 开通 DashScope 服务 +4. 进入 **API-KEY 管理** +5. 创建并复制 API Key + +### 在 SurfSense 中配置 + +| 字段 | 值 | 说明 | +|------|-----|------| +| **Configuration Name** | `通义千问 Max` | 配置名称（自定义） | +| **Provider** | `ALIBABA_QWEN` | 选择阿里通义千问 | +| **Model Name** | `qwen-max` | 推荐模型
其他选项: `qwen-plus`, `qwen-turbo` | +| **API Key** | `sk-xxx...` | 你的 DashScope API Key | +| **API Base URL** | `https://dashscope.aliyuncs.com/compatible-mode/v1` | 阿里云 API 地址 | +| **Parameters** | _(留空)_ | 使用默认参数 | + +### 示例配置 + +``` +Configuration Name: 通义千问 Max +Provider: ALIBABA_QWEN +Model Name: qwen-max +API Key: sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +API Base URL: https://dashscope.aliyuncs.com/compatible-mode/v1 +``` + +### 可用模型 + +- **qwen-max**: 最强性能，适合复杂任务 +- **qwen-plus**: 性价比高，适合日常使用（推荐） +- **qwen-turbo**: 速度快，适合简单任务 + +### 定价 +- 请访问 [阿里云百炼定价](https://help.aliyun.com/zh/model-studio/getting-started/billing) 查看最新价格 + +--- + +## 3️⃣ 月之暗面 Kimi (Moonshot) 配置 + +### 获取 API Key + +1. 访问 [Moonshot AI 开放平台](https://platform.moonshot.cn/) +2. 注册并登录账号 +3. 进入 **API Key 管理** +4. 创建新的 API Key +5. 复制 API Key + +### 在 SurfSense 中配置 + +| 字段 | 值 | 说明 | +|------|-----|------| +| **Configuration Name** | `Kimi` | 配置名称（自定义） | +| **Provider** | `MOONSHOT` | 选择月之暗面 Kimi | +| **Model Name** | `moonshot-v1-32k` | 推荐模型
其他选项: `moonshot-v1-8k`, `moonshot-v1-128k` | +| **API Key** | `sk-xxx...` | 你的 Moonshot API Key | +| **API Base URL** | `https://api.moonshot.cn/v1` | Moonshot API 地址 | +| **Parameters** | _(留空)_ | 使用默认参数 | + +### 示例配置 + +``` +Configuration Name: Kimi 32K +Provider: MOONSHOT +Model Name: moonshot-v1-32k +API Key: sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +API Base URL: https://api.moonshot.cn/v1 +``` + +### 可用模型 + +- **moonshot-v1-8k**: 8K 上下文（基础版） +- **moonshot-v1-32k**: 32K 上下文（推荐） +- **moonshot-v1-128k**: 128K 上下文（长文本专用） + +### 定价 +- 请访问 [Moonshot AI 定价](https://platform.moonshot.cn/pricing) 查看最新价格 + +--- + +## 4️⃣ 智谱 AI GLM (Zhipu) 配置 + +### 获取 API Key + +1. 访问 [智谱 AI 开放平台](https://open.bigmodel.cn/) +2. 注册并登录账号 +3. 进入 **API 管理** +4. 创建新的 API Key +5. 复制 API Key + +### 在 SurfSense 中配置 + +| 字段 | 值 | 说明 | +|------|-----|------| +| **Configuration Name** | `GLM-4` | 配置名称（自定义） | +| **Provider** | `ZHIPU` | 选择智谱 AI | +| **Model Name** | `glm-4` | 推荐模型
其他选项: `glm-4-flash`, `glm-3-turbo` | +| **API Key** | `xxx.yyy...` | 你的智谱 API Key | +| **API Base URL** | `https://open.bigmodel.cn/api/paas/v4` | 智谱 API 地址 | +| **Parameters** | _(留空)_ | 使用默认参数 | + +### 示例配置 + +``` +Configuration Name: GLM-4 +Provider: ZHIPU +Model Name: glm-4 +API Key: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.xxxxxxxxxxxxxxxx +API Base URL: https://open.bigmodel.cn/api/paas/v4 +``` + +### 可用模型 + +- **glm-4**: GLM-4 旗舰模型（推荐） +- **glm-4-flash**: 快速推理版本 +- **glm-3-turbo**: 高性价比版本 + +### 定价 +- 请访问 [智谱 AI 定价](https://open.bigmodel.cn/pricing) 查看最新价格 + +--- + +## ⚙️ 高级配置 | Advanced Configuration + +### 自定义参数 | Custom Parameters + +你可以在 **Parameters** 字段中添加自定义参数（JSON 格式）： + +```json +{ + "temperature": 0.7, + "max_tokens": 2000, + "top_p": 0.9 +} +``` + +### 常用参数说明 + +| 参数 | 说明 | 默认值 | 范围 | +|------|------|--------|------| +| `temperature` | 控制输出随机性，越高越随机 | 0.7 | 0.0 - 1.0 | +| `max_tokens` | 最大输出 Token 数 | 模型默认 | 1 - 模型上限 | +| `top_p` | 核采样参数 | 1.0 | 0.0 - 1.0 | + +--- + +## 🔧 故障排除 | Troubleshooting + +### 常见问题 + +#### 1. **错误: "Invalid API Key"** +- ✅ 检查 API Key 是否正确复制（无多余空格） +- ✅ 确认 API Key 是否已激活 +- ✅ 检查账户余额是否充足 + +#### 2. **错误: "Connection timeout"** +- ✅ 确认 API Base URL 是否正确 +- ✅ 检查网络连接 +- ✅ 确认防火墙是否允许访问 + +#### 3. **错误: "Model not found"** +- ✅ 确认模型名称是否拼写正确 +- ✅ 检查该模型是否已开通 +- ✅ 参照上方文档确认可用模型名称 + +#### 4. **文档处理卡住 (IN_PROGRESS)** +- ✅ 检查模型名称中是否有多余空格 +- ✅ 确认 API Key 有效且有额度 +- ✅ 查看后端日志: `docker compose logs backend` + +### 查看日志 + +```bash +# 查看后端日志 +docker compose logs backend --tail 100 + +# 实时查看日志 +docker compose logs -f backend + +# 搜索错误 +docker compose logs backend | grep -i "error" +``` + +--- + +## 💡 最佳实践 | Best Practices + +### 1. 模型选择建议 + +| 任务类型 | 推荐模型 | 说明 | +|---------|---------|------| +| **文档摘要** | Qwen-Plus, GLM-4 | 平衡性能和成本 | +| **代码分析** | DeepSeek-Coder | 代码专用 | +| **长文本处理** | Kimi 128K | 超长上下文 | +| **快速响应** | Qwen-Turbo, GLM-4-Flash | 速度优先 | + +### 2. 成本优化 + +- 🎯 **Long Context LLM**: 使用 Qwen-Plus 或 GLM-4（处理文档摘要） +- ⚡ **Fast LLM**: 使用 Qwen-Turbo 或 GLM-4-Flash（快速对话） +- 🧠 **Strategic LLM**: 使用 Qwen-Max 或 DeepSeek-Chat（复杂推理） + +### 3. API Key 安全 + +- ❌ 不要在公开代码中硬编码 API Key +- ✅ 定期轮换 API Key +- ✅ 为不同用途创建不同的 Key +- ✅ 设置合理的额度限制 + +--- + +## 📚 相关资源 | Resources + +### 官方文档 + +- [DeepSeek 文档](https://platform.deepseek.com/docs) +- [阿里云百炼文档](https://help.aliyun.com/zh/model-studio/) +- [Moonshot AI 文档](https://platform.moonshot.cn/docs) +- [智谱 AI 文档](https://open.bigmodel.cn/dev/api) + +### SurfSense 文档 + +- [安装指南](../README.md) +- [贡献指南](../CONTRIBUTING.md) +- [部署指南](../DEPLOYMENT_GUIDE.md) + +--- + +## 🆘 需要帮助？ | Need Help? + +如果遇到问题，可以通过以下方式获取帮助： + +- 💬 [GitHub Issues](https://github.com/MODSetter/SurfSense/issues) +- 💬 [Discord Community](https://discord.gg/ejRNvftDp9) +- 📧 Email: [项目维护者邮箱] + +--- + +## 🔄 更新日志 | Changelog + +- **2025-01-12**: 初始版本，添加 DeepSeek、Qwen、Kimi、GLM 支持 + +--- + +**祝你使用愉快！Happy coding with Chinese LLMs! 🚀** + diff --git a/surfsense_backend/alembic/env.py b/surfsense_backend/alembic/env.py index fd9740ee2..ae58ac5f9 100644 --- a/surfsense_backend/alembic/env.py +++ b/surfsense_backend/alembic/env.py @@ -20,6 +20,12 @@ from app.db import Base # Assuming your Base is defined in app.db # access to the values within the .ini file in use. config = context.config +# Override SQLAlchemy URL from environment variables when available +# 如果环境变量提供了数据库连接字符串，则优先使用该配置 +database_url = os.getenv("DATABASE_URL") +if database_url: + config.set_main_option("sqlalchemy.url", database_url) + # Interpret the config file for Python logging. # This line sets up loggers basically. if config.config_file_name is not None: diff --git a/surfsense_backend/alembic/versions/26_add_chinese_llm_providers.py b/surfsense_backend/alembic/versions/26_add_chinese_llm_providers.py new file mode 100644 index 000000000..f08a96738 --- /dev/null +++ b/surfsense_backend/alembic/versions/26_add_chinese_llm_providers.py @@ -0,0 +1,118 @@ +"""Add Chinese LLM providers to LiteLLMProvider enum +添加国产 LLM 提供商到 LiteLLMProvider 枚举 + +Revision ID: 26 +Revises: 25 +""" + +from collections.abc import Sequence + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "26" +down_revision: str | None = "25" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """ + Add Chinese LLM providers to LiteLLMProvider enum. + 添加国产 LLM 提供商到 LiteLLMProvider 枚举。 + + Adds support for: + - DEEPSEEK: DeepSeek AI models + - ALIBABA_QWEN: Alibaba Qwen (通义千问) models + - MOONSHOT: Moonshot AI (月之暗面 Kimi) models + - ZHIPU: Zhipu AI (智谱 GLM) models + """ + + # Add DEEPSEEK to the enum if it doesn't already exist + # 如果不存在则添加 DEEPSEEK 到枚举 + op.execute( + """ + DO $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_enum + WHERE enumtypid = 'litellmprovider'::regtype + AND enumlabel = 'DEEPSEEK' + ) THEN + ALTER TYPE litellmprovider ADD VALUE 'DEEPSEEK'; + END IF; + END$$; + """ + ) + + # Add ALIBABA_QWEN to the enum if it doesn't already exist + # 如果不存在则添加 ALIBABA_QWEN 到枚举 + op.execute( + """ + DO $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_enum + WHERE enumtypid = 'litellmprovider'::regtype + AND enumlabel = 'ALIBABA_QWEN' + ) THEN + ALTER TYPE litellmprovider ADD VALUE 'ALIBABA_QWEN'; + END IF; + END$$; + """ + ) + + # Add MOONSHOT to the enum if it doesn't already exist + # 如果不存在则添加 MOONSHOT 到枚举 + op.execute( + """ + DO $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_enum + WHERE enumtypid = 'litellmprovider'::regtype + AND enumlabel = 'MOONSHOT' + ) THEN + ALTER TYPE litellmprovider ADD VALUE 'MOONSHOT'; + END IF; + END$$; + """ + ) + + # Add ZHIPU to the enum if it doesn't already exist + # 如果不存在则添加 ZHIPU 到枚举 + op.execute( + """ + DO $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_enum + WHERE enumtypid = 'litellmprovider'::regtype + AND enumlabel = 'ZHIPU' + ) THEN + ALTER TYPE litellmprovider ADD VALUE 'ZHIPU'; + END IF; + END$$; + """ + ) + + +def downgrade() -> None: + """ + Remove Chinese LLM providers from LiteLLMProvider enum. + 从 LiteLLMProvider 枚举中移除国产 LLM 提供商。 + + Note: PostgreSQL doesn't support removing enum values directly. + This would require recreating the enum type and updating all dependent objects. + For safety, this downgrade is a no-op. + + 注意：PostgreSQL 不支持直接删除枚举值。 + 这需要重建枚举类型并更新所有依赖对象。 + 为了安全起见，此降级操作为空操作。 + """ + # PostgreSQL doesn't support removing enum values directly + # This would require a complex migration recreating the enum + # PostgreSQL 不支持直接删除枚举值 + # 这需要复杂的迁移来重建枚举 + pass + diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py index eb33145cf..9c75b6310 100644 --- a/surfsense_backend/app/db.py +++ b/surfsense_backend/app/db.py @@ -78,6 +78,10 @@ class ChatType(str, Enum): class LiteLLMProvider(str, Enum): + """ + Enum for LLM providers supported by LiteLLM. + LiteLLM 支持的 LLM 提供商枚举。 + """ OPENAI = "OPENAI" ANTHROPIC = "ANTHROPIC" GROQ = "GROQ" @@ -101,6 +105,11 @@ class LiteLLMProvider(str, Enum): ALEPH_ALPHA = "ALEPH_ALPHA" PETALS = "PETALS" COMETAPI = "COMETAPI" + # Chinese LLM Providers (OpenAI-compatible) / 国产 LLM 提供商（OpenAI 兼容） + DEEPSEEK = "DEEPSEEK" # DeepSeek + ALIBABA_QWEN = "ALIBABA_QWEN" # 阿里通义千问 + MOONSHOT = "MOONSHOT" # 月之暗面 (Kimi) + ZHIPU = "ZHIPU" # 智谱 AI (GLM) CUSTOM = "CUSTOM" diff --git a/surfsense_backend/app/routes/documents_routes.py b/surfsense_backend/app/routes/documents_routes.py index dd7b56033..63d7d2178 100644 --- a/surfsense_backend/app/routes/documents_routes.py +++ b/surfsense_backend/app/routes/documents_routes.py @@ -1070,6 +1070,7 @@ async def process_file_in_background( }, ) except Exception as e: + await session.rollback() await task_logger.log_task_failure( log_entry, f"Failed to process file: {filename}", diff --git a/surfsense_backend/app/services/llm_service.py b/surfsense_backend/app/services/llm_service.py index d9299549c..b3491ddd7 100644 --- a/surfsense_backend/app/services/llm_service.py +++ b/surfsense_backend/app/services/llm_service.py @@ -83,11 +83,11 @@ async def get_user_llm_instance( ) return None - # Build the model string for litellm + # Build the model string for litellm / 构建 LiteLLM 的模型字符串 if llm_config.custom_provider: model_string = f"{llm_config.custom_provider}/{llm_config.model_name}" else: - # Map provider enum to litellm format + # Map provider enum to litellm format / 将提供商枚举映射为 LiteLLM 格式 provider_map = { "OPENAI": "openai", "ANTHROPIC": "anthropic", @@ -99,6 +99,11 @@ async def get_user_llm_instance( "AZURE_OPENAI": "azure", "OPENROUTER": "openrouter", "COMETAPI": "cometapi", + # Chinese LLM providers (OpenAI-compatible) / 国产 LLM（OpenAI 兼容） + "DEEPSEEK": "openai", # DeepSeek uses OpenAI-compatible API + "ALIBABA_QWEN": "openai", # Qwen uses OpenAI-compatible API + "MOONSHOT": "openai", # Moonshot (Kimi) uses OpenAI-compatible API + "ZHIPU": "openai", # Zhipu (GLM) uses OpenAI-compatible API # Add more mappings as needed } provider_prefix = provider_map.get( diff --git a/surfsense_backend/app/services/task_logging_service.py b/surfsense_backend/app/services/task_logging_service.py index 39316b71f..784525b90 100644 --- a/surfsense_backend/app/services/task_logging_service.py +++ b/surfsense_backend/app/services/task_logging_service.py @@ -73,6 +73,16 @@ class TaskLoggingService: Returns: Log: The updated log entry """ + # Ensure session is in a valid state / 确保 session 处于有效状态 + if not self.session.is_active: + await self.session.rollback() + + # Refresh log_entry to avoid expired state / 刷新 log_entry 避免过期状态 + try: + await self.session.refresh(log_entry) + except Exception: + pass + # Update the existing log entry log_entry.status = LogStatus.SUCCESS log_entry.message = message @@ -114,6 +124,17 @@ class TaskLoggingService: Returns: Log: The updated log entry """ + # Ensure session is in a valid state / 确保 session 处于有效状态 + if not self.session.is_active: + await self.session.rollback() + + # Refresh log_entry to avoid expired state / 刷新 log_entry 避免过期状态 + try: + await self.session.refresh(log_entry) + except Exception: + # If refresh fails, the object might be detached / 如果刷新失败，对象可能已分离 + pass + # Update the existing log entry log_entry.status = LogStatus.FAILED log_entry.level = LogLevel.ERROR @@ -161,6 +182,16 @@ class TaskLoggingService: Returns: Log: The updated log entry """ + # Ensure session is in a valid state / 确保 session 处于有效状态 + if not self.session.is_active: + await self.session.rollback() + + # Refresh log_entry to avoid expired state / 刷新 log_entry 避免过期状态 + try: + await self.session.refresh(log_entry) + except Exception: + pass + log_entry.message = progress_message if progress_metadata: diff --git a/surfsense_web/components/settings/model-config-manager.tsx b/surfsense_web/components/settings/model-config-manager.tsx index 7384337b6..0cfb5feb2 100644 --- a/surfsense_web/components/settings/model-config-manager.tsx +++ b/surfsense_web/components/settings/model-config-manager.tsx @@ -90,6 +90,17 @@ export function ModelConfigManager({ searchSpaceId }: ModelConfigManagerProps) { setFormData((prev) => ({ ...prev, [field]: value })); }; + // Handle provider change with auto-fill API Base URL / 处理 Provider 变更并自动填充 API Base URL + const handleProviderChange = (providerValue: string) => { + const provider = LLM_PROVIDERS.find((p) => p.value === providerValue); + setFormData((prev) => ({ + ...prev, + provider: providerValue, + // Auto-fill API Base URL if provider has a default / 如果提供商有默认值则自动填充 + api_base: provider?.apiBase || prev.api_base, + })); + }; + const handleSubmit = async (e: React.FormEvent) => { e.preventDefault(); if (!formData.name || !formData.provider || !formData.model_name || !formData.api_key) { @@ -468,7 +479,7 @@ export function ModelConfigManager({ searchSpaceId }: ModelConfigManagerProps) { handleInputChange("api_base", e.target.value)} /> + {selectedProvider?.apiBase && formData.api_base === selectedProvider.apiBase && ( +

+ + Using recommended API endpoint for {selectedProvider.label} +

+ )} + {selectedProvider?.apiBase && !formData.api_base && ( +

+ + ⚠️ API Base URL is required for {selectedProvider.label}. Click to auto-fill: + +

+ )}
{/* Optional Inference Parameters */} diff --git a/surfsense_web/contracts/enums/llm-providers.ts b/surfsense_web/contracts/enums/llm-providers.ts index 753276fde..814ea16cf 100644 --- a/surfsense_web/contracts/enums/llm-providers.ts +++ b/surfsense_web/contracts/enums/llm-providers.ts @@ -3,6 +3,7 @@ export interface LLMProvider { label: string; example: string; description: string; + apiBase?: string; // Default API Base URL for the provider / 提供商的默认 API Base URL } export const LLM_PROVIDERS: LLMProvider[] = [ @@ -90,6 +91,35 @@ export const LLM_PROVIDERS: LLMProvider[] = [ example: "gpt-5-mini, claude-sonnet-4-5", description: "Access 500+ AI models through one unified API", }, + // Chinese LLM Providers / 国产 LLM 提供商 + { + value: "DEEPSEEK", + label: "DeepSeek", + example: "deepseek-chat, deepseek-coder", + description: "Chinese high-performance AI models", + apiBase: "https://api.deepseek.com", + }, + { + value: "ALIBABA_QWEN", + label: "Qwen", + example: "qwen-max, qwen-plus, qwen-turbo", + description: "Alibaba Cloud Qwen LLM", + apiBase: "https://dashscope.aliyuncs.com/compatible-mode/v1", + }, + { + value: "MOONSHOT", + label: "Kimi", + example: "moonshot-v1-8k, moonshot-v1-32k, moonshot-v1-128k", + description: "Moonshot AI Kimi models", + apiBase: "https://api.moonshot.cn/v1", + }, + { + value: "ZHIPU", + label: "GLM", + example: "glm-4, glm-4-flash, glm-3-turbo", + description: "Zhipu AI GLM series models", + apiBase: "https://open.bigmodel.cn/api/paas/v4", + }, { value: "CUSTOM", label: "Custom Provider", From d2ffd12480b9d99ca7d895bba0261c83c867ff30 Mon Sep 17 00:00:00 2001 From: Aki-07 Date: Sun, 12 Oct 2025 20:43:45 +0530 Subject: [PATCH 16/35] feat(back): add SearxNG connector type --- .../versions/26_add_searxng_connector_enum.py | 42 +++++++++++++++++++ surfsense_backend/app/db.py | 1 + surfsense_backend/app/utils/validators.py | 33 +++++++++++++-- 3 files changed, 73 insertions(+), 3 deletions(-) create mode 100644 surfsense_backend/alembic/versions/26_add_searxng_connector_enum.py diff --git a/surfsense_backend/alembic/versions/26_add_searxng_connector_enum.py b/surfsense_backend/alembic/versions/26_add_searxng_connector_enum.py new file mode 100644 index 000000000..ca6ac2b55 --- /dev/null +++ b/surfsense_backend/alembic/versions/26_add_searxng_connector_enum.py @@ -0,0 +1,42 @@ +"""Add SearxNG connector enum value + +Revision ID: 26 +Revises: 25 +Create Date: 2025-01-18 00:00:00.000000 + +""" + +from collections.abc import Sequence + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "26" +down_revision: str | None = "25" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Safely add SEARXNG_API to searchsourceconnectortype enum.""" + op.execute( + """ + DO $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_type t + JOIN pg_enum e ON t.oid = e.enumtypid + WHERE t.typname = 'searchsourceconnectortype' AND e.enumlabel = 'SEARXNG_API' + ) THEN + ALTER TYPE searchsourceconnectortype ADD VALUE 'SEARXNG_API'; + END IF; + END + $$; + """ + ) + + +def downgrade() -> None: + """Downgrade not supported for enum edits.""" + pass + diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py index eb33145cf..ab0db700e 100644 --- a/surfsense_backend/app/db.py +++ b/surfsense_backend/app/db.py @@ -55,6 +55,7 @@ class DocumentType(str, Enum): class SearchSourceConnectorType(str, Enum): SERPER_API = "SERPER_API" # NOT IMPLEMENTED YET : DON'T REMEMBER WHY : MOST PROBABLY BECAUSE WE NEED TO CRAWL THE RESULTS RETURNED BY IT TAVILY_API = "TAVILY_API" + SEARXNG_API = "SEARXNG_API" LINKUP_API = "LINKUP_API" SLACK_CONNECTOR = "SLACK_CONNECTOR" NOTION_CONNECTOR = "NOTION_CONNECTOR" diff --git a/surfsense_backend/app/utils/validators.py b/surfsense_backend/app/utils/validators.py index 437d23b55..7e29860ba 100644 --- a/surfsense_backend/app/utils/validators.py +++ b/surfsense_backend/app/utils/validators.py @@ -424,6 +424,22 @@ def validate_connector_config( connector_rules = { "SERPER_API": {"required": ["SERPER_API_KEY"], "validators": {}}, "TAVILY_API": {"required": ["TAVILY_API_KEY"], "validators": {}}, + "SEARXNG_API": { + "required": ["SEARXNG_HOST"], + "optional": [ + "SEARXNG_API_KEY", + "SEARXNG_ENGINES", + "SEARXNG_CATEGORIES", + "SEARXNG_LANGUAGE", + "SEARXNG_SAFESEARCH", + "SEARXNG_VERIFY_SSL", + ], + "validators": { + "SEARXNG_HOST": lambda: validate_url_field( + "SEARXNG_HOST", "SearxNG" + ) + }, + }, "LINKUP_API": {"required": ["LINKUP_API_KEY"], "validators": {}}, "SLACK_CONNECTOR": {"required": ["SLACK_BOT_TOKEN"], "validators": {}}, "NOTION_CONNECTOR": { @@ -484,10 +500,21 @@ def validate_connector_config( if not rules: return config # Unknown connector type, pass through - # Validate required keys match exactly - if set(config.keys()) != set(rules["required"]): + required_keys = set(rules["required"]) + optional_keys = set(rules.get("optional", [])) + config_keys = set(config.keys()) + + # Validate that no unexpected keys are present + if not config_keys.issubset(required_keys | optional_keys): + allowed_keys = list(required_keys | optional_keys) raise ValueError( - f"For {connector_type_str} connector type, config must only contain these keys: {rules['required']}" + f"For {connector_type_str} connector type, config may only contain these keys: {allowed_keys}" + ) + + # Validate that all required keys are present + if not required_keys.issubset(config_keys): + raise ValueError( + f"For {connector_type_str} connector type, config must include these keys: {sorted(required_keys)}" ) # Apply custom validators first (these check format before emptiness) From 020c5ed04eba9a185170fd0f525d998e4344691f Mon Sep 17 00:00:00 2001 From: Aki-07 Date: Sun, 12 Oct 2025 20:44:00 +0530 Subject: [PATCH 17/35] feat(back): add SearxNG search integration --- .../app/services/connector_service.py | 185 ++++++++++++++++++ 1 file changed, 185 insertions(+) diff --git a/surfsense_backend/app/services/connector_service.py b/surfsense_backend/app/services/connector_service.py index 83659a5c4..79061d409 100644 --- a/surfsense_backend/app/services/connector_service.py +++ b/surfsense_backend/app/services/connector_service.py @@ -1,6 +1,8 @@ import asyncio from typing import Any +from urllib.parse import urljoin +import httpx from linkup import LinkupClient from sqlalchemy import func from sqlalchemy.ext.asyncio import AsyncSession @@ -294,6 +296,189 @@ class ConnectorService: "sources": [], }, [] + async def search_searxng( + self, + user_query: str, + user_id: str, + search_space_id: int, + top_k: int = 20, + ) -> tuple: + """ + Search using a configured SearxNG instance and return both sources and documents. + """ + searx_connector = await self.get_connector_by_type( + user_id, SearchSourceConnectorType.SEARXNG_API, search_space_id + ) + + if not searx_connector: + return { + "id": 11, + "name": "SearxNG Search", + "type": "SEARXNG_API", + "sources": [], + }, [] + + config = searx_connector.config or {} + host = config.get("SEARXNG_HOST") + + if not host: + print("SearxNG connector is missing SEARXNG_HOST configuration") + return { + "id": 11, + "name": "SearxNG Search", + "type": "SEARXNG_API", + "sources": [], + }, [] + + api_key = config.get("SEARXNG_API_KEY") + engines = config.get("SEARXNG_ENGINES") + categories = config.get("SEARXNG_CATEGORIES") + language = config.get("SEARXNG_LANGUAGE") + safesearch = config.get("SEARXNG_SAFESEARCH") + + def _parse_bool(value: Any, default: bool = True) -> bool: + if isinstance(value, bool): + return value + if isinstance(value, str): + lowered = value.strip().lower() + if lowered in {"true", "1", "yes", "on"}: + return True + if lowered in {"false", "0", "no", "off"}: + return False + return default + + verify_ssl = _parse_bool(config.get("SEARXNG_VERIFY_SSL", True)) + + safesearch_value: int | None = None + if isinstance(safesearch, str): + safesearch_clean = safesearch.strip() + if safesearch_clean.isdigit(): + safesearch_value = int(safesearch_clean) + elif isinstance(safesearch, (int, float)): + safesearch_value = int(safesearch) + + def _format_list(value: Any) -> str | None: + if value is None: + return None + if isinstance(value, str): + value = value.strip() + return value or None + if isinstance(value, (list, tuple, set)): + cleaned = [str(item).strip() for item in value if str(item).strip()] + return ",".join(cleaned) if cleaned else None + return str(value) + + params: dict[str, Any] = { + "q": user_query, + "format": "json", + "language": language or "", + "limit": max(1, min(top_k, 50)), + } + + engines_param = _format_list(engines) + if engines_param: + params["engines"] = engines_param + + categories_param = _format_list(categories) + if categories_param: + params["categories"] = categories_param + + if safesearch_value is not None: + params["safesearch"] = safesearch_value + + if not params.get("language"): + params.pop("language") + + headers = {"Accept": "application/json"} + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + + searx_endpoint = urljoin(host if host.endswith("/") else f"{host}/", "search") + + try: + async with httpx.AsyncClient(timeout=20.0, verify=verify_ssl) as client: + response = await client.get( + searx_endpoint, + params=params, + headers=headers, + ) + response.raise_for_status() + except httpx.HTTPError as exc: + print(f"Error searching with SearxNG: {exc!s}") + return { + "id": 11, + "name": "SearxNG Search", + "type": "SEARXNG_API", + "sources": [], + }, [] + + try: + data = response.json() + except ValueError: + print("Failed to decode JSON response from SearxNG") + return { + "id": 11, + "name": "SearxNG Search", + "type": "SEARXNG_API", + "sources": [], + }, [] + + searx_results = data.get("results", []) + if not searx_results: + return { + "id": 11, + "name": "SearxNG Search", + "type": "SEARXNG_API", + "sources": [], + }, [] + + sources_list: list[dict[str, Any]] = [] + documents: list[dict[str, Any]] = [] + + async with self.counter_lock: + for result in searx_results: + description = result.get("content") or result.get("snippet") or "" + if len(description) > 160: + description = f"{description[:157]}..." + + source = { + "id": self.source_id_counter, + "title": result.get("title", "SearxNG Result"), + "description": description, + "url": result.get("url", ""), + } + sources_list.append(source) + + metadata = { + "url": result.get("url", ""), + "engines": result.get("engines", []), + "category": result.get("category"), + "source": "SEARXNG_API", + } + + document = { + "chunk_id": self.source_id_counter, + "content": description or result.get("content", ""), + "score": result.get("score", 0.0), + "document": { + "id": self.source_id_counter, + "title": result.get("title", "SearxNG Result"), + "document_type": "SEARXNG_API", + "metadata": metadata, + }, + } + documents.append(document) + self.source_id_counter += 1 + + result_object = { + "id": 11, + "name": "SearxNG Search", + "type": "SEARXNG_API", + "sources": sources_list, + } + + return result_object, documents + # Initialize Tavily client with API key from connector config tavily_api_key = tavily_connector.config.get("TAVILY_API_KEY") tavily_client = TavilyClient(api_key=tavily_api_key) From ec3e4888d816765ab6e106e95f569be771b743ab Mon Sep 17 00:00:00 2001 From: Aki-07 Date: Sun, 12 Oct 2025 20:44:14 +0530 Subject: [PATCH 18/35] feat(back): surface SearxNG results in researcher workflow --- .../app/agents/researcher/nodes.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/surfsense_backend/app/agents/researcher/nodes.py b/surfsense_backend/app/agents/researcher/nodes.py index 0835fb861..eaa5bc3fe 100644 --- a/surfsense_backend/app/agents/researcher/nodes.py +++ b/surfsense_backend/app/agents/researcher/nodes.py @@ -1032,6 +1032,30 @@ async def fetch_relevant_documents( } ) + elif connector == "SEARXNG_API": + ( + source_object, + searx_chunks, + ) = await connector_service.search_searxng( + user_query=reformulated_query, + user_id=user_id, + search_space_id=search_space_id, + top_k=top_k, + ) + + if source_object: + all_sources.append(source_object) + all_raw_documents.extend(searx_chunks) + + if streaming_service and writer: + writer( + { + "yield_value": streaming_service.format_terminal_info_delta( + f"🌐 Found {len(searx_chunks)} SearxNG results related to your query" + ) + } + ) + elif connector == "LINKUP_API": linkup_mode = "standard" From 0f56de340d4cd738c7103a79ab7abca31d102193 Mon Sep 17 00:00:00 2001 From: Aki-07 Date: Sun, 12 Oct 2025 20:44:30 +0530 Subject: [PATCH 19/35] docs: mention SearxNG integration --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cf0424fe2..ee3f5e0d1 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,7 @@ Open source and easy to deploy locally. ### ℹ️ **External Sources** - Search Engines (Tavily, LinkUp) +- SearxNG (self-hosted instances) - Slack - Linear - Jira @@ -303,4 +304,3 @@ For detailed contribution guidelines, please see our [CONTRIBUTING.md](CONTRIBUT --- --- - From 467da1e345b801d2ba48e1eb36ab43b8182f216f Mon Sep 17 00:00:00 2001 From: Aki-07 Date: Sun, 12 Oct 2025 20:44:43 +0530 Subject: [PATCH 20/35] feat(web): add SearxNG connector metadata --- surfsense_web/components/chat/SourceDetailSheet.tsx | 5 +++-- surfsense_web/contracts/enums/connector.ts | 1 + surfsense_web/contracts/enums/connectorIcons.tsx | 2 ++ surfsense_web/hooks/use-connectors.ts | 2 +- surfsense_web/lib/connectors/utils.ts | 1 + 5 files changed, 8 insertions(+), 3 deletions(-) diff --git a/surfsense_web/components/chat/SourceDetailSheet.tsx b/surfsense_web/components/chat/SourceDetailSheet.tsx index 4f7d129e4..138d004db 100644 --- a/surfsense_web/components/chat/SourceDetailSheet.tsx +++ b/surfsense_web/components/chat/SourceDetailSheet.tsx @@ -52,7 +52,8 @@ export function SourceDetailSheet({ const [summaryOpen, setSummaryOpen] = useState(false); // Check if this is a source type that should render directly from node - const isDirectRenderSource = sourceType === "TAVILY_API" || sourceType === "LINKUP_API"; + const isDirectRenderSource = + sourceType === "TAVILY_API" || sourceType === "LINKUP_API" || sourceType === "SEARXNG_API"; useEffect(() => { if (open && chunkId && !isDirectRenderSource) { @@ -108,7 +109,7 @@ export function SourceDetailSheet({ )} - {/* Direct render for TAVILY_API and LINKUP_API */} + {/* Direct render for web search providers */} {isDirectRenderSource && (
diff --git a/surfsense_web/contracts/enums/connector.ts b/surfsense_web/contracts/enums/connector.ts index a49314e2e..39c6fcfcf 100644 --- a/surfsense_web/contracts/enums/connector.ts +++ b/surfsense_web/contracts/enums/connector.ts @@ -1,6 +1,7 @@ export enum EnumConnectorName { SERPER_API = "SERPER_API", TAVILY_API = "TAVILY_API", + SEARXNG_API = "SEARXNG_API", LINKUP_API = "LINKUP_API", SLACK_CONNECTOR = "SLACK_CONNECTOR", NOTION_CONNECTOR = "NOTION_CONNECTOR", diff --git a/surfsense_web/contracts/enums/connectorIcons.tsx b/surfsense_web/contracts/enums/connectorIcons.tsx index f34a10398..1d17f49e1 100644 --- a/surfsense_web/contracts/enums/connectorIcons.tsx +++ b/surfsense_web/contracts/enums/connectorIcons.tsx @@ -32,6 +32,8 @@ export const getConnectorIcon = (connectorType: EnumConnectorName | string, clas return ; case EnumConnectorName.TAVILY_API: return ; + case EnumConnectorName.SEARXNG_API: + return ; case EnumConnectorName.SLACK_CONNECTOR: return ; case EnumConnectorName.NOTION_CONNECTOR: diff --git a/surfsense_web/hooks/use-connectors.ts b/surfsense_web/hooks/use-connectors.ts index 9e9f2dc53..63ee7b372 100644 --- a/surfsense_web/hooks/use-connectors.ts +++ b/surfsense_web/hooks/use-connectors.ts @@ -23,7 +23,7 @@ export const getConnectorTypeDisplay = (type: string): string => { const typeMap: Record = { SERPER_API: "Serper API", TAVILY_API: "Tavily API", - // Add other connector types here as needed + SEARXNG_API: "SearxNG", }; return typeMap[type] || type; }; diff --git a/surfsense_web/lib/connectors/utils.ts b/surfsense_web/lib/connectors/utils.ts index 798204d94..5b542d32d 100644 --- a/surfsense_web/lib/connectors/utils.ts +++ b/surfsense_web/lib/connectors/utils.ts @@ -3,6 +3,7 @@ export const getConnectorTypeDisplay = (type: string): string => { const typeMap: Record = { SERPER_API: "Serper API", TAVILY_API: "Tavily API", + SEARXNG_API: "SearxNG", SLACK_CONNECTOR: "Slack", NOTION_CONNECTOR: "Notion", GITHUB_CONNECTOR: "GitHub", From 3f82ad6cf119ca561acc3741b89557792921d761 Mon Sep 17 00:00:00 2001 From: Aki-07 Date: Sun, 12 Oct 2025 20:44:59 +0530 Subject: [PATCH 21/35] feat(web): support editing SearxNG connector settings --- .../components/editConnector/types.ts | 7 + .../hooks/use-connector-edit-page.ts | 165 ++++++++++++++++++ 2 files changed, 172 insertions(+) diff --git a/surfsense_web/components/editConnector/types.ts b/surfsense_web/components/editConnector/types.ts index 4e91ef305..c069972ba 100644 --- a/surfsense_web/components/editConnector/types.ts +++ b/surfsense_web/components/editConnector/types.ts @@ -30,6 +30,13 @@ export const editConnectorSchema = z.object({ NOTION_INTEGRATION_TOKEN: z.string().optional(), SERPER_API_KEY: z.string().optional(), TAVILY_API_KEY: z.string().optional(), + SEARXNG_HOST: z.string().optional(), + SEARXNG_API_KEY: z.string().optional(), + SEARXNG_ENGINES: z.string().optional(), + SEARXNG_CATEGORIES: z.string().optional(), + SEARXNG_LANGUAGE: z.string().optional(), + SEARXNG_SAFESEARCH: z.string().optional(), + SEARXNG_VERIFY_SSL: z.string().optional(), LINEAR_API_KEY: z.string().optional(), LINKUP_API_KEY: z.string().optional(), DISCORD_BOT_TOKEN: z.string().optional(), diff --git a/surfsense_web/hooks/use-connector-edit-page.ts b/surfsense_web/hooks/use-connector-edit-page.ts index fe90db3c8..ae8d977d7 100644 --- a/surfsense_web/hooks/use-connector-edit-page.ts +++ b/surfsense_web/hooks/use-connector-edit-page.ts @@ -16,6 +16,38 @@ import { useSearchSourceConnectors, } from "@/hooks/use-search-source-connectors"; +const normalizeListInput = (value: unknown): string[] => { + if (Array.isArray(value)) { + return value.map((item) => String(item).trim()).filter((item) => item.length > 0); + } + if (typeof value === "string") { + return value + .split(",") + .map((item) => item.trim()) + .filter((item) => item.length > 0); + } + return []; +}; + +const arraysEqual = (a: string[], b: string[]): boolean => { + if (a.length !== b.length) return false; + return a.every((value, index) => value === b[index]); +}; + +const normalizeBoolean = (value: unknown): boolean | null => { + if (typeof value === "boolean") return value; + if (typeof value === "string") { + const lowered = value.trim().toLowerCase(); + if (["true", "1", "yes", "on"].includes(lowered)) return true; + if (["false", "0", "no", "off"].includes(lowered)) return false; + } + if (typeof value === "number") { + if (value === 1) return true; + if (value === 0) return false; + } + return null; +}; + export function useConnectorEditPage(connectorId: number, searchSpaceId: string) { const router = useRouter(); const { @@ -48,6 +80,13 @@ export function useConnectorEditPage(connectorId: number, searchSpaceId: string) NOTION_INTEGRATION_TOKEN: "", SERPER_API_KEY: "", TAVILY_API_KEY: "", + SEARXNG_HOST: "", + SEARXNG_API_KEY: "", + SEARXNG_ENGINES: "", + SEARXNG_CATEGORIES: "", + SEARXNG_LANGUAGE: "", + SEARXNG_SAFESEARCH: "", + SEARXNG_VERIFY_SSL: "", LINEAR_API_KEY: "", DISCORD_BOT_TOKEN: "", CONFLUENCE_BASE_URL: "", @@ -74,6 +113,23 @@ export function useConnectorEditPage(connectorId: number, searchSpaceId: string) NOTION_INTEGRATION_TOKEN: config.NOTION_INTEGRATION_TOKEN || "", SERPER_API_KEY: config.SERPER_API_KEY || "", TAVILY_API_KEY: config.TAVILY_API_KEY || "", + SEARXNG_HOST: config.SEARXNG_HOST || "", + SEARXNG_API_KEY: config.SEARXNG_API_KEY || "", + SEARXNG_ENGINES: Array.isArray(config.SEARXNG_ENGINES) + ? config.SEARXNG_ENGINES.join(", ") + : config.SEARXNG_ENGINES || "", + SEARXNG_CATEGORIES: Array.isArray(config.SEARXNG_CATEGORIES) + ? config.SEARXNG_CATEGORIES.join(", ") + : config.SEARXNG_CATEGORIES || "", + SEARXNG_LANGUAGE: config.SEARXNG_LANGUAGE || "", + SEARXNG_SAFESEARCH: + config.SEARXNG_SAFESEARCH !== undefined && config.SEARXNG_SAFESEARCH !== null + ? String(config.SEARXNG_SAFESEARCH) + : "", + SEARXNG_VERIFY_SSL: + config.SEARXNG_VERIFY_SSL !== undefined && config.SEARXNG_VERIFY_SSL !== null + ? String(config.SEARXNG_VERIFY_SSL) + : "", LINEAR_API_KEY: config.LINEAR_API_KEY || "", LINKUP_API_KEY: config.LINKUP_API_KEY || "", DISCORD_BOT_TOKEN: config.DISCORD_BOT_TOKEN || "", @@ -238,6 +294,88 @@ export function useConnectorEditPage(connectorId: number, searchSpaceId: string) newConfig = { TAVILY_API_KEY: formData.TAVILY_API_KEY }; } break; + case "SEARXNG_API": { + const host = (formData.SEARXNG_HOST || "").trim(); + if (!host) { + toast.error("SearxNG host is required."); + setIsSaving(false); + return; + } + + const candidateConfig: Record = { SEARXNG_HOST: host }; + let hasChanges = host !== (originalConfig.SEARXNG_HOST || "").trim(); + + const apiKey = (formData.SEARXNG_API_KEY || "").trim(); + const originalApiKey = (originalConfig.SEARXNG_API_KEY || "").trim(); + if (apiKey !== originalApiKey) { + candidateConfig.SEARXNG_API_KEY = apiKey || null; + hasChanges = true; + } + + const newEngines = normalizeListInput(formData.SEARXNG_ENGINES || ""); + const originalEngines = normalizeListInput(originalConfig.SEARXNG_ENGINES); + if (!arraysEqual(newEngines, originalEngines)) { + candidateConfig.SEARXNG_ENGINES = newEngines; + hasChanges = true; + } + + const newCategories = normalizeListInput(formData.SEARXNG_CATEGORIES || ""); + const originalCategories = normalizeListInput(originalConfig.SEARXNG_CATEGORIES); + if (!arraysEqual(newCategories, originalCategories)) { + candidateConfig.SEARXNG_CATEGORIES = newCategories; + hasChanges = true; + } + + const language = (formData.SEARXNG_LANGUAGE || "").trim(); + const originalLanguage = (originalConfig.SEARXNG_LANGUAGE || "").trim(); + if (language !== originalLanguage) { + candidateConfig.SEARXNG_LANGUAGE = language || null; + hasChanges = true; + } + + const safesearchRaw = (formData.SEARXNG_SAFESEARCH || "").trim(); + const originalSafesearch = originalConfig.SEARXNG_SAFESEARCH; + if (safesearchRaw) { + const parsed = Number(safesearchRaw); + if (Number.isNaN(parsed)) { + toast.error("SearxNG SafeSearch must be a number."); + setIsSaving(false); + return; + } + if (parsed !== Number(originalSafesearch)) { + candidateConfig.SEARXNG_SAFESEARCH = parsed; + hasChanges = true; + } + } else if (originalSafesearch !== undefined && originalSafesearch !== null) { + candidateConfig.SEARXNG_SAFESEARCH = null; + hasChanges = true; + } + + const verifyRaw = (formData.SEARXNG_VERIFY_SSL || "").trim().toLowerCase(); + const originalVerifyBool = normalizeBoolean(originalConfig.SEARXNG_VERIFY_SSL); + if (verifyRaw) { + let parsedBool: boolean | null = null; + if (["true", "1", "yes", "on"].includes(verifyRaw)) parsedBool = true; + else if (["false", "0", "no", "off"].includes(verifyRaw)) parsedBool = false; + if (parsedBool === null) { + toast.error("SearxNG SSL verification must be true or false."); + setIsSaving(false); + return; + } + if (parsedBool !== originalVerifyBool) { + candidateConfig.SEARXNG_VERIFY_SSL = parsedBool; + hasChanges = true; + } + } else if (originalVerifyBool !== null) { + candidateConfig.SEARXNG_VERIFY_SSL = null; + hasChanges = true; + } + + if (hasChanges) { + newConfig = candidateConfig; + } + break; + } case "LINEAR_CONNECTOR": if (formData.LINEAR_API_KEY !== originalConfig.LINEAR_API_KEY) { @@ -367,6 +505,33 @@ export function useConnectorEditPage(connectorId: number, searchSpaceId: string) editForm.setValue("SERPER_API_KEY", newlySavedConfig.SERPER_API_KEY || ""); } else if (connector.connector_type === "TAVILY_API") { editForm.setValue("TAVILY_API_KEY", newlySavedConfig.TAVILY_API_KEY || ""); + } else if (connector.connector_type === "SEARXNG_API") { + editForm.setValue("SEARXNG_HOST", newlySavedConfig.SEARXNG_HOST || ""); + editForm.setValue("SEARXNG_API_KEY", newlySavedConfig.SEARXNG_API_KEY || ""); + editForm.setValue( + "SEARXNG_ENGINES", + normalizeListInput(newlySavedConfig.SEARXNG_ENGINES).join(", ") + ); + editForm.setValue( + "SEARXNG_CATEGORIES", + normalizeListInput(newlySavedConfig.SEARXNG_CATEGORIES).join(", ") + ); + editForm.setValue( + "SEARXNG_LANGUAGE", + newlySavedConfig.SEARXNG_LANGUAGE || "" + ); + editForm.setValue( + "SEARXNG_SAFESEARCH", + newlySavedConfig.SEARXNG_SAFESEARCH === null || + newlySavedConfig.SEARXNG_SAFESEARCH === undefined + ? "" + : String(newlySavedConfig.SEARXNG_SAFESEARCH) + ); + const verifyValue = normalizeBoolean(newlySavedConfig.SEARXNG_VERIFY_SSL); + editForm.setValue( + "SEARXNG_VERIFY_SSL", + verifyValue === null ? "" : String(verifyValue) + ); } else if (connector.connector_type === "LINEAR_CONNECTOR") { editForm.setValue("LINEAR_API_KEY", newlySavedConfig.LINEAR_API_KEY || ""); } else if (connector.connector_type === "LINKUP_API") { From 0f252573cf9d45019e8af302ab22c14f1f8d1e1e Mon Sep 17 00:00:00 2001 From: Aki-07 Date: Sun, 12 Oct 2025 20:45:14 +0530 Subject: [PATCH 22/35] feat(web): add SearxNG connector creation flow --- .../[search_space_id]/connectors/add/page.tsx | 7 + .../connectors/add/searxng/page.tsx | 360 ++++++++++++++++++ 2 files changed, 367 insertions(+) create mode 100644 surfsense_web/app/dashboard/[search_space_id]/connectors/add/searxng/page.tsx diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/page.tsx index a8bb9cfe8..1c5aa049d 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/page.tsx @@ -45,6 +45,13 @@ const connectorCategories: ConnectorCategory[] = [ icon: getConnectorIcon(EnumConnectorName.TAVILY_API, "h-6 w-6"), status: "available", }, + { + id: "searxng", + title: "SearxNG", + description: "Use your own SearxNG meta-search instance for web results.", + icon: getConnectorIcon(EnumConnectorName.SEARXNG_API, "h-6 w-6"), + status: "available", + }, { id: "linkup-api", title: "Linkup API", diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/searxng/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/searxng/page.tsx new file mode 100644 index 000000000..ca736a8cd --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/searxng/page.tsx @@ -0,0 +1,360 @@ +"use client"; + +import { zodResolver } from "@hookform/resolvers/zod"; +import { ArrowLeft, Check, Info, Loader2 } from "lucide-react"; +import { motion } from "motion/react"; +import { useParams, useRouter } from "next/navigation"; +import { useState } from "react"; +import { useForm } from "react-hook-form"; +import { toast } from "sonner"; +import * as z from "zod"; +import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; +import { Button } from "@/components/ui/button"; +import { + Card, + CardContent, + CardDescription, + CardFooter, + CardHeader, + CardTitle, +} from "@/components/ui/card"; +import { + Form, + FormControl, + FormDescription, + FormField, + FormItem, + FormLabel, + FormMessage, +} from "@/components/ui/form"; +import { Input } from "@/components/ui/input"; +import { Switch } from "@/components/ui/switch"; +import { EnumConnectorName } from "@/contracts/enums/connector"; +import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; +import { useSearchSourceConnectors } from "@/hooks/use-search-source-connectors"; + +const searxngFormSchema = z.object({ + name: z.string().min(3, { + message: "Connector name must be at least 3 characters.", + }), + host: z + .string({ required_error: "Host is required." }) + .url({ message: "Enter a valid SearxNG host URL (e.g. https://searxng.example.org)." }), + api_key: z.string().optional(), + engines: z.string().optional(), + categories: z.string().optional(), + language: z.string().optional(), + safesearch: z + .string() + .regex(/^[0-2]?$/, { message: "SafeSearch must be 0, 1, or 2." }) + .optional(), + verify_ssl: z.boolean().default(true), +}); + +type SearxngFormValues = z.infer; + +const parseCommaSeparated = (value?: string | null) => { + if (!value) return undefined; + const items = value + .split(",") + .map((item) => item.trim()) + .filter((item) => item.length > 0); + return items.length > 0 ? items : undefined; +}; + +export default function SearxngConnectorPage() { + const router = useRouter(); + const params = useParams(); + const searchSpaceId = params.search_space_id as string; + const [isSubmitting, setIsSubmitting] = useState(false); + const { createConnector } = useSearchSourceConnectors(); + + const form = useForm({ + resolver: zodResolver(searxngFormSchema), + defaultValues: { + name: "SearxNG Connector", + host: "", + api_key: "", + engines: "", + categories: "", + language: "", + safesearch: "", + verify_ssl: true, + }, + }); + + const onSubmit = async (values: SearxngFormValues) => { + setIsSubmitting(true); + try { + const config: Record = { + SEARXNG_HOST: values.host.trim(), + }; + + const apiKey = values.api_key?.trim(); + if (apiKey) config.SEARXNG_API_KEY = apiKey; + + const engines = parseCommaSeparated(values.engines); + if (engines) config.SEARXNG_ENGINES = engines; + + const categories = parseCommaSeparated(values.categories); + if (categories) config.SEARXNG_CATEGORIES = categories; + + const language = values.language?.trim(); + if (language) config.SEARXNG_LANGUAGE = language; + + const safesearch = values.safesearch?.trim(); + if (safesearch) { + const parsed = Number(safesearch); + if (!Number.isNaN(parsed)) { + config.SEARXNG_SAFESEARCH = parsed; + } + } + + // Include verify flag only when disabled to keep config minimal + if (values.verify_ssl === false) { + config.SEARXNG_VERIFY_SSL = false; + } + + await createConnector( + { + name: values.name, + connector_type: EnumConnectorName.SEARXNG_API, + config, + is_indexable: false, + last_indexed_at: null, + }, + parseInt(searchSpaceId) + ); + + toast.success("SearxNG connector created successfully!"); + router.push(`/dashboard/${searchSpaceId}/connectors`); + } catch (error) { + console.error("Error creating SearxNG connector:", error); + toast.error(error instanceof Error ? error.message : "Failed to create connector"); + } finally { + setIsSubmitting(false); + } + }; + + return ( +
+ + +
+
+
+ {getConnectorIcon(EnumConnectorName.SEARXNG_API, "h-6 w-6")} +
+
+

Connect SearxNG

+

+ Bring your self-hosted SearxNG meta-search engine into SurfSense. +

+
+
+
+ + + + + Connect SearxNG + + Integrate SurfSense with any SearxNG instance to broaden your search coverage while + preserving privacy and control. + + + + + + SearxNG Instance Required + + You need access to a running SearxNG instance. Refer to the{" "} + + SearxNG installation guide + {" "} + for setup instructions. If your instance requires an API key, include it below. + + + + + + ( + + Connector Name + + + + A friendly name to identify this connector. + + + )} + /> + + ( + + SearxNG Host + + + + + Provide the full base URL to your SearxNG instance. Include the protocol + (http/https). + + + + )} + /> + + ( + + API Key (optional) + + + + + Leave empty if your SearxNG instance does not enforce API keys. + + + + )} + /> + +
+ ( + + Engines (optional) + + + + Comma-separated list to target specific engines. + + + )} + /> + + ( + + Categories (optional) + + + + Comma-separated list of SearxNG categories. + + + )} + /> +
+ +
+ ( + + Preferred Language (optional) + + + + + IETF language tag (e.g. en, en-US). Leave blank to inherit defaults. + + + + )} + /> + + ( + + SafeSearch Level (optional) + + + + + Set 0, 1, or 2 to adjust SafeSearch filtering. Leave blank to use the instance + default. + + + + )} + /> +
+ + ( + +
+ Verify SSL Certificates + + Disable only when connecting to instances with self-signed certificates. + +
+ + + + + )} + /> + + + + + + + + + +
+ ); +} From ff08daccb1fba3e326b67de91e5f846e80a15d36 Mon Sep 17 00:00:00 2001 From: Aki-07 Date: Sun, 12 Oct 2025 21:29:12 +0530 Subject: [PATCH 23/35] fix: restore Tavily search and tighten SearxNG config --- .../app/services/connector_service.py | 161 +++++++++--------- .../hooks/use-connector-edit-page.ts | 9 +- 2 files changed, 89 insertions(+), 81 deletions(-) diff --git a/surfsense_backend/app/services/connector_service.py b/surfsense_backend/app/services/connector_service.py index 79061d409..0017fabaf 100644 --- a/surfsense_backend/app/services/connector_service.py +++ b/surfsense_backend/app/services/connector_service.py @@ -296,6 +296,84 @@ class ConnectorService: "sources": [], }, [] + # Initialize Tavily client with API key from connector config + tavily_api_key = tavily_connector.config.get("TAVILY_API_KEY") + tavily_client = TavilyClient(api_key=tavily_api_key) + + # Perform search with Tavily + try: + response = tavily_client.search( + query=user_query, + max_results=top_k, + search_depth="advanced", # Use advanced search for better results + ) + + # Extract results from Tavily response + tavily_results = response.get("results", []) + + # Early return if no results + if not tavily_results: + return { + "id": 3, + "name": "Tavily Search", + "type": "TAVILY_API", + "sources": [], + }, [] + + # Process each result and create sources directly without deduplication + sources_list = [] + documents = [] + + async with self.counter_lock: + for _i, result in enumerate(tavily_results): + # Create a source entry + source = { + "id": self.source_id_counter, + "title": result.get("title", "Tavily Result"), + "description": result.get("content", "")[:100], + "url": result.get("url", ""), + } + sources_list.append(source) + + # Create a document entry + document = { + "chunk_id": self.source_id_counter, + "content": result.get("content", ""), + "score": result.get("score", 0.0), + "document": { + "id": self.source_id_counter, + "title": result.get("title", "Tavily Result"), + "document_type": "TAVILY_API", + "metadata": { + "url": result.get("url", ""), + "published_date": result.get("published_date", ""), + "source": "TAVILY_API", + }, + }, + } + documents.append(document) + self.source_id_counter += 1 + + # Create result object + result_object = { + "id": 3, + "name": "Tavily Search", + "type": "TAVILY_API", + "sources": sources_list, + } + + return result_object, documents + + except Exception as e: + # Log the error and return empty results + print(f"Error searching with Tavily: {e!s}") + return { + "id": 3, + "name": "Tavily Search", + "type": "TAVILY_API", + "sources": [], + }, [] + async def search_searxng( self, user_query: str, @@ -357,6 +435,9 @@ class ConnectorService: elif isinstance(safesearch, (int, float)): safesearch_value = int(safesearch) + if safesearch_value is not None and not (0 <= safesearch_value <= 2): + safesearch_value = None + def _format_list(value: Any) -> str | None: if value is None: return None @@ -391,7 +472,7 @@ class ConnectorService: headers = {"Accept": "application/json"} if api_key: - headers["Authorization"] = f"Bearer {api_key}" + headers["X-API-KEY"] = api_key searx_endpoint = urljoin(host if host.endswith("/") else f"{host}/", "search") @@ -479,84 +560,6 @@ class ConnectorService: return result_object, documents - # Initialize Tavily client with API key from connector config - tavily_api_key = tavily_connector.config.get("TAVILY_API_KEY") - tavily_client = TavilyClient(api_key=tavily_api_key) - - # Perform search with Tavily - try: - response = tavily_client.search( - query=user_query, - max_results=top_k, - search_depth="advanced", # Use advanced search for better results - ) - - # Extract results from Tavily response - tavily_results = response.get("results", []) - - # Early return if no results - if not tavily_results: - return { - "id": 3, - "name": "Tavily Search", - "type": "TAVILY_API", - "sources": [], - }, [] - - # Process each result and create sources directly without deduplication - sources_list = [] - documents = [] - - async with self.counter_lock: - for _i, result in enumerate(tavily_results): - # Create a source entry - source = { - "id": self.source_id_counter, - "title": result.get("title", "Tavily Result"), - "description": result.get("content", "")[:100], - "url": result.get("url", ""), - } - sources_list.append(source) - - # Create a document entry - document = { - "chunk_id": self.source_id_counter, - "content": result.get("content", ""), - "score": result.get("score", 0.0), - "document": { - "id": self.source_id_counter, - "title": result.get("title", "Tavily Result"), - "document_type": "TAVILY_API", - "metadata": { - "url": result.get("url", ""), - "published_date": result.get("published_date", ""), - "source": "TAVILY_API", - }, - }, - } - documents.append(document) - self.source_id_counter += 1 - - # Create result object - result_object = { - "id": 3, - "name": "Tavily Search", - "type": "TAVILY_API", - "sources": sources_list, - } - - return result_object, documents - - except Exception as e: - # Log the error and return empty results - print(f"Error searching with Tavily: {e!s}") - return { - "id": 3, - "name": "Tavily Search", - "type": "TAVILY_API", - "sources": [], - }, [] - async def search_slack( self, user_query: str, diff --git a/surfsense_web/hooks/use-connector-edit-page.ts b/surfsense_web/hooks/use-connector-edit-page.ts index ae8d977d7..ea6a3fe10 100644 --- a/surfsense_web/hooks/use-connector-edit-page.ts +++ b/surfsense_web/hooks/use-connector-edit-page.ts @@ -337,8 +337,13 @@ export function useConnectorEditPage(connectorId: number, searchSpaceId: string) const originalSafesearch = originalConfig.SEARXNG_SAFESEARCH; if (safesearchRaw) { const parsed = Number(safesearchRaw); - if (Number.isNaN(parsed)) { - toast.error("SearxNG SafeSearch must be a number."); + if ( + Number.isNaN(parsed) || + !Number.isInteger(parsed) || + parsed < 0 || + parsed > 2 + ) { + toast.error("SearxNG SafeSearch must be 0, 1, or 2."); setIsSaving(false); return; } From 120d60465e33bdb6f51195a3d9f23a47c5091ef6 Mon Sep 17 00:00:00 2001 From: Tarun Date: Sun, 12 Oct 2025 23:40:46 +0530 Subject: [PATCH 24/35] refactor: streamline language instruction handling across prompts --- .../app/agents/researcher/prompts.py | 9 +++++---- .../app/agents/researcher/qna_agent/prompts.py | 11 +++-------- .../researcher/sub_section_writer/prompts.py | 11 +++-------- surfsense_backend/app/routes/chats_routes.py | 18 +++++++----------- 4 files changed, 18 insertions(+), 31 deletions(-) diff --git a/surfsense_backend/app/agents/researcher/prompts.py b/surfsense_backend/app/agents/researcher/prompts.py index b7265602a..868a78851 100644 --- a/surfsense_backend/app/agents/researcher/prompts.py +++ b/surfsense_backend/app/agents/researcher/prompts.py @@ -1,10 +1,11 @@ import datetime - -def get_answer_outline_system_prompt(language: str | None = None) -> str: - language_instruction = "" +def _build_language_instruction(language: str | None = None): if language: - language_instruction = f"\n\nIMPORTANT: Please respond in {language} language. All your responses, explanations, and analysis should be written in {language}." + return f"\n\nIMPORTANT: Please respond in {language} language. All your responses, explanations, and analysis should be written in {language}." + return "" +def get_answer_outline_system_prompt(language: str | None = None) -> str: + language_instruction = _build_language_instruction(language) return f""" Today's date: {datetime.datetime.now().strftime("%Y-%m-%d")} diff --git a/surfsense_backend/app/agents/researcher/qna_agent/prompts.py b/surfsense_backend/app/agents/researcher/qna_agent/prompts.py index deb5dd59f..de17ec933 100644 --- a/surfsense_backend/app/agents/researcher/qna_agent/prompts.py +++ b/surfsense_backend/app/agents/researcher/qna_agent/prompts.py @@ -1,5 +1,5 @@ import datetime - +from ..prompts import _build_language_instruction def get_qna_citation_system_prompt(chat_history: str | None = None, language: str | None = None): chat_history_section = ( @@ -17,10 +17,7 @@ NO CHAT HISTORY PROVIDED ) # Add language instruction if specified - language_instruction = "" - if language: - language_instruction = f"\n\nIMPORTANT: Please respond in {language} language. All your responses, explanations, and analysis should be written in {language}." - + language_instruction = _build_language_instruction(language) return f""" Today's date: {datetime.datetime.now().strftime("%Y-%m-%d")} You are SurfSense, an advanced AI research assistant that provides detailed, well-researched answers to user questions by synthesizing information from multiple personal knowledge sources.{language_instruction} @@ -170,9 +167,7 @@ NO CHAT HISTORY PROVIDED ) # Add language instruction if specified - language_instruction = "" - if language: - language_instruction = f"\n\nIMPORTANT: Please respond in {language} language. All your responses, explanations, and analysis should be written in {language}." + language_instruction = _build_language_instruction(language) return f""" Today's date: {datetime.datetime.now().strftime("%Y-%m-%d")} diff --git a/surfsense_backend/app/agents/researcher/sub_section_writer/prompts.py b/surfsense_backend/app/agents/researcher/sub_section_writer/prompts.py index 3954d47e5..a6a561bf1 100644 --- a/surfsense_backend/app/agents/researcher/sub_section_writer/prompts.py +++ b/surfsense_backend/app/agents/researcher/sub_section_writer/prompts.py @@ -1,6 +1,5 @@ import datetime - - +from ..prompts import _build_language_instruction def get_citation_system_prompt(chat_history: str | None = None, language: str | None = None): chat_history_section = ( f""" @@ -17,9 +16,7 @@ NO CHAT HISTORY PROVIDED ) # Add language instruction if specified - language_instruction = "" - if language: - language_instruction = f"\n\nIMPORTANT: Please respond in {language} language. All your responses, explanations, and analysis should be written in {language}." + language_instruction = _build_language_instruction(language) return f""" Today's date: {datetime.datetime.now().strftime("%Y-%m-%d")} @@ -177,9 +174,7 @@ NO CHAT HISTORY PROVIDED ) # Add language instruction if specified - language_instruction = "" - if language: - language_instruction = f"\n\nIMPORTANT: Please respond in {language} language. All your responses, explanations, and analysis should be written in {language}." + language_instruction = _build_language_instruction(language) return f""" Today's date: {datetime.datetime.now().strftime("%Y-%m-%d")} diff --git a/surfsense_backend/app/routes/chats_routes.py b/surfsense_backend/app/routes/chats_routes.py index d30bf5451..0874d2611 100644 --- a/surfsense_backend/app/routes/chats_routes.py +++ b/surfsense_backend/app/routes/chats_routes.py @@ -75,33 +75,29 @@ async def handle_chat_data( ) ) user_preference = language_result.scalars().first() - print("UserSearchSpacePreference:", user_preference) + # print("UserSearchSpacePreference:", user_preference) language = None if user_preference and user_preference.search_space and user_preference.search_space.llm_configs: llm_configs = user_preference.search_space.llm_configs - # print(f"Found {len(llm_configs)} LLM Configs") - # for i, config in enumerate(llm_configs): - # print(f" Config {i+1}: name={config.name}, provider={config.provider}, language={getattr(config, 'language', None)}") for preferred_llm in [user_preference.fast_llm, user_preference.long_context_llm, user_preference.strategic_llm]: if preferred_llm and getattr(preferred_llm, 'language', None): language = preferred_llm.language - # print(f"Using language from preferred LLM: {preferred_llm.name} -> {language}") break - # no preferred llM has language use first available LLM config - if not language: - first_llm_config = llm_configs[0] - language = getattr(first_llm_config, 'language', None) - # print(f"Using language from first LLM config: {first_llm_config.name} -> {language}") + + if not language: + first_llm_config = llm_configs[0] + language = getattr(first_llm_config, 'language', None) + except HTTPException: raise HTTPException( status_code=403, detail="You don't have access to this search space" ) from None - # print("Language selected:", language) + langchain_chat_history = [] for message in messages[:-1]: if message["role"] == "user": From 6f7e8e003f24e0c4f239e96d90770f85bfc08e33 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Mon, 13 Oct 2025 03:17:28 +0530 Subject: [PATCH 25/35] fix: improve table layout and prevent content overflow in LogsTable. --- .../[search_space_id]/logs/(manage)/page.tsx | 32 +++++++++++++++---- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/surfsense_web/app/dashboard/[search_space_id]/logs/(manage)/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/logs/(manage)/page.tsx index f8a0a593e..6773d2aad 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/logs/(manage)/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/logs/(manage)/page.tsx @@ -839,7 +839,7 @@ function LogsTable({ animate={{ opacity: 1, y: 0 }} transition={{ delay: 0.3 }} > - +
{table.getHeaderGroups().map((headerGroup: any) => ( @@ -847,7 +847,11 @@ function LogsTable({ {header.isPlaceholder ? null : header.column.getCanSort() ? ( + + ); + } + + // Default rendering for other columns + return ( + + {flexRender(cell.column.columnDef.cell, cell.getContext())} + + ); + })} + + )) + ) : ( + + + No logs found. + + + )} +
From a3f50ebc4dbad1524c01c6b8022b986d193b443f Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Sun, 12 Oct 2025 20:15:27 -0700 Subject: [PATCH 27/35] feat: added missed migration --- ...associate_connectors_with_search_spaces.py | 1 - .../versions/24_fix_null_chat_types.py | 1 - ...25_migrate_llm_configs_to_search_spaces.py | 1 - .../26_add_language_column_to_llm_configs.py | 69 +++++++++++++++++++ .../app/agents/researcher/configuration.py | 3 +- .../app/agents/researcher/prompts.py | 3 + .../app/agents/researcher/qna_agent/nodes.py | 6 +- .../agents/researcher/qna_agent/prompts.py | 14 ++-- .../researcher/sub_section_writer/prompts.py | 15 ++-- surfsense_backend/app/routes/chats_routes.py | 43 +++++++----- .../app/routes/llm_config_routes.py | 8 +-- 11 files changed, 126 insertions(+), 38 deletions(-) create mode 100644 surfsense_backend/alembic/versions/26_add_language_column_to_llm_configs.py diff --git a/surfsense_backend/alembic/versions/23_associate_connectors_with_search_spaces.py b/surfsense_backend/alembic/versions/23_associate_connectors_with_search_spaces.py index 20e9d7840..a693b9ec5 100644 --- a/surfsense_backend/alembic/versions/23_associate_connectors_with_search_spaces.py +++ b/surfsense_backend/alembic/versions/23_associate_connectors_with_search_spaces.py @@ -2,7 +2,6 @@ Revision ID: '23' Revises: '22' -Create Date: 2025-01-10 12:00:00.000000 """ diff --git a/surfsense_backend/alembic/versions/24_fix_null_chat_types.py b/surfsense_backend/alembic/versions/24_fix_null_chat_types.py index 35313d27b..e0d371f1e 100644 --- a/surfsense_backend/alembic/versions/24_fix_null_chat_types.py +++ b/surfsense_backend/alembic/versions/24_fix_null_chat_types.py @@ -2,7 +2,6 @@ Revision ID: 24 Revises: 23 -Create Date: 2025-01-10 14:00:00.000000 """ diff --git a/surfsense_backend/alembic/versions/25_migrate_llm_configs_to_search_spaces.py b/surfsense_backend/alembic/versions/25_migrate_llm_configs_to_search_spaces.py index 116a3c687..c9966599c 100644 --- a/surfsense_backend/alembic/versions/25_migrate_llm_configs_to_search_spaces.py +++ b/surfsense_backend/alembic/versions/25_migrate_llm_configs_to_search_spaces.py @@ -2,7 +2,6 @@ Revision ID: 25 Revises: 24 -Create Date: 2025-01-10 14:00:00.000000 Changes: 1. Migrate llm_configs from user association to search_space association diff --git a/surfsense_backend/alembic/versions/26_add_language_column_to_llm_configs.py b/surfsense_backend/alembic/versions/26_add_language_column_to_llm_configs.py new file mode 100644 index 000000000..e5cdc37d7 --- /dev/null +++ b/surfsense_backend/alembic/versions/26_add_language_column_to_llm_configs.py @@ -0,0 +1,69 @@ +"""Add language column to llm_configs + +Revision ID: 26 +Revises: 25 + +Changes: +1. Add language column to llm_configs table with default value of 'English' +""" + +from collections.abc import Sequence + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "26" +down_revision: str | None = "25" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Add language column to llm_configs table.""" + + from sqlalchemy import inspect + + conn = op.get_bind() + inspector = inspect(conn) + + # Get existing columns + llm_config_columns = [col["name"] for col in inspector.get_columns("llm_configs")] + + # Add language column if it doesn't exist + if "language" not in llm_config_columns: + op.add_column( + "llm_configs", + sa.Column( + "language", + sa.String(length=50), + nullable=True, + server_default="English", + ), + ) + + # Update existing rows to have 'English' as default + op.execute( + """ + UPDATE llm_configs + SET language = 'English' + WHERE language IS NULL + """ + ) + + +def downgrade() -> None: + """Remove language column from llm_configs table.""" + + from sqlalchemy import inspect + + conn = op.get_bind() + inspector = inspect(conn) + + # Get existing columns + llm_config_columns = [col["name"] for col in inspector.get_columns("llm_configs")] + + # Drop language column if it exists + if "language" in llm_config_columns: + op.drop_column("llm_configs", "language") diff --git a/surfsense_backend/app/agents/researcher/configuration.py b/surfsense_backend/app/agents/researcher/configuration.py index e8ee856de..24d8c819e 100644 --- a/surfsense_backend/app/agents/researcher/configuration.py +++ b/surfsense_backend/app/agents/researcher/configuration.py @@ -37,8 +37,7 @@ class Configuration: search_mode: SearchMode research_mode: ResearchMode document_ids_to_add_in_context: list[int] - language: str | None = None - + language: str | None = None @classmethod def from_runnable_config( diff --git a/surfsense_backend/app/agents/researcher/prompts.py b/surfsense_backend/app/agents/researcher/prompts.py index 868a78851..825772a24 100644 --- a/surfsense_backend/app/agents/researcher/prompts.py +++ b/surfsense_backend/app/agents/researcher/prompts.py @@ -1,9 +1,12 @@ import datetime + def _build_language_instruction(language: str | None = None): if language: return f"\n\nIMPORTANT: Please respond in {language} language. All your responses, explanations, and analysis should be written in {language}." return "" + + def get_answer_outline_system_prompt(language: str | None = None) -> str: language_instruction = _build_language_instruction(language) diff --git a/surfsense_backend/app/agents/researcher/qna_agent/nodes.py b/surfsense_backend/app/agents/researcher/qna_agent/nodes.py index 20374b706..c4e79d685 100644 --- a/surfsense_backend/app/agents/researcher/qna_agent/nodes.py +++ b/surfsense_backend/app/agents/researcher/qna_agent/nodes.py @@ -102,7 +102,7 @@ async def answer_question(state: State, config: RunnableConfig) -> dict[str, Any user_query = configuration.user_query user_id = configuration.user_id search_space_id = configuration.search_space_id - language = configuration.language + language = configuration.language # Get user's fast LLM llm = await get_user_fast_llm(state.db_session, user_id, search_space_id) if not llm: @@ -127,7 +127,9 @@ async def answer_question(state: State, config: RunnableConfig) -> dict[str, Any """ # Use initial system prompt for token calculation - initial_system_prompt = get_qna_citation_system_prompt(chat_history_str, language) + initial_system_prompt = get_qna_citation_system_prompt( + chat_history_str, language + ) base_messages = [ SystemMessage(content=initial_system_prompt), HumanMessage(content=base_human_message_template), diff --git a/surfsense_backend/app/agents/researcher/qna_agent/prompts.py b/surfsense_backend/app/agents/researcher/qna_agent/prompts.py index de17ec933..9c35f90cc 100644 --- a/surfsense_backend/app/agents/researcher/qna_agent/prompts.py +++ b/surfsense_backend/app/agents/researcher/qna_agent/prompts.py @@ -1,7 +1,11 @@ import datetime + from ..prompts import _build_language_instruction -def get_qna_citation_system_prompt(chat_history: str | None = None, language: str | None = None): + +def get_qna_citation_system_prompt( + chat_history: str | None = None, language: str | None = None +): chat_history_section = ( f""" @@ -15,7 +19,7 @@ NO CHAT HISTORY PROVIDED """ ) - + # Add language instruction if specified language_instruction = _build_language_instruction(language) return f""" @@ -151,7 +155,9 @@ Make sure your response: """ -def get_qna_no_documents_system_prompt(chat_history: str | None = None, language: str | None = None): +def get_qna_no_documents_system_prompt( + chat_history: str | None = None, language: str | None = None +): chat_history_section = ( f""" @@ -165,7 +171,7 @@ NO CHAT HISTORY PROVIDED """ ) - + # Add language instruction if specified language_instruction = _build_language_instruction(language) diff --git a/surfsense_backend/app/agents/researcher/sub_section_writer/prompts.py b/surfsense_backend/app/agents/researcher/sub_section_writer/prompts.py index a6a561bf1..3c34eb474 100644 --- a/surfsense_backend/app/agents/researcher/sub_section_writer/prompts.py +++ b/surfsense_backend/app/agents/researcher/sub_section_writer/prompts.py @@ -1,6 +1,11 @@ import datetime + from ..prompts import _build_language_instruction -def get_citation_system_prompt(chat_history: str | None = None, language: str | None = None): + + +def get_citation_system_prompt( + chat_history: str | None = None, language: str | None = None +): chat_history_section = ( f""" @@ -14,7 +19,7 @@ NO CHAT HISTORY PROVIDED """ ) - + # Add language instruction if specified language_instruction = _build_language_instruction(language) @@ -158,7 +163,9 @@ Make sure your response: """ -def get_no_documents_system_prompt(chat_history: str | None = None, language: str | None = None): +def get_no_documents_system_prompt( + chat_history: str | None = None, language: str | None = None +): chat_history_section = ( f""" @@ -172,7 +179,7 @@ NO CHAT HISTORY PROVIDED """ ) - + # Add language instruction if specified language_instruction = _build_language_instruction(language) diff --git a/surfsense_backend/app/routes/chats_routes.py b/surfsense_backend/app/routes/chats_routes.py index 0874d2611..e003dc260 100644 --- a/surfsense_backend/app/routes/chats_routes.py +++ b/surfsense_backend/app/routes/chats_routes.py @@ -6,7 +6,6 @@ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select from sqlalchemy.orm import selectinload - from app.db import Chat, SearchSpace, User, UserSearchSpacePreference, get_async_session from app.schemas import ( AISDKChatRequest, @@ -64,47 +63,53 @@ async def handle_chat_data( language_result = await session.execute( select(UserSearchSpacePreference) .options( - selectinload(UserSearchSpacePreference.search_space).selectinload(SearchSpace.llm_configs), + selectinload(UserSearchSpacePreference.search_space).selectinload( + SearchSpace.llm_configs + ), selectinload(UserSearchSpacePreference.long_context_llm), selectinload(UserSearchSpacePreference.fast_llm), - selectinload(UserSearchSpacePreference.strategic_llm) + selectinload(UserSearchSpacePreference.strategic_llm), ) .filter( - UserSearchSpacePreference.search_space_id == search_space_id, - UserSearchSpacePreference.user_id == user.id + UserSearchSpacePreference.search_space_id == search_space_id, + UserSearchSpacePreference.user_id == user.id, ) ) user_preference = language_result.scalars().first() # print("UserSearchSpacePreference:", user_preference) - + language = None - if user_preference and user_preference.search_space and user_preference.search_space.llm_configs: + if ( + user_preference + and user_preference.search_space + and user_preference.search_space.llm_configs + ): llm_configs = user_preference.search_space.llm_configs - - - for preferred_llm in [user_preference.fast_llm, user_preference.long_context_llm, user_preference.strategic_llm]: - if preferred_llm and getattr(preferred_llm, 'language', None): + + for preferred_llm in [ + user_preference.fast_llm, + user_preference.long_context_llm, + user_preference.strategic_llm, + ]: + if preferred_llm and getattr(preferred_llm, "language", None): language = preferred_llm.language break - - + if not language: first_llm_config = llm_configs[0] - language = getattr(first_llm_config, 'language', None) - - + language = getattr(first_llm_config, "language", None) + except HTTPException: raise HTTPException( status_code=403, detail="You don't have access to this search space" ) from None - + langchain_chat_history = [] for message in messages[:-1]: if message["role"] == "user": langchain_chat_history.append(HumanMessage(content=message["content"])) elif message["role"] == "assistant": langchain_chat_history.append(AIMessage(content=message["content"])) - response = StreamingResponse( stream_connector_search_results( @@ -117,7 +122,7 @@ async def handle_chat_data( langchain_chat_history, search_mode_str, document_ids_to_add_in_context, - language, + language, ) ) diff --git a/surfsense_backend/app/routes/llm_config_routes.py b/surfsense_backend/app/routes/llm_config_routes.py index 896f7be41..ec8ea5846 100644 --- a/surfsense_backend/app/routes/llm_config_routes.py +++ b/surfsense_backend/app/routes/llm_config_routes.py @@ -299,10 +299,10 @@ async def update_user_llm_preferences( # Validate that all provided LLM config IDs belong to the search space update_data = preferences.model_dump(exclude_unset=True) - + # Store language from configs to validate consistency languages = set() - + for _key, llm_config_id in update_data.items(): if llm_config_id is not None: # Verify the LLM config belongs to the search space @@ -318,10 +318,10 @@ async def update_user_llm_preferences( status_code=404, detail=f"LLM configuration {llm_config_id} not found in this search space", ) - + # Collect language for consistency check languages.add(llm_config.language) - + # Check if all selected LLM configs have the same language if len(languages) > 1: raise HTTPException( From 19c748bf9756a2c9efc4748670bff5598edac37a Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Mon, 13 Oct 2025 21:11:16 +0530 Subject: [PATCH 28/35] feat: implement MessageDetails component for displaying log details in a dialog --- .../[search_space_id]/logs/(manage)/page.tsx | 228 +++++++++--------- 1 file changed, 110 insertions(+), 118 deletions(-) diff --git a/surfsense_web/app/dashboard/[search_space_id]/logs/(manage)/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/logs/(manage)/page.tsx index fc2105274..1d7cae855 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/logs/(manage)/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/logs/(manage)/page.tsx @@ -122,6 +122,55 @@ const logStatusConfig = { FAILED: { icon: X, color: "text-red-600", bgColor: "bg-red-50" }, } as const; +function MessageDetails({ + message, + taskName, + metadata, + createdAt, + children, +}: { + message: string; + taskName?: string; + metadata?: any; + createdAt?: string; + children: React.ReactNode; +}) { + return ( + + {children} + +
+
+ Log details + {createdAt && ( +

+ {new Date(createdAt).toLocaleString()} +

+ )} +
+
+ Close +
+
+ +
+ {taskName && ( +
+ {taskName} +
+ )} + +
+ {message} +
+
+ + + + + ); +} + const columns: ColumnDef[] = [ { id: "select", @@ -219,19 +268,22 @@ const columns: ColumnDef[] = [ cell: ({ row }) => { const message = row.getValue("message") as string; const taskName = row.original.log_metadata?.task_name; + const createdAt = row.getValue("created_at") as string; return ( -
- {taskName && ( -
- {taskName} -
- )} -
- {message.length > 100 ? `${message.substring(0, 100)}...` : message} -
-
- ); + +
+ {taskName && ( +
+ {taskName} +
+ )} +
+ {message.length > 100 ? `${message.substring(0, 100)}...` : message} +
+
+ + ); }, size: 400, }, @@ -777,12 +829,6 @@ function LogsTable({ onRefresh: () => void; id: string; }) { - const [expandedRows, setExpandedRows] = useState>({}); - - const toggleRowExpanded = (rowId: string) => { - setExpandedRows((prev) => ({ ...prev, [rowId]: !prev[rowId] })); - }; - if (loading) { return ( - {table.getRowModel().rows?.length ? ( - table.getRowModel().rows.map((row: any, index: number) => ( - - {row.getVisibleCells().map((cell: any) => { - const isCreatedAt = cell.column.id === "created_at"; - const isMessage = cell.column.id === "message"; - const isExpanded = Boolean(expandedRows[row.id]); - - // If this is the Message column, render custom inline expandable content - if (isMessage) { - const message = row.getValue("message") as string; - const taskName = row.original.log_metadata?.task_name; - const createdAt = row.getValue("created_at") as string; - - return ( - - {/* Click the preview to toggle expand/collapse */} - - - ); - } - - // Default rendering for other columns - return ( - - {flexRender(cell.column.columnDef.cell, cell.getContext())} - - ); - })} - - )) - ) : ( - - - No logs found. - - - )} - + {table.getRowModel().rows?.length ? ( + table.getRowModel().rows.map((row: any, index: number) => ( + + {row.getVisibleCells().map((cell: any) => { + const isCreatedAt = cell.column.id === "created_at"; + const isMessage = cell.column.id === "message"; + return ( + + {flexRender(cell.column.columnDef.cell, cell.getContext())} + + ); + })} + + )) + ) : ( + + + No logs found. + + + )} + From ea509b7d57be89397339d0f46fae1dbc90093222 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Mon, 13 Oct 2025 21:20:33 +0530 Subject: [PATCH 29/35] refactor: clean up formatting and indentation in MessageDetails component --- .../[search_space_id]/logs/(manage)/page.tsx | 124 ++++++++++-------- 1 file changed, 67 insertions(+), 57 deletions(-) diff --git a/surfsense_web/app/dashboard/[search_space_id]/logs/(manage)/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/logs/(manage)/page.tsx index 1d7cae855..f89e4153f 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/logs/(manage)/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/logs/(manage)/page.tsx @@ -123,52 +123,52 @@ const logStatusConfig = { } as const; function MessageDetails({ - message, - taskName, - metadata, - createdAt, - children, + message, + taskName, + metadata, + createdAt, + children, }: { - message: string; - taskName?: string; - metadata?: any; - createdAt?: string; - children: React.ReactNode; + message: string; + taskName?: string; + metadata?: any; + createdAt?: string; + children: React.ReactNode; }) { - return ( - - {children} - -
-
- Log details - {createdAt && ( -

- {new Date(createdAt).toLocaleString()} -

- )} -
-
- Close -
-
+ return ( + + {children} + +
+
+ Log details + {createdAt && ( +

+ {new Date(createdAt).toLocaleString()} +

+ )} +
+
+ Close +
+
-
- {taskName && ( -
- {taskName} -
- )} +
+ {taskName && ( +
+ {taskName} +
+ )} -
- {message} -
-
+
+ {message} +
+
- - - - ); + + + + ); } const columns: ColumnDef[] = [ @@ -271,19 +271,27 @@ const columns: ColumnDef[] = [ const createdAt = row.getValue("created_at") as string; return ( - -
- {taskName && ( -
- {taskName} -
- )} -
- {message.length > 100 ? `${message.substring(0, 100)}...` : message} -
-
- - ); + +
+ {taskName && ( +
+ {taskName} +
+ )} +
+ {message.length > 100 ? `${message.substring(0, 100)}...` : message} +
+
+ + ); }, size: 400, }, @@ -954,19 +962,21 @@ function LogsTable({ {row.getVisibleCells().map((cell: any) => { const isCreatedAt = cell.column.id === "created_at"; const isMessage = cell.column.id === "message"; - return ( + return ( {flexRender(cell.column.columnDef.cell, cell.getContext())} ); - })} + })} )) ) : ( From 576dc192609430a89bdabf0c34f943fbf259adf8 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Mon, 13 Oct 2025 13:57:58 -0700 Subject: [PATCH 30/35] refactor: updated SearxNG connector migration and fixed validation for SearxNG host URL --- ...onnector_enum.py => 27_add_searxng_connector_enum.py} | 9 ++++----- surfsense_backend/app/services/connector_service.py | 4 ++-- surfsense_backend/app/utils/validators.py | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) rename surfsense_backend/alembic/versions/{26_add_searxng_connector_enum.py => 27_add_searxng_connector_enum.py} (91%) diff --git a/surfsense_backend/alembic/versions/26_add_searxng_connector_enum.py b/surfsense_backend/alembic/versions/27_add_searxng_connector_enum.py similarity index 91% rename from surfsense_backend/alembic/versions/26_add_searxng_connector_enum.py rename to surfsense_backend/alembic/versions/27_add_searxng_connector_enum.py index ca6ac2b55..7834a8671 100644 --- a/surfsense_backend/alembic/versions/26_add_searxng_connector_enum.py +++ b/surfsense_backend/alembic/versions/27_add_searxng_connector_enum.py @@ -1,7 +1,7 @@ """Add SearxNG connector enum value -Revision ID: 26 -Revises: 25 +Revision ID: 27 +Revises: 26 Create Date: 2025-01-18 00:00:00.000000 """ @@ -11,8 +11,8 @@ from collections.abc import Sequence from alembic import op # revision identifiers, used by Alembic. -revision: str = "26" -down_revision: str | None = "25" +revision: str = "27" +down_revision: str | None = "26" branch_labels: str | Sequence[str] | None = None depends_on: str | Sequence[str] | None = None @@ -39,4 +39,3 @@ def upgrade() -> None: def downgrade() -> None: """Downgrade not supported for enum edits.""" pass - diff --git a/surfsense_backend/app/services/connector_service.py b/surfsense_backend/app/services/connector_service.py index 0017fabaf..ca323f510 100644 --- a/surfsense_backend/app/services/connector_service.py +++ b/surfsense_backend/app/services/connector_service.py @@ -432,7 +432,7 @@ class ConnectorService: safesearch_clean = safesearch.strip() if safesearch_clean.isdigit(): safesearch_value = int(safesearch_clean) - elif isinstance(safesearch, (int, float)): + elif isinstance(safesearch, int | float): safesearch_value = int(safesearch) if safesearch_value is not None and not (0 <= safesearch_value <= 2): @@ -444,7 +444,7 @@ class ConnectorService: if isinstance(value, str): value = value.strip() return value or None - if isinstance(value, (list, tuple, set)): + if isinstance(value, list | tuple | set): cleaned = [str(item).strip() for item in value if str(item).strip()] return ",".join(cleaned) if cleaned else None return str(value) diff --git a/surfsense_backend/app/utils/validators.py b/surfsense_backend/app/utils/validators.py index 7e29860ba..1a3b30de6 100644 --- a/surfsense_backend/app/utils/validators.py +++ b/surfsense_backend/app/utils/validators.py @@ -412,7 +412,7 @@ def validate_connector_config( raise ValueError(f"Invalid email format for {connector_name} connector") def validate_url_field(key: str, connector_name: str) -> None: - if not validators.url(config.get(key, "")): + if not validators.url(config.get(key, "").strip(), simple_host=True): raise ValueError(f"Invalid base URL format for {connector_name} connector") def validate_list_field(key: str, field_name: str) -> None: From acd900c41db7daa12008cc776b2034939a1bdc2d Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Mon, 13 Oct 2025 14:01:55 -0700 Subject: [PATCH 31/35] ruff fix --- surfsense_backend/app/utils/validators.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/surfsense_backend/app/utils/validators.py b/surfsense_backend/app/utils/validators.py index 1a3b30de6..0d6bd5341 100644 --- a/surfsense_backend/app/utils/validators.py +++ b/surfsense_backend/app/utils/validators.py @@ -435,9 +435,7 @@ def validate_connector_config( "SEARXNG_VERIFY_SSL", ], "validators": { - "SEARXNG_HOST": lambda: validate_url_field( - "SEARXNG_HOST", "SearxNG" - ) + "SEARXNG_HOST": lambda: validate_url_field("SEARXNG_HOST", "SearxNG") }, }, "LINKUP_API": {"required": ["LINKUP_API_KEY"], "validators": {}}, From cbe7f18a2b5716fed8f9b12b63d65a54204e801b Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Mon, 13 Oct 2025 14:04:38 -0700 Subject: [PATCH 32/35] biome checks --- .../connectors/add/searxng/page.tsx | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/searxng/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/searxng/page.tsx index ca736a8cd..4fd406706 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/searxng/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/searxng/page.tsx @@ -258,7 +258,9 @@ export default function SearxngConnectorPage() { - Comma-separated list to target specific engines. + + Comma-separated list to target specific engines. + )} @@ -273,7 +275,9 @@ export default function SearxngConnectorPage() { - Comma-separated list of SearxNG categories. + + Comma-separated list of SearxNG categories. + )} @@ -308,8 +312,8 @@ export default function SearxngConnectorPage() { - Set 0, 1, or 2 to adjust SafeSearch filtering. Leave blank to use the instance - default. + Set 0, 1, or 2 to adjust SafeSearch filtering. Leave blank to use the + instance default. From ba5bb91a7be86ed1f4f729011a62dccf3f7aff82 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Mon, 13 Oct 2025 20:07:32 -0700 Subject: [PATCH 33/35] chore: removed chinese comments to pass ruff checks and updated migration nos --- surfsense_backend/alembic/env.py | 1 - ...=> 28_add_chinese_litellmprovider_enum.py} | 39 +++++++------------ surfsense_backend/app/db.py | 11 +++--- surfsense_backend/app/services/llm_service.py | 10 ++--- .../app/services/task_logging_service.py | 38 ++++++++---------- 5 files changed, 41 insertions(+), 58 deletions(-) rename surfsense_backend/alembic/versions/{26_add_chinese_llm_providers.py => 28_add_chinese_litellmprovider_enum.py} (76%) diff --git a/surfsense_backend/alembic/env.py b/surfsense_backend/alembic/env.py index ae58ac5f9..bd8c20356 100644 --- a/surfsense_backend/alembic/env.py +++ b/surfsense_backend/alembic/env.py @@ -21,7 +21,6 @@ from app.db import Base # Assuming your Base is defined in app.db config = context.config # Override SQLAlchemy URL from environment variables when available -# 如果环境变量提供了数据库连接字符串，则优先使用该配置 database_url = os.getenv("DATABASE_URL") if database_url: config.set_main_option("sqlalchemy.url", database_url) diff --git a/surfsense_backend/alembic/versions/26_add_chinese_llm_providers.py b/surfsense_backend/alembic/versions/28_add_chinese_litellmprovider_enum.py similarity index 76% rename from surfsense_backend/alembic/versions/26_add_chinese_llm_providers.py rename to surfsense_backend/alembic/versions/28_add_chinese_litellmprovider_enum.py index f08a96738..af36c2ea1 100644 --- a/surfsense_backend/alembic/versions/26_add_chinese_llm_providers.py +++ b/surfsense_backend/alembic/versions/28_add_chinese_litellmprovider_enum.py @@ -1,8 +1,7 @@ """Add Chinese LLM providers to LiteLLMProvider enum -添加国产 LLM 提供商到 LiteLLMProvider 枚举 -Revision ID: 26 -Revises: 25 +Revision ID: 28 +Revises: 27 """ from collections.abc import Sequence @@ -10,8 +9,8 @@ from collections.abc import Sequence from alembic import op # revision identifiers, used by Alembic. -revision: str = "26" -down_revision: str | None = "25" +revision: str = "28" +down_revision: str | None = "27" branch_labels: str | Sequence[str] | None = None depends_on: str | Sequence[str] | None = None @@ -19,17 +18,15 @@ depends_on: str | Sequence[str] | None = None def upgrade() -> None: """ Add Chinese LLM providers to LiteLLMProvider enum. - 添加国产 LLM 提供商到 LiteLLMProvider 枚举。 - + Adds support for: - DEEPSEEK: DeepSeek AI models - - ALIBABA_QWEN: Alibaba Qwen (通义千问) models - - MOONSHOT: Moonshot AI (月之暗面 Kimi) models - - ZHIPU: Zhipu AI (智谱 GLM) models + - ALIBABA_QWEN: Alibaba Qwen models + - MOONSHOT: Moonshot AI models + - ZHIPU: Zhipu AI models """ - + # Add DEEPSEEK to the enum if it doesn't already exist - # 如果不存在则添加 DEEPSEEK 到枚举 op.execute( """ DO $$ @@ -44,9 +41,8 @@ def upgrade() -> None: END$$; """ ) - + # Add ALIBABA_QWEN to the enum if it doesn't already exist - # 如果不存在则添加 ALIBABA_QWEN 到枚举 op.execute( """ DO $$ @@ -61,9 +57,8 @@ def upgrade() -> None: END$$; """ ) - + # Add MOONSHOT to the enum if it doesn't already exist - # 如果不存在则添加 MOONSHOT 到枚举 op.execute( """ DO $$ @@ -78,9 +73,8 @@ def upgrade() -> None: END$$; """ ) - + # Add ZHIPU to the enum if it doesn't already exist - # 如果不存在则添加 ZHIPU 到枚举 op.execute( """ DO $$ @@ -100,19 +94,14 @@ def upgrade() -> None: def downgrade() -> None: """ Remove Chinese LLM providers from LiteLLMProvider enum. - 从 LiteLLMProvider 枚举中移除国产 LLM 提供商。 - + Note: PostgreSQL doesn't support removing enum values directly. This would require recreating the enum type and updating all dependent objects. For safety, this downgrade is a no-op. - - 注意：PostgreSQL 不支持直接删除枚举值。 - 这需要重建枚举类型并更新所有依赖对象。 - 为了安全起见，此降级操作为空操作。 + """ # PostgreSQL doesn't support removing enum values directly # This would require a complex migration recreating the enum # PostgreSQL 不支持直接删除枚举值 # 这需要复杂的迁移来重建枚举 pass - diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py index 2e5bf059e..80920c8a9 100644 --- a/surfsense_backend/app/db.py +++ b/surfsense_backend/app/db.py @@ -83,6 +83,7 @@ class LiteLLMProvider(str, Enum): Enum for LLM providers supported by LiteLLM. LiteLLM 支持的 LLM 提供商枚举。 """ + OPENAI = "OPENAI" ANTHROPIC = "ANTHROPIC" GROQ = "GROQ" @@ -106,11 +107,11 @@ class LiteLLMProvider(str, Enum): ALEPH_ALPHA = "ALEPH_ALPHA" PETALS = "PETALS" COMETAPI = "COMETAPI" - # Chinese LLM Providers (OpenAI-compatible) / 国产 LLM 提供商（OpenAI 兼容） - DEEPSEEK = "DEEPSEEK" # DeepSeek - ALIBABA_QWEN = "ALIBABA_QWEN" # 阿里通义千问 - MOONSHOT = "MOONSHOT" # 月之暗面 (Kimi) - ZHIPU = "ZHIPU" # 智谱 AI (GLM) + # Chinese LLM Providers (OpenAI-compatible) + DEEPSEEK = "DEEPSEEK" + ALIBABA_QWEN = "ALIBABA_QWEN" + MOONSHOT = "MOONSHOT" + ZHIPU = "ZHIPU" CUSTOM = "CUSTOM" diff --git a/surfsense_backend/app/services/llm_service.py b/surfsense_backend/app/services/llm_service.py index b3491ddd7..e322eb401 100644 --- a/surfsense_backend/app/services/llm_service.py +++ b/surfsense_backend/app/services/llm_service.py @@ -99,11 +99,11 @@ async def get_user_llm_instance( "AZURE_OPENAI": "azure", "OPENROUTER": "openrouter", "COMETAPI": "cometapi", - # Chinese LLM providers (OpenAI-compatible) / 国产 LLM（OpenAI 兼容） - "DEEPSEEK": "openai", # DeepSeek uses OpenAI-compatible API - "ALIBABA_QWEN": "openai", # Qwen uses OpenAI-compatible API - "MOONSHOT": "openai", # Moonshot (Kimi) uses OpenAI-compatible API - "ZHIPU": "openai", # Zhipu (GLM) uses OpenAI-compatible API + # Chinese LLM providers (OpenAI-compatible) + "DEEPSEEK": "openai", # DeepSeek uses OpenAI-compatible API + "ALIBABA_QWEN": "openai", # Qwen uses OpenAI-compatible API + "MOONSHOT": "openai", # Moonshot (Kimi) uses OpenAI-compatible API + "ZHIPU": "openai", # Zhipu (GLM) uses OpenAI-compatible API # Add more mappings as needed } provider_prefix = provider_map.get( diff --git a/surfsense_backend/app/services/task_logging_service.py b/surfsense_backend/app/services/task_logging_service.py index 784525b90..6ba9d0432 100644 --- a/surfsense_backend/app/services/task_logging_service.py +++ b/surfsense_backend/app/services/task_logging_service.py @@ -1,3 +1,4 @@ +import contextlib import logging from datetime import datetime from typing import Any @@ -73,16 +74,14 @@ class TaskLoggingService: Returns: Log: The updated log entry """ - # Ensure session is in a valid state / 确保 session 处于有效状态 + # Ensure session is in a valid state if not self.session.is_active: await self.session.rollback() - - # Refresh log_entry to avoid expired state / 刷新 log_entry 避免过期状态 - try: + + # Refresh log_entry to avoid expired state + with contextlib.suppress(Exception): await self.session.refresh(log_entry) - except Exception: - pass - + # Update the existing log entry log_entry.status = LogStatus.SUCCESS log_entry.message = message @@ -124,17 +123,14 @@ class TaskLoggingService: Returns: Log: The updated log entry """ - # Ensure session is in a valid state / 确保 session 处于有效状态 + # Ensure session is in a valid state if not self.session.is_active: await self.session.rollback() - - # Refresh log_entry to avoid expired state / 刷新 log_entry 避免过期状态 - try: + + # Refresh log_entry to avoid expired state + with contextlib.suppress(Exception): await self.session.refresh(log_entry) - except Exception: - # If refresh fails, the object might be detached / 如果刷新失败，对象可能已分离 - pass - + # Update the existing log entry log_entry.status = LogStatus.FAILED log_entry.level = LogLevel.ERROR @@ -182,16 +178,14 @@ class TaskLoggingService: Returns: Log: The updated log entry """ - # Ensure session is in a valid state / 确保 session 处于有效状态 + # Ensure session is in a valid state if not self.session.is_active: await self.session.rollback() - - # Refresh log_entry to avoid expired state / 刷新 log_entry 避免过期状态 - try: + + # Refresh log_entry to avoid expired state + with contextlib.suppress(Exception): await self.session.refresh(log_entry) - except Exception: - pass - + log_entry.message = progress_message if progress_metadata: From a8a3ad2b42f34588077815b69ad282d02e8c9b48 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Mon, 13 Oct 2025 20:11:11 -0700 Subject: [PATCH 34/35] biome fixes --- .../settings/model-config-manager.tsx | 47 +++++++++---------- 1 file changed, 22 insertions(+), 25 deletions(-) diff --git a/surfsense_web/components/settings/model-config-manager.tsx b/surfsense_web/components/settings/model-config-manager.tsx index dad744379..5c4618baf 100644 --- a/surfsense_web/components/settings/model-config-manager.tsx +++ b/surfsense_web/components/settings/model-config-manager.tsx @@ -37,8 +37,8 @@ import { SelectTrigger, SelectValue, } from "@/components/ui/select"; -import { LLM_PROVIDERS } from "@/contracts/enums/llm-providers"; import { LANGUAGES } from "@/contracts/enums/languages"; +import { LLM_PROVIDERS } from "@/contracts/enums/llm-providers"; import { type CreateLLMConfig, type LLMConfig, useLLMConfigs } from "@/hooks/use-llm-configs"; import InferenceParamsEditor from "../inference-params-editor"; @@ -489,10 +489,7 @@ export function ModelConfigManager({ searchSpaceId }: ModelConfigManagerProps) {
- {formData.provider && ( @@ -548,23 +545,23 @@ export function ModelConfigManager({ searchSpaceId }: ModelConfigManagerProps) {
- - -
+ + +
@@ -580,7 +577,7 @@ export function ModelConfigManager({ searchSpaceId }: ModelConfigManagerProps) {