From 55845d68efc32afcf74b26f4133921c00b11bc6e Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 17:38:39 +0200 Subject: [PATCH 01/28] feat: return 3 suggestion options from vision autocomplete agent --- .../agents/autocomplete/autocomplete_agent.py | 91 ++++++++++++++----- 1 file changed, 67 insertions(+), 24 deletions(-) diff --git a/surfsense_backend/app/agents/autocomplete/autocomplete_agent.py b/surfsense_backend/app/agents/autocomplete/autocomplete_agent.py index c6a071b0f..36b5bc086 100644 --- a/surfsense_backend/app/agents/autocomplete/autocomplete_agent.py +++ b/surfsense_backend/app/agents/autocomplete/autocomplete_agent.py @@ -14,7 +14,9 @@ LLM call — the window title is used directly as the KB search query. from __future__ import annotations import asyncio +import json import logging +import re import uuid from collections.abc import AsyncGenerator from typing import Any @@ -61,13 +63,21 @@ Key behavior: - If the text area already has text, continue it naturally — typically just a sentence or two. Rules: -- Output ONLY the text to be inserted. No quotes, no explanations, no meta-commentary. - Be CONCISE. Prefer a single paragraph or a few sentences. Autocomplete is a quick assist, not a full draft. - Match the tone and formality of the surrounding context. - If the screen shows code, write code. If it shows a casual chat, be casual. If it shows a formal email, be formal. - Do NOT describe the screenshot or explain your reasoning. - Do NOT cite or reference documents explicitly — just let the knowledge inform your writing naturally. -- If you cannot determine what to write, output nothing. +- If you cannot determine what to write, output an empty JSON array: [] + +## Output Format + +You MUST provide exactly 3 different suggestion options. Each should be a distinct, plausible completion — vary the tone, detail level, or angle. + +Return your suggestions as a JSON array of exactly 3 strings. Output ONLY the JSON array, nothing else — no markdown fences, no explanation, no commentary. + +Example format: +["First suggestion text here.", "Second suggestion — a different take.", "Third option with another approach."] ## Filesystem Tools `ls`, `read_file`, `write_file`, `edit_file`, `glob`, `grep` @@ -264,6 +274,50 @@ async def create_autocomplete_agent( return agent, kb +# --------------------------------------------------------------------------- +# JSON suggestion parsing (robust fallback) +# --------------------------------------------------------------------------- + + +def _parse_suggestions(raw: str) -> list[str]: + """Extract a list of suggestion strings from the agent's output. + + Tries, in order: + 1. Direct ``json.loads`` + 2. Extract content between ```json ... ``` fences + 3. Find the first ``[`` … ``]`` span + Falls back to wrapping the raw text as a single suggestion. + """ + text = raw.strip() + if not text: + return [] + + for candidate in _json_candidates(text): + try: + parsed = json.loads(candidate) + if isinstance(parsed, list) and all(isinstance(s, str) for s in parsed): + return [s for s in parsed if s.strip()] + except (json.JSONDecodeError, ValueError): + continue + + return [text] + + +def _json_candidates(text: str) -> list[str]: + """Yield candidate JSON strings from raw text.""" + candidates = [text] + + fence = re.search(r"```(?:json)?\s*\n?(.*?)```", text, re.DOTALL) + if fence: + candidates.append(fence.group(1).strip()) + + bracket = re.search(r"\[.*]", text, re.DOTALL) + if bracket: + candidates.append(bracket.group(0)) + + return candidates + + # --------------------------------------------------------------------------- # Streaming helper # --------------------------------------------------------------------------- @@ -285,7 +339,7 @@ async def stream_autocomplete_agent( thread_id = uuid.uuid4().hex config = {"configurable": {"thread_id": thread_id}} - current_text_id: str | None = None + text_buffer: list[str] = [] active_tool_depth = 0 thinking_step_counter = 0 tool_step_ids: dict[str, str] = {} @@ -315,14 +369,12 @@ async def stream_autocomplete_agent( if emit_message_start: yield streaming_service.format_message_start() - # Emit an initial "Generating completion" step so the UI immediately - # shows activity once the agent starts its first LLM call. gen_step_id = next_thinking_step_id() last_active_step_id = gen_step_id - step_titles[gen_step_id] = "Generating completion" + step_titles[gen_step_id] = "Generating suggestions" yield streaming_service.format_thinking_step( step_id=gen_step_id, - title="Generating completion", + title="Generating suggestions", status="in_progress", ) @@ -341,15 +393,7 @@ async def stream_autocomplete_agent( if chunk and hasattr(chunk, "content"): content = chunk.content if content and isinstance(content, str): - if current_text_id is None: - step_event = complete_current_step() - if step_event: - yield step_event - current_text_id = streaming_service.generate_text_id() - yield streaming_service.format_text_start(current_text_id) - yield streaming_service.format_text_delta( - current_text_id, content - ) + text_buffer.append(content) elif event_type == "on_tool_start": active_tool_depth += 1 @@ -357,10 +401,6 @@ async def stream_autocomplete_agent( run_id = event.get("run_id", "") tool_input = event.get("data", {}).get("input", {}) - if current_text_id is not None: - yield streaming_service.format_text_end(current_text_id) - current_text_id = None - step_event = complete_current_step() if step_event: yield step_event @@ -393,19 +433,22 @@ async def stream_autocomplete_agent( if last_active_step_id == step_id: last_active_step_id = None - if current_text_id is not None: - yield streaming_service.format_text_end(current_text_id) step_event = complete_current_step() if step_event: yield step_event + raw_text = "".join(text_buffer) + suggestions = _parse_suggestions(raw_text) + + yield streaming_service.format_data( + "suggestions", {"options": suggestions} + ) + yield streaming_service.format_finish() yield streaming_service.format_done() except Exception as e: logger.error(f"Autocomplete agent streaming error: {e}", exc_info=True) - if current_text_id is not None: - yield streaming_service.format_text_end(current_text_id) yield streaming_service.format_error("Autocomplete failed. Please try again.") yield streaming_service.format_done() From 2602248e7a552a19be0ac4e005f2c8183e18995a Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 17:43:40 +0200 Subject: [PATCH 02/28] feat: handle multi-option suggestions in suggestion page UI --- surfsense_web/app/desktop/suggestion/page.tsx | 68 ++++++++++++++----- 1 file changed, 50 insertions(+), 18 deletions(-) diff --git a/surfsense_web/app/desktop/suggestion/page.tsx b/surfsense_web/app/desktop/suggestion/page.tsx index 8d9095320..e98da9a1c 100644 --- a/surfsense_web/app/desktop/suggestion/page.tsx +++ b/surfsense_web/app/desktop/suggestion/page.tsx @@ -14,6 +14,10 @@ type SSEEvent = | { type: "data-thinking-step"; data: { id: string; title: string; status: string; items: string[] }; + } + | { + type: "data-suggestions"; + data: { options: string[] }; }; interface AgentStep { @@ -70,10 +74,11 @@ function StepIcon({ status }: { status: string }) { export default function SuggestionPage() { const api = useElectronAPI(); - const [suggestion, setSuggestion] = useState(""); + const [options, setOptions] = useState([]); const [isLoading, setIsLoading] = useState(true); const [error, setError] = useState(null); const [steps, setSteps] = useState([]); + const [expandedOption, setExpandedOption] = useState(null); const abortRef = useRef(null); const isDesktop = !!api?.onAutocompleteContext; @@ -99,9 +104,10 @@ export default function SuggestionPage() { abortRef.current = controller; setIsLoading(true); - setSuggestion(""); + setOptions([]); setError(null); setSteps([]); + setExpandedOption(null); let token = getBearerToken(); if (!token) { @@ -165,8 +171,8 @@ export default function SuggestionPage() { try { const parsed: SSEEvent = JSON.parse(data); - if (parsed.type === "text-delta") { - setSuggestion((prev) => prev + parsed.delta); + if (parsed.type === "data-suggestions") { + setOptions(parsed.data.options); } else if (parsed.type === "error") { setError(friendlyError(parsed.errorText)); } else if (parsed.type === "data-thinking-step") { @@ -226,7 +232,7 @@ export default function SuggestionPage() { ); } - const showLoading = isLoading && !suggestion; + const showLoading = isLoading && options.length === 0; if (showLoading) { return ( @@ -258,29 +264,55 @@ export default function SuggestionPage() { ); } - const handleAccept = () => { - if (suggestion) { - api?.acceptSuggestion?.(suggestion); - } + const handleSelect = (text: string) => { + api?.acceptSuggestion?.(text); }; const handleDismiss = () => { api?.dismissSuggestion?.(); }; - if (!suggestion) return null; + const TRUNCATE_LENGTH = 120; + + if (options.length === 0) return null; return (
-

{suggestion}

+
+ {options.map((option, index) => { + const isExpanded = expandedOption === index; + const needsTruncation = option.length > TRUNCATE_LENGTH; + const displayText = + needsTruncation && !isExpanded + ? option.slice(0, TRUNCATE_LENGTH) + "…" + : option; + + return ( + + )} + + ); + })} +
- )} - +
); })}
From 70807cccd1cb55f47ea136c1c5cf8399fd5eec63 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 18:22:16 +0200 Subject: [PATCH 06/28] fix: hide scrollbar during streaming to prevent UI flicker --- surfsense_web/app/desktop/suggestion/suggestion.css | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/surfsense_web/app/desktop/suggestion/suggestion.css b/surfsense_web/app/desktop/suggestion/suggestion.css index e0c56857d..f5471cf37 100644 --- a/surfsense_web/app/desktop/suggestion/suggestion.css +++ b/surfsense_web/app/desktop/suggestion/suggestion.css @@ -127,6 +127,10 @@ body:has(.suggestion-body) { max-height: 340px; } +.agent-activity::-webkit-scrollbar { + display: none; +} + .activity-initial { display: flex; align-items: center; From 5439b3991b6f61fd547ae8ef1d2020e04f6ec1a3 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 18:25:06 +0200 Subject: [PATCH 07/28] fix: auto-dismiss overlay when no suggestions are available --- surfsense_web/app/desktop/suggestion/page.tsx | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/surfsense_web/app/desktop/suggestion/page.tsx b/surfsense_web/app/desktop/suggestion/page.tsx index af1c50be0..e458f6615 100644 --- a/surfsense_web/app/desktop/suggestion/page.tsx +++ b/surfsense_web/app/desktop/suggestion/page.tsx @@ -97,6 +97,14 @@ export default function SuggestionPage() { return () => clearTimeout(timer); }, [error, api]); + useEffect(() => { + if (isLoading || error || options.length > 0) return; + const timer = setTimeout(() => { + api?.dismissSuggestion?.(); + }, AUTO_DISMISS_MS); + return () => clearTimeout(timer); + }, [isLoading, error, options, api]); + const fetchSuggestion = useCallback( async (screenshot: string, searchSpaceId: string, appName?: string, windowTitle?: string) => { abortRef.current?.abort(); From 879945eeae05d22845a2dd682a6e3eb1cbeb578d Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 18:49:04 +0200 Subject: [PATCH 08/28] Add VisionProvider enum, VisionLLMConfig table, and vision RBAC permissions --- surfsense_backend/app/db.py | 77 ++++++++++++++++++++++++++++++++++++- 1 file changed, 76 insertions(+), 1 deletion(-) diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py index 6e9553307..4689313f7 100644 --- a/surfsense_backend/app/db.py +++ b/surfsense_backend/app/db.py @@ -260,6 +260,24 @@ class ImageGenProvider(StrEnum): NSCALE = "NSCALE" +class VisionProvider(StrEnum): + OPENAI = "OPENAI" + ANTHROPIC = "ANTHROPIC" + GOOGLE = "GOOGLE" + AZURE_OPENAI = "AZURE_OPENAI" + VERTEX_AI = "VERTEX_AI" + BEDROCK = "BEDROCK" + XAI = "XAI" + OPENROUTER = "OPENROUTER" + OLLAMA = "OLLAMA" + GROQ = "GROQ" + TOGETHER_AI = "TOGETHER_AI" + FIREWORKS_AI = "FIREWORKS_AI" + DEEPSEEK = "DEEPSEEK" + MISTRAL = "MISTRAL" + CUSTOM = "CUSTOM" + + class LogLevel(StrEnum): DEBUG = "DEBUG" INFO = "INFO" @@ -377,6 +395,11 @@ class Permission(StrEnum): IMAGE_GENERATIONS_READ = "image_generations:read" IMAGE_GENERATIONS_DELETE = "image_generations:delete" + # Vision LLM Configs + VISION_CONFIGS_CREATE = "vision_configs:create" + VISION_CONFIGS_READ = "vision_configs:read" + VISION_CONFIGS_DELETE = "vision_configs:delete" + # Connectors CONNECTORS_CREATE = "connectors:create" CONNECTORS_READ = "connectors:read" @@ -445,6 +468,9 @@ DEFAULT_ROLE_PERMISSIONS = { # Image Generations (create and read, no delete) Permission.IMAGE_GENERATIONS_CREATE.value, Permission.IMAGE_GENERATIONS_READ.value, + # Vision Configs (create and read, no delete) + Permission.VISION_CONFIGS_CREATE.value, + Permission.VISION_CONFIGS_READ.value, # Connectors (no delete) Permission.CONNECTORS_CREATE.value, Permission.CONNECTORS_READ.value, @@ -478,6 +504,8 @@ DEFAULT_ROLE_PERMISSIONS = { Permission.VIDEO_PRESENTATIONS_READ.value, # Image Generations (read only) Permission.IMAGE_GENERATIONS_READ.value, + # Vision Configs (read only) + Permission.VISION_CONFIGS_READ.value, # Connectors (read only) Permission.CONNECTORS_READ.value, # Logs (read only) @@ -1263,6 +1291,35 @@ class ImageGenerationConfig(BaseModel, TimestampMixin): user = relationship("User", back_populates="image_generation_configs") +class VisionLLMConfig(BaseModel, TimestampMixin): + __tablename__ = "vision_llm_configs" + + name = Column(String(100), nullable=False, index=True) + description = Column(String(500), nullable=True) + + provider = Column(SQLAlchemyEnum(VisionProvider), nullable=False) + custom_provider = Column(String(100), nullable=True) + model_name = Column(String(100), nullable=False) + + api_key = Column(String, nullable=False) + api_base = Column(String(500), nullable=True) + api_version = Column(String(50), nullable=True) + + litellm_params = Column(JSON, nullable=True, default={}) + + search_space_id = Column( + Integer, ForeignKey("searchspaces.id", ondelete="CASCADE"), nullable=False + ) + search_space = relationship( + "SearchSpace", back_populates="vision_llm_configs" + ) + + user_id = Column( + UUID(as_uuid=True), ForeignKey("user.id", ondelete="CASCADE"), nullable=False + ) + user = relationship("User", back_populates="vision_llm_configs") + + class ImageGeneration(BaseModel, TimestampMixin): """ Stores image generation requests and results using litellm.aimage_generation(). @@ -1351,7 +1408,7 @@ class SearchSpace(BaseModel, TimestampMixin): image_generation_config_id = Column( Integer, nullable=True, default=0 ) # For image generation, defaults to Auto mode - vision_llm_id = Column( + vision_llm_config_id = Column( Integer, nullable=True, default=0 ) # For vision/screenshot analysis, defaults to Auto mode @@ -1432,6 +1489,12 @@ class SearchSpace(BaseModel, TimestampMixin): order_by="ImageGenerationConfig.id", cascade="all, delete-orphan", ) + vision_llm_configs = relationship( + "VisionLLMConfig", + back_populates="search_space", + order_by="VisionLLMConfig.id", + cascade="all, delete-orphan", + ) # RBAC relationships roles = relationship( @@ -1961,6 +2024,12 @@ if config.AUTH_TYPE == "GOOGLE": passive_deletes=True, ) + vision_llm_configs = relationship( + "VisionLLMConfig", + back_populates="user", + passive_deletes=True, + ) + # User memories for personalized AI responses memories = relationship( "UserMemory", @@ -2075,6 +2144,12 @@ else: passive_deletes=True, ) + vision_llm_configs = relationship( + "VisionLLMConfig", + back_populates="user", + passive_deletes=True, + ) + # User memories for personalized AI responses memories = relationship( "UserMemory", From 32a3356f55cc3c9003df88e1d529eca5cf8babbc Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 18:50:51 +0200 Subject: [PATCH 09/28] Add migration 120: vision_llm_configs table and column rename --- .../120_add_vision_llm_configs_table.py | 190 ++++++++++++++++++ 1 file changed, 190 insertions(+) create mode 100644 surfsense_backend/alembic/versions/120_add_vision_llm_configs_table.py diff --git a/surfsense_backend/alembic/versions/120_add_vision_llm_configs_table.py b/surfsense_backend/alembic/versions/120_add_vision_llm_configs_table.py new file mode 100644 index 000000000..c0c915388 --- /dev/null +++ b/surfsense_backend/alembic/versions/120_add_vision_llm_configs_table.py @@ -0,0 +1,190 @@ +"""Add vision LLM configs table and rename preference column + +Revision ID: 120 +Revises: 119 + +Changes: +1. Create visionprovider enum type +2. Create vision_llm_configs table +3. Rename vision_llm_id -> vision_llm_config_id on searchspaces +4. Add vision config permissions to existing system roles +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import ENUM as PG_ENUM, UUID + +from alembic import op + +revision: str = "120" +down_revision: str | None = "119" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +VISION_PROVIDER_VALUES = ( + "OPENAI", + "ANTHROPIC", + "GOOGLE", + "AZURE_OPENAI", + "VERTEX_AI", + "BEDROCK", + "XAI", + "OPENROUTER", + "OLLAMA", + "GROQ", + "TOGETHER_AI", + "FIREWORKS_AI", + "DEEPSEEK", + "MISTRAL", + "CUSTOM", +) + + +def upgrade() -> None: + connection = op.get_bind() + + # 1. Create visionprovider enum + connection.execute( + sa.text( + """ + DO $$ + BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'visionprovider') THEN + CREATE TYPE visionprovider AS ENUM ( + 'OPENAI', 'ANTHROPIC', 'GOOGLE', 'AZURE_OPENAI', 'VERTEX_AI', + 'BEDROCK', 'XAI', 'OPENROUTER', 'OLLAMA', 'GROQ', + 'TOGETHER_AI', 'FIREWORKS_AI', 'DEEPSEEK', 'MISTRAL', 'CUSTOM' + ); + END IF; + END + $$; + """ + ) + ) + + # 2. Create vision_llm_configs table + result = connection.execute( + sa.text( + "SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = 'vision_llm_configs')" + ) + ) + if not result.scalar(): + op.create_table( + "vision_llm_configs", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("name", sa.String(100), nullable=False), + sa.Column("description", sa.String(500), nullable=True), + sa.Column( + "provider", + PG_ENUM(*VISION_PROVIDER_VALUES, name="visionprovider", create_type=False), + nullable=False, + ), + sa.Column("custom_provider", sa.String(100), nullable=True), + sa.Column("model_name", sa.String(100), nullable=False), + sa.Column("api_key", sa.String(), nullable=False), + sa.Column("api_base", sa.String(500), nullable=True), + sa.Column("api_version", sa.String(50), nullable=True), + sa.Column("litellm_params", sa.JSON(), nullable=True), + sa.Column("search_space_id", sa.Integer(), nullable=False), + sa.Column("user_id", UUID(as_uuid=True), nullable=False), + sa.Column( + "created_at", + sa.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.PrimaryKeyConstraint("id"), + sa.ForeignKeyConstraint( + ["search_space_id"], ["searchspaces.id"], ondelete="CASCADE" + ), + sa.ForeignKeyConstraint( + ["user_id"], ["user.id"], ondelete="CASCADE" + ), + ) + op.execute( + "CREATE INDEX IF NOT EXISTS ix_vision_llm_configs_name " + "ON vision_llm_configs (name)" + ) + op.execute( + "CREATE INDEX IF NOT EXISTS ix_vision_llm_configs_search_space_id " + "ON vision_llm_configs (search_space_id)" + ) + + # 3. Rename vision_llm_id -> vision_llm_config_id on searchspaces + existing_columns = [ + col["name"] for col in sa.inspect(connection).get_columns("searchspaces") + ] + if "vision_llm_id" in existing_columns and "vision_llm_config_id" not in existing_columns: + op.alter_column("searchspaces", "vision_llm_id", new_column_name="vision_llm_config_id") + elif "vision_llm_config_id" not in existing_columns: + op.add_column( + "searchspaces", + sa.Column("vision_llm_config_id", sa.Integer(), nullable=True, server_default="0"), + ) + + # 4. Add vision config permissions to existing system roles + connection.execute( + sa.text( + """ + UPDATE search_space_roles + SET permissions = array_cat( + permissions, + ARRAY['vision_configs:create', 'vision_configs:read'] + ) + WHERE is_system_role = true + AND name = 'Editor' + AND NOT ('vision_configs:create' = ANY(permissions)) + """ + ) + ) + connection.execute( + sa.text( + """ + UPDATE search_space_roles + SET permissions = array_cat( + permissions, + ARRAY['vision_configs:read'] + ) + WHERE is_system_role = true + AND name = 'Viewer' + AND NOT ('vision_configs:read' = ANY(permissions)) + """ + ) + ) + + +def downgrade() -> None: + connection = op.get_bind() + + # Remove permissions + connection.execute( + sa.text( + """ + UPDATE search_space_roles + SET permissions = array_remove( + array_remove( + array_remove(permissions, 'vision_configs:create'), + 'vision_configs:read' + ), + 'vision_configs:delete' + ) + WHERE is_system_role = true + """ + ) + ) + + # Rename column back + existing_columns = [ + col["name"] for col in sa.inspect(connection).get_columns("searchspaces") + ] + if "vision_llm_config_id" in existing_columns: + op.alter_column("searchspaces", "vision_llm_config_id", new_column_name="vision_llm_id") + + # Drop table and enum + op.execute("DROP INDEX IF EXISTS ix_vision_llm_configs_search_space_id") + op.execute("DROP INDEX IF EXISTS ix_vision_llm_configs_name") + op.execute("DROP TABLE IF EXISTS vision_llm_configs") + op.execute("DROP TYPE IF EXISTS visionprovider") From ecfcc6101112bcb59d95469785d9acb0b656319a Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 18:52:37 +0200 Subject: [PATCH 10/28] Add VisionLLMConfig Pydantic schemas --- surfsense_backend/app/schemas/__init__.py | 13 ++++ surfsense_backend/app/schemas/vision_llm.py | 75 +++++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100644 surfsense_backend/app/schemas/vision_llm.py diff --git a/surfsense_backend/app/schemas/__init__.py b/surfsense_backend/app/schemas/__init__.py index b94a30c19..fdf34672b 100644 --- a/surfsense_backend/app/schemas/__init__.py +++ b/surfsense_backend/app/schemas/__init__.py @@ -125,6 +125,13 @@ from .video_presentations import ( VideoPresentationRead, VideoPresentationUpdate, ) +from .vision_llm import ( + GlobalVisionLLMConfigRead, + VisionLLMConfigCreate, + VisionLLMConfigPublic, + VisionLLMConfigRead, + VisionLLMConfigUpdate, +) __all__ = [ # Folder schemas @@ -163,6 +170,8 @@ __all__ = [ "FolderUpdate", "GlobalImageGenConfigRead", "GlobalNewLLMConfigRead", + # Vision LLM Config schemas + "GlobalVisionLLMConfigRead", "GoogleDriveIndexRequest", "GoogleDriveIndexingOptions", # Base schemas @@ -264,4 +273,8 @@ __all__ = [ "VideoPresentationCreate", "VideoPresentationRead", "VideoPresentationUpdate", + "VisionLLMConfigCreate", + "VisionLLMConfigPublic", + "VisionLLMConfigRead", + "VisionLLMConfigUpdate", ] diff --git a/surfsense_backend/app/schemas/vision_llm.py b/surfsense_backend/app/schemas/vision_llm.py new file mode 100644 index 000000000..ab2e609dc --- /dev/null +++ b/surfsense_backend/app/schemas/vision_llm.py @@ -0,0 +1,75 @@ +import uuid +from datetime import datetime +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field + +from app.db import VisionProvider + + +class VisionLLMConfigBase(BaseModel): + name: str = Field(..., max_length=100) + description: str | None = Field(None, max_length=500) + provider: VisionProvider = Field(...) + custom_provider: str | None = Field(None, max_length=100) + model_name: str = Field(..., max_length=100) + api_key: str = Field(...) + api_base: str | None = Field(None, max_length=500) + api_version: str | None = Field(None, max_length=50) + litellm_params: dict[str, Any] | None = Field(default=None) + + +class VisionLLMConfigCreate(VisionLLMConfigBase): + search_space_id: int = Field(...) + + +class VisionLLMConfigUpdate(BaseModel): + name: str | None = Field(None, max_length=100) + description: str | None = Field(None, max_length=500) + provider: VisionProvider | None = None + custom_provider: str | None = Field(None, max_length=100) + model_name: str | None = Field(None, max_length=100) + api_key: str | None = None + api_base: str | None = Field(None, max_length=500) + api_version: str | None = Field(None, max_length=50) + litellm_params: dict[str, Any] | None = None + + +class VisionLLMConfigRead(VisionLLMConfigBase): + id: int + created_at: datetime + search_space_id: int + user_id: uuid.UUID + + model_config = ConfigDict(from_attributes=True) + + +class VisionLLMConfigPublic(BaseModel): + id: int + name: str + description: str | None = None + provider: VisionProvider + custom_provider: str | None = None + model_name: str + api_base: str | None = None + api_version: str | None = None + litellm_params: dict[str, Any] | None = None + created_at: datetime + search_space_id: int + user_id: uuid.UUID + + model_config = ConfigDict(from_attributes=True) + + +class GlobalVisionLLMConfigRead(BaseModel): + id: int = Field(...) + name: str + description: str | None = None + provider: str + custom_provider: str | None = None + model_name: str + api_base: str | None = None + api_version: str | None = None + litellm_params: dict[str, Any] | None = None + is_global: bool = True + is_auto_mode: bool = False From 362cd3590c2c68b9bed4a461f00af839e6130c73 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 18:59:33 +0200 Subject: [PATCH 11/28] Add VisionLLMRouterService for Auto mode routing --- .../app/services/vision_llm_router_service.py | 193 ++++++++++++++++++ 1 file changed, 193 insertions(+) create mode 100644 surfsense_backend/app/services/vision_llm_router_service.py diff --git a/surfsense_backend/app/services/vision_llm_router_service.py b/surfsense_backend/app/services/vision_llm_router_service.py new file mode 100644 index 000000000..0d782ab2b --- /dev/null +++ b/surfsense_backend/app/services/vision_llm_router_service.py @@ -0,0 +1,193 @@ +import logging +from typing import Any + +from litellm import Router + +logger = logging.getLogger(__name__) + +VISION_AUTO_MODE_ID = 0 + +VISION_PROVIDER_MAP = { + "OPENAI": "openai", + "ANTHROPIC": "anthropic", + "GOOGLE": "gemini", + "AZURE_OPENAI": "azure", + "VERTEX_AI": "vertex_ai", + "BEDROCK": "bedrock", + "XAI": "xai", + "OPENROUTER": "openrouter", + "OLLAMA": "ollama_chat", + "GROQ": "groq", + "TOGETHER_AI": "together_ai", + "FIREWORKS_AI": "fireworks_ai", + "DEEPSEEK": "openai", + "MISTRAL": "mistral", + "CUSTOM": "custom", +} + + +class VisionLLMRouterService: + _instance = None + _router: Router | None = None + _model_list: list[dict] = [] + _router_settings: dict = {} + _initialized: bool = False + + def __new__(cls): + if cls._instance is None: + cls._instance = super().__new__(cls) + return cls._instance + + @classmethod + def get_instance(cls) -> "VisionLLMRouterService": + if cls._instance is None: + cls._instance = cls() + return cls._instance + + @classmethod + def initialize( + cls, + global_configs: list[dict], + router_settings: dict | None = None, + ) -> None: + instance = cls.get_instance() + + if instance._initialized: + logger.debug("Vision LLM Router already initialized, skipping") + return + + model_list = [] + for config in global_configs: + deployment = cls._config_to_deployment(config) + if deployment: + model_list.append(deployment) + + if not model_list: + logger.warning( + "No valid vision LLM configs found for router initialization" + ) + return + + instance._model_list = model_list + instance._router_settings = router_settings or {} + + default_settings = { + "routing_strategy": "usage-based-routing", + "num_retries": 3, + "allowed_fails": 3, + "cooldown_time": 60, + "retry_after": 5, + } + + final_settings = {**default_settings, **instance._router_settings} + + try: + instance._router = Router( + model_list=model_list, + routing_strategy=final_settings.get( + "routing_strategy", "usage-based-routing" + ), + num_retries=final_settings.get("num_retries", 3), + allowed_fails=final_settings.get("allowed_fails", 3), + cooldown_time=final_settings.get("cooldown_time", 60), + set_verbose=False, + ) + instance._initialized = True + logger.info( + "Vision LLM Router initialized with %d deployments, strategy: %s", + len(model_list), + final_settings.get("routing_strategy"), + ) + except Exception as e: + logger.error(f"Failed to initialize Vision LLM Router: {e}") + instance._router = None + + @classmethod + def _config_to_deployment(cls, config: dict) -> dict | None: + try: + if not config.get("model_name") or not config.get("api_key"): + return None + + if config.get("custom_provider"): + model_string = f"{config['custom_provider']}/{config['model_name']}" + else: + provider = config.get("provider", "").upper() + provider_prefix = VISION_PROVIDER_MAP.get(provider, provider.lower()) + model_string = f"{provider_prefix}/{config['model_name']}" + + litellm_params: dict[str, Any] = { + "model": model_string, + "api_key": config.get("api_key"), + } + + if config.get("api_base"): + litellm_params["api_base"] = config["api_base"] + + if config.get("api_version"): + litellm_params["api_version"] = config["api_version"] + + if config.get("litellm_params"): + litellm_params.update(config["litellm_params"]) + + deployment: dict[str, Any] = { + "model_name": "auto", + "litellm_params": litellm_params, + } + + if config.get("rpm"): + deployment["rpm"] = config["rpm"] + if config.get("tpm"): + deployment["tpm"] = config["tpm"] + + return deployment + + except Exception as e: + logger.warning(f"Failed to convert vision config to deployment: {e}") + return None + + @classmethod + def get_router(cls) -> Router | None: + instance = cls.get_instance() + return instance._router + + @classmethod + def is_initialized(cls) -> bool: + instance = cls.get_instance() + return instance._initialized and instance._router is not None + + @classmethod + def get_model_count(cls) -> int: + instance = cls.get_instance() + return len(instance._model_list) + + +def is_vision_auto_mode(config_id: int | None) -> bool: + return config_id == VISION_AUTO_MODE_ID + + +def build_vision_model_string( + provider: str, model_name: str, custom_provider: str | None +) -> str: + if custom_provider: + return f"{custom_provider}/{model_name}" + prefix = VISION_PROVIDER_MAP.get(provider.upper(), provider.lower()) + return f"{prefix}/{model_name}" + + +def get_global_vision_llm_config(config_id: int) -> dict | None: + from app.config import config + + if config_id == VISION_AUTO_MODE_ID: + return { + "id": VISION_AUTO_MODE_ID, + "name": "Auto (Fastest)", + "provider": "AUTO", + "model_name": "auto", + "is_auto_mode": True, + } + if config_id > 0: + return None + for cfg in config.GLOBAL_VISION_LLM_CONFIGS: + if cfg.get("id") == config_id: + return cfg + return None From 7448f27ee01ce1619ff02f61ab3b12b6f085e60e Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 19:02:18 +0200 Subject: [PATCH 12/28] Add vision LLM config loading and router initialization to Config --- surfsense_backend/app/config/__init__.py | 67 ++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index 7c4baf923..4c49a4f8b 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -102,6 +102,44 @@ def load_global_image_gen_configs(): return [] +def load_global_vision_llm_configs(): + global_config_file = BASE_DIR / "app" / "config" / "global_llm_config.yaml" + + if not global_config_file.exists(): + return [] + + try: + with open(global_config_file, encoding="utf-8") as f: + data = yaml.safe_load(f) + return data.get("global_vision_llm_configs", []) + except Exception as e: + print(f"Warning: Failed to load global vision LLM configs: {e}") + return [] + + +def load_vision_llm_router_settings(): + default_settings = { + "routing_strategy": "usage-based-routing", + "num_retries": 3, + "allowed_fails": 3, + "cooldown_time": 60, + } + + global_config_file = BASE_DIR / "app" / "config" / "global_llm_config.yaml" + + if not global_config_file.exists(): + return default_settings + + try: + with open(global_config_file, encoding="utf-8") as f: + data = yaml.safe_load(f) + settings = data.get("vision_llm_router_settings", {}) + return {**default_settings, **settings} + except Exception as e: + print(f"Warning: Failed to load vision LLM router settings: {e}") + return default_settings + + def load_image_gen_router_settings(): """ Load router settings for image generation Auto mode from YAML file. @@ -182,6 +220,29 @@ def initialize_image_gen_router(): print(f"Warning: Failed to initialize Image Generation Router: {e}") +def initialize_vision_llm_router(): + vision_configs = load_global_vision_llm_configs() + router_settings = load_vision_llm_router_settings() + + if not vision_configs: + print( + "Info: No global vision LLM configs found, " + "Vision LLM Auto mode will not be available" + ) + return + + try: + from app.services.vision_llm_router_service import VisionLLMRouterService + + VisionLLMRouterService.initialize(vision_configs, router_settings) + print( + f"Info: Vision LLM Router initialized with {len(vision_configs)} models " + f"(strategy: {router_settings.get('routing_strategy', 'usage-based-routing')})" + ) + except Exception as e: + print(f"Warning: Failed to initialize Vision LLM Router: {e}") + + class Config: # Check if ffmpeg is installed if not is_ffmpeg_installed(): @@ -335,6 +396,12 @@ class Config: # Router settings for Image Generation Auto mode IMAGE_GEN_ROUTER_SETTINGS = load_image_gen_router_settings() + # Global Vision LLM Configurations (optional) + GLOBAL_VISION_LLM_CONFIGS = load_global_vision_llm_configs() + + # Router settings for Vision LLM Auto mode + VISION_LLM_ROUTER_SETTINGS = load_vision_llm_router_settings() + # Chonkie Configuration | Edit this to your needs EMBEDDING_MODEL = os.getenv("EMBEDDING_MODEL") # Azure OpenAI credentials from environment variables From bdbc4ce4a1cc4b6d4426e7537a7818ec19a5bfb3 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 19:04:03 +0200 Subject: [PATCH 13/28] Add vision LLM config CRUD and global configs routes --- .../app/routes/vision_llm_routes.py | 267 ++++++++++++++++++ 1 file changed, 267 insertions(+) create mode 100644 surfsense_backend/app/routes/vision_llm_routes.py diff --git a/surfsense_backend/app/routes/vision_llm_routes.py b/surfsense_backend/app/routes/vision_llm_routes.py new file mode 100644 index 000000000..29d1a2757 --- /dev/null +++ b/surfsense_backend/app/routes/vision_llm_routes.py @@ -0,0 +1,267 @@ +import logging + +from fastapi import APIRouter, Depends, HTTPException +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.config import config +from app.db import ( + Permission, + User, + VisionLLMConfig, + get_async_session, +) +from app.schemas import ( + GlobalVisionLLMConfigRead, + VisionLLMConfigCreate, + VisionLLMConfigRead, + VisionLLMConfigUpdate, +) +from app.users import current_active_user +from app.utils.rbac import check_permission + +router = APIRouter() +logger = logging.getLogger(__name__) + + +# ============================================================================= +# Global Vision LLM Configs (from YAML) +# ============================================================================= + + +@router.get( + "/global-vision-llm-configs", + response_model=list[GlobalVisionLLMConfigRead], +) +async def get_global_vision_llm_configs( + user: User = Depends(current_active_user), +): + try: + global_configs = config.GLOBAL_VISION_LLM_CONFIGS + safe_configs = [] + + if global_configs and len(global_configs) > 0: + safe_configs.append( + { + "id": 0, + "name": "Auto (Fastest)", + "description": "Automatically routes across available vision LLM providers.", + "provider": "AUTO", + "custom_provider": None, + "model_name": "auto", + "api_base": None, + "api_version": None, + "litellm_params": {}, + "is_global": True, + "is_auto_mode": True, + } + ) + + for cfg in global_configs: + safe_configs.append( + { + "id": cfg.get("id"), + "name": cfg.get("name"), + "description": cfg.get("description"), + "provider": cfg.get("provider"), + "custom_provider": cfg.get("custom_provider"), + "model_name": cfg.get("model_name"), + "api_base": cfg.get("api_base") or None, + "api_version": cfg.get("api_version") or None, + "litellm_params": cfg.get("litellm_params", {}), + "is_global": True, + } + ) + + return safe_configs + except Exception as e: + logger.exception("Failed to fetch global vision LLM configs") + raise HTTPException( + status_code=500, detail=f"Failed to fetch configs: {e!s}" + ) from e + + +# ============================================================================= +# VisionLLMConfig CRUD +# ============================================================================= + + +@router.post("/vision-llm-configs", response_model=VisionLLMConfigRead) +async def create_vision_llm_config( + config_data: VisionLLMConfigCreate, + session: AsyncSession = Depends(get_async_session), + user: User = Depends(current_active_user), +): + try: + await check_permission( + session, + user, + config_data.search_space_id, + Permission.VISION_CONFIGS_CREATE.value, + "You don't have permission to create vision LLM configs in this search space", + ) + + db_config = VisionLLMConfig(**config_data.model_dump(), user_id=user.id) + session.add(db_config) + await session.commit() + await session.refresh(db_config) + return db_config + + except HTTPException: + raise + except Exception as e: + await session.rollback() + logger.exception("Failed to create VisionLLMConfig") + raise HTTPException( + status_code=500, detail=f"Failed to create config: {e!s}" + ) from e + + +@router.get("/vision-llm-configs", response_model=list[VisionLLMConfigRead]) +async def list_vision_llm_configs( + search_space_id: int, + skip: int = 0, + limit: int = 100, + session: AsyncSession = Depends(get_async_session), + user: User = Depends(current_active_user), +): + try: + await check_permission( + session, + user, + search_space_id, + Permission.VISION_CONFIGS_READ.value, + "You don't have permission to view vision LLM configs in this search space", + ) + + result = await session.execute( + select(VisionLLMConfig) + .filter(VisionLLMConfig.search_space_id == search_space_id) + .order_by(VisionLLMConfig.created_at.desc()) + .offset(skip) + .limit(limit) + ) + return result.scalars().all() + + except HTTPException: + raise + except Exception as e: + logger.exception("Failed to list VisionLLMConfigs") + raise HTTPException( + status_code=500, detail=f"Failed to fetch configs: {e!s}" + ) from e + + +@router.get( + "/vision-llm-configs/{config_id}", response_model=VisionLLMConfigRead +) +async def get_vision_llm_config( + config_id: int, + session: AsyncSession = Depends(get_async_session), + user: User = Depends(current_active_user), +): + try: + result = await session.execute( + select(VisionLLMConfig).filter(VisionLLMConfig.id == config_id) + ) + db_config = result.scalars().first() + if not db_config: + raise HTTPException(status_code=404, detail="Config not found") + + await check_permission( + session, + user, + db_config.search_space_id, + Permission.VISION_CONFIGS_READ.value, + "You don't have permission to view vision LLM configs in this search space", + ) + return db_config + + except HTTPException: + raise + except Exception as e: + logger.exception("Failed to get VisionLLMConfig") + raise HTTPException( + status_code=500, detail=f"Failed to fetch config: {e!s}" + ) from e + + +@router.put( + "/vision-llm-configs/{config_id}", response_model=VisionLLMConfigRead +) +async def update_vision_llm_config( + config_id: int, + update_data: VisionLLMConfigUpdate, + session: AsyncSession = Depends(get_async_session), + user: User = Depends(current_active_user), +): + try: + result = await session.execute( + select(VisionLLMConfig).filter(VisionLLMConfig.id == config_id) + ) + db_config = result.scalars().first() + if not db_config: + raise HTTPException(status_code=404, detail="Config not found") + + await check_permission( + session, + user, + db_config.search_space_id, + Permission.VISION_CONFIGS_CREATE.value, + "You don't have permission to update vision LLM configs in this search space", + ) + + for key, value in update_data.model_dump(exclude_unset=True).items(): + setattr(db_config, key, value) + + await session.commit() + await session.refresh(db_config) + return db_config + + except HTTPException: + raise + except Exception as e: + await session.rollback() + logger.exception("Failed to update VisionLLMConfig") + raise HTTPException( + status_code=500, detail=f"Failed to update config: {e!s}" + ) from e + + +@router.delete("/vision-llm-configs/{config_id}", response_model=dict) +async def delete_vision_llm_config( + config_id: int, + session: AsyncSession = Depends(get_async_session), + user: User = Depends(current_active_user), +): + try: + result = await session.execute( + select(VisionLLMConfig).filter(VisionLLMConfig.id == config_id) + ) + db_config = result.scalars().first() + if not db_config: + raise HTTPException(status_code=404, detail="Config not found") + + await check_permission( + session, + user, + db_config.search_space_id, + Permission.VISION_CONFIGS_DELETE.value, + "You don't have permission to delete vision LLM configs in this search space", + ) + + await session.delete(db_config) + await session.commit() + return { + "message": "Vision LLM config deleted successfully", + "id": config_id, + } + + except HTTPException: + raise + except Exception as e: + await session.rollback() + logger.exception("Failed to delete VisionLLMConfig") + raise HTTPException( + status_code=500, detail=f"Failed to delete config: {e!s}" + ) from e From 43b8862ac77b3351cd068d762563ff985568447e Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 19:16:51 +0200 Subject: [PATCH 14/28] Update get_vision_llm to use dedicated VisionLLMConfig system --- surfsense_backend/app/services/llm_service.py | 119 +++++++++++++++++- 1 file changed, 113 insertions(+), 6 deletions(-) diff --git a/surfsense_backend/app/services/llm_service.py b/surfsense_backend/app/services/llm_service.py index 7c0f9e7e3..e531aeabb 100644 --- a/surfsense_backend/app/services/llm_service.py +++ b/surfsense_backend/app/services/llm_service.py @@ -32,7 +32,6 @@ logger = logging.getLogger(__name__) class LLMRole: AGENT = "agent" # For agent/chat operations DOCUMENT_SUMMARY = "document_summary" # For document summarization - VISION = "vision" # For vision/screenshot analysis def get_global_llm_config(llm_config_id: int) -> dict | None: @@ -188,7 +187,7 @@ async def get_search_space_llm_instance( Args: session: Database session search_space_id: Search Space ID - role: LLM role ('agent', 'document_summary', or 'vision') + role: LLM role ('agent' or 'document_summary') Returns: ChatLiteLLM or ChatLiteLLMRouter instance, or None if not found @@ -210,8 +209,6 @@ async def get_search_space_llm_instance( llm_config_id = search_space.agent_llm_id elif role == LLMRole.DOCUMENT_SUMMARY: llm_config_id = search_space.document_summary_llm_id - elif role == LLMRole.VISION: - llm_config_id = search_space.vision_llm_id else: logger.error(f"Invalid LLM role: {role}") return None @@ -411,8 +408,118 @@ async def get_document_summary_llm( async def get_vision_llm( session: AsyncSession, search_space_id: int ) -> ChatLiteLLM | ChatLiteLLMRouter | None: - """Get the search space's vision LLM instance for screenshot analysis.""" - return await get_search_space_llm_instance(session, search_space_id, LLMRole.VISION) + """Get the search space's vision LLM instance for screenshot analysis. + + Resolves from the dedicated VisionLLMConfig system: + - Auto mode (ID 0): VisionLLMRouterService + - Global (negative ID): YAML configs + - DB (positive ID): VisionLLMConfig table + """ + from app.db import VisionLLMConfig + from app.services.vision_llm_router_service import ( + VISION_PROVIDER_MAP, + VisionLLMRouterService, + get_global_vision_llm_config, + is_vision_auto_mode, + ) + + try: + result = await session.execute( + select(SearchSpace).where(SearchSpace.id == search_space_id) + ) + search_space = result.scalars().first() + if not search_space: + logger.error(f"Search space {search_space_id} not found") + return None + + config_id = search_space.vision_llm_config_id + if config_id is None: + logger.error( + f"No vision LLM configured for search space {search_space_id}" + ) + return None + + if is_vision_auto_mode(config_id): + if not VisionLLMRouterService.is_initialized(): + logger.error( + "Vision Auto mode requested but Vision LLM Router not initialized" + ) + return None + try: + return ChatLiteLLMRouter( + router=VisionLLMRouterService.get_router(), + streaming=True, + ) + except Exception as e: + logger.error(f"Failed to create vision ChatLiteLLMRouter: {e}") + return None + + if config_id < 0: + global_cfg = get_global_vision_llm_config(config_id) + if not global_cfg: + logger.error(f"Global vision LLM config {config_id} not found") + return None + + if global_cfg.get("custom_provider"): + model_string = ( + f"{global_cfg['custom_provider']}/{global_cfg['model_name']}" + ) + else: + prefix = VISION_PROVIDER_MAP.get( + global_cfg["provider"].upper(), + global_cfg["provider"].lower(), + ) + model_string = f"{prefix}/{global_cfg['model_name']}" + + litellm_kwargs = { + "model": model_string, + "api_key": global_cfg["api_key"], + } + if global_cfg.get("api_base"): + litellm_kwargs["api_base"] = global_cfg["api_base"] + if global_cfg.get("litellm_params"): + litellm_kwargs.update(global_cfg["litellm_params"]) + + return ChatLiteLLM(**litellm_kwargs) + + result = await session.execute( + select(VisionLLMConfig).where( + VisionLLMConfig.id == config_id, + VisionLLMConfig.search_space_id == search_space_id, + ) + ) + vision_cfg = result.scalars().first() + if not vision_cfg: + logger.error( + f"Vision LLM config {config_id} not found in search space {search_space_id}" + ) + return None + + if vision_cfg.custom_provider: + model_string = f"{vision_cfg.custom_provider}/{vision_cfg.model_name}" + else: + prefix = VISION_PROVIDER_MAP.get( + vision_cfg.provider.value.upper(), + vision_cfg.provider.value.lower(), + ) + model_string = f"{prefix}/{vision_cfg.model_name}" + + litellm_kwargs = { + "model": model_string, + "api_key": vision_cfg.api_key, + } + if vision_cfg.api_base: + litellm_kwargs["api_base"] = vision_cfg.api_base + if vision_cfg.litellm_params: + litellm_kwargs.update(vision_cfg.litellm_params) + + return ChatLiteLLM(**litellm_kwargs) + + except Exception as e: + logger.error( + f"Error getting vision LLM for search space {search_space_id}: {e!s}" + ) + return None # Backward-compatible alias (LLM preferences are now per-search-space, not per-user) From 6d85821ae95836978f63ccf18ddbfd693e2dd956 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 19:20:28 +0200 Subject: [PATCH 15/28] Wire vision_llm_config_id in preferences, register vision LLM router --- surfsense_backend/app/routes/__init__.py | 2 + .../app/routes/search_spaces_routes.py | 74 +++++++++++++++++-- .../app/schemas/new_llm_config.py | 10 +-- 3 files changed, 75 insertions(+), 11 deletions(-) diff --git a/surfsense_backend/app/routes/__init__.py b/surfsense_backend/app/routes/__init__.py index 22631bc1d..02367606b 100644 --- a/surfsense_backend/app/routes/__init__.py +++ b/surfsense_backend/app/routes/__init__.py @@ -49,6 +49,7 @@ from .stripe_routes import router as stripe_router from .surfsense_docs_routes import router as surfsense_docs_router from .teams_add_connector_route import router as teams_add_connector_router from .video_presentations_routes import router as video_presentations_router +from .vision_llm_routes import router as vision_llm_router from .youtube_routes import router as youtube_router router = APIRouter() @@ -68,6 +69,7 @@ router.include_router( ) # Video presentation status and streaming router.include_router(reports_router) # Report CRUD and multi-format export router.include_router(image_generation_router) # Image generation via litellm +router.include_router(vision_llm_router) # Vision LLM configs for screenshot analysis router.include_router(search_source_connectors_router) router.include_router(google_calendar_add_connector_router) router.include_router(google_gmail_add_connector_router) diff --git a/surfsense_backend/app/routes/search_spaces_routes.py b/surfsense_backend/app/routes/search_spaces_routes.py index c4f1ab035..78be97aa1 100644 --- a/surfsense_backend/app/routes/search_spaces_routes.py +++ b/surfsense_backend/app/routes/search_spaces_routes.py @@ -14,6 +14,7 @@ from app.db import ( SearchSpaceMembership, SearchSpaceRole, User, + VisionLLMConfig, get_async_session, get_default_roles_config, ) @@ -483,6 +484,63 @@ async def _get_image_gen_config_by_id( return None +async def _get_vision_llm_config_by_id( + session: AsyncSession, config_id: int | None +) -> dict | None: + if config_id is None: + return None + + if config_id == 0: + return { + "id": 0, + "name": "Auto (Fastest)", + "description": "Automatically routes requests across available vision LLM providers", + "provider": "AUTO", + "model_name": "auto", + "is_global": True, + "is_auto_mode": True, + } + + if config_id < 0: + for cfg in config.GLOBAL_VISION_LLM_CONFIGS: + if cfg.get("id") == config_id: + return { + "id": cfg.get("id"), + "name": cfg.get("name"), + "description": cfg.get("description"), + "provider": cfg.get("provider"), + "custom_provider": cfg.get("custom_provider"), + "model_name": cfg.get("model_name"), + "api_base": cfg.get("api_base") or None, + "api_version": cfg.get("api_version") or None, + "litellm_params": cfg.get("litellm_params", {}), + "is_global": True, + } + return None + + result = await session.execute( + select(VisionLLMConfig).filter(VisionLLMConfig.id == config_id) + ) + db_config = result.scalars().first() + if db_config: + return { + "id": db_config.id, + "name": db_config.name, + "description": db_config.description, + "provider": db_config.provider.value if db_config.provider else None, + "custom_provider": db_config.custom_provider, + "model_name": db_config.model_name, + "api_base": db_config.api_base, + "api_version": db_config.api_version, + "litellm_params": db_config.litellm_params or {}, + "created_at": db_config.created_at.isoformat() + if db_config.created_at + else None, + "search_space_id": db_config.search_space_id, + } + return None + + @router.get( "/search-spaces/{search_space_id}/llm-preferences", response_model=LLMPreferencesRead, @@ -522,17 +580,19 @@ async def get_llm_preferences( image_generation_config = await _get_image_gen_config_by_id( session, search_space.image_generation_config_id ) - vision_llm = await _get_llm_config_by_id(session, search_space.vision_llm_id) + vision_llm_config = await _get_vision_llm_config_by_id( + session, search_space.vision_llm_config_id + ) return LLMPreferencesRead( agent_llm_id=search_space.agent_llm_id, document_summary_llm_id=search_space.document_summary_llm_id, image_generation_config_id=search_space.image_generation_config_id, - vision_llm_id=search_space.vision_llm_id, + vision_llm_config_id=search_space.vision_llm_config_id, agent_llm=agent_llm, document_summary_llm=document_summary_llm, image_generation_config=image_generation_config, - vision_llm=vision_llm, + vision_llm_config=vision_llm_config, ) except HTTPException: @@ -592,17 +652,19 @@ async def update_llm_preferences( image_generation_config = await _get_image_gen_config_by_id( session, search_space.image_generation_config_id ) - vision_llm = await _get_llm_config_by_id(session, search_space.vision_llm_id) + vision_llm_config = await _get_vision_llm_config_by_id( + session, search_space.vision_llm_config_id + ) return LLMPreferencesRead( agent_llm_id=search_space.agent_llm_id, document_summary_llm_id=search_space.document_summary_llm_id, image_generation_config_id=search_space.image_generation_config_id, - vision_llm_id=search_space.vision_llm_id, + vision_llm_config_id=search_space.vision_llm_config_id, agent_llm=agent_llm, document_summary_llm=document_summary_llm, image_generation_config=image_generation_config, - vision_llm=vision_llm, + vision_llm_config=vision_llm_config, ) except HTTPException: diff --git a/surfsense_backend/app/schemas/new_llm_config.py b/surfsense_backend/app/schemas/new_llm_config.py index 6c76ca512..a466f2c99 100644 --- a/surfsense_backend/app/schemas/new_llm_config.py +++ b/surfsense_backend/app/schemas/new_llm_config.py @@ -182,8 +182,8 @@ class LLMPreferencesRead(BaseModel): image_generation_config_id: int | None = Field( None, description="ID of the image generation config to use" ) - vision_llm_id: int | None = Field( - None, description="ID of the LLM config to use for vision/screenshot analysis" + vision_llm_config_id: int | None = Field( + None, description="ID of the vision LLM config to use for vision/screenshot analysis" ) agent_llm: dict[str, Any] | None = Field( None, description="Full config for agent LLM" @@ -194,7 +194,7 @@ class LLMPreferencesRead(BaseModel): image_generation_config: dict[str, Any] | None = Field( None, description="Full config for image generation" ) - vision_llm: dict[str, Any] | None = Field( + vision_llm_config: dict[str, Any] | None = Field( None, description="Full config for vision LLM" ) @@ -213,6 +213,6 @@ class LLMPreferencesUpdate(BaseModel): image_generation_config_id: int | None = Field( None, description="ID of the image generation config to use" ) - vision_llm_id: int | None = Field( - None, description="ID of the LLM config to use for vision/screenshot analysis" + vision_llm_config_id: int | None = Field( + None, description="ID of the vision LLM config to use for vision/screenshot analysis" ) From 4a675b64f4b066a6148126df0121ccfbdc9dbe0a Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 19:21:10 +0200 Subject: [PATCH 16/28] Initialize vision LLM router at app and celery startup --- surfsense_backend/app/app.py | 8 +++++++- surfsense_backend/app/celery_app.py | 7 ++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/surfsense_backend/app/app.py b/surfsense_backend/app/app.py index bba2f1f3a..7b2b421ac 100644 --- a/surfsense_backend/app/app.py +++ b/surfsense_backend/app/app.py @@ -25,7 +25,12 @@ from app.agents.new_chat.checkpointer import ( close_checkpointer, setup_checkpointer_tables, ) -from app.config import config, initialize_image_gen_router, initialize_llm_router +from app.config import ( + config, + initialize_image_gen_router, + initialize_llm_router, + initialize_vision_llm_router, +) from app.db import User, create_db_and_tables, get_async_session from app.routes import router as crud_router from app.routes.auth_routes import router as auth_router @@ -223,6 +228,7 @@ async def lifespan(app: FastAPI): await setup_checkpointer_tables() initialize_llm_router() initialize_image_gen_router() + initialize_vision_llm_router() try: await asyncio.wait_for(seed_surfsense_docs(), timeout=120) except TimeoutError: diff --git a/surfsense_backend/app/celery_app.py b/surfsense_backend/app/celery_app.py index 684da6a13..bf2fdcb39 100644 --- a/surfsense_backend/app/celery_app.py +++ b/surfsense_backend/app/celery_app.py @@ -18,10 +18,15 @@ def init_worker(**kwargs): This ensures the Auto mode (LiteLLM Router) is available for background tasks like document summarization and image generation. """ - from app.config import initialize_image_gen_router, initialize_llm_router + from app.config import ( + initialize_image_gen_router, + initialize_llm_router, + initialize_vision_llm_router, + ) initialize_llm_router() initialize_image_gen_router() + initialize_vision_llm_router() # Get Celery configuration from environment From 3369b8a83210e9afc62f9c0db91d87a13fb0c478 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 19:24:43 +0200 Subject: [PATCH 17/28] Add frontend vision LLM config types, API, atoms, and role manager wiring --- .../vision-llm-config-mutation.atoms.ts | 84 +++++++++++++++ .../vision-llm-config-query.atoms.ts | 27 +++++ .../components/settings/llm-role-manager.tsx | 66 +++++++++--- .../contracts/enums/vision-providers.ts | 102 ++++++++++++++++++ .../contracts/types/new-llm-config.types.ts | 99 +++++++++++++++-- .../lib/apis/vision-llm-config-api.service.ts | 58 ++++++++++ surfsense_web/lib/query-client/cache-keys.ts | 5 + 7 files changed, 421 insertions(+), 20 deletions(-) create mode 100644 surfsense_web/atoms/vision-llm-config/vision-llm-config-mutation.atoms.ts create mode 100644 surfsense_web/atoms/vision-llm-config/vision-llm-config-query.atoms.ts create mode 100644 surfsense_web/contracts/enums/vision-providers.ts create mode 100644 surfsense_web/lib/apis/vision-llm-config-api.service.ts diff --git a/surfsense_web/atoms/vision-llm-config/vision-llm-config-mutation.atoms.ts b/surfsense_web/atoms/vision-llm-config/vision-llm-config-mutation.atoms.ts new file mode 100644 index 000000000..b1aa01c6b --- /dev/null +++ b/surfsense_web/atoms/vision-llm-config/vision-llm-config-mutation.atoms.ts @@ -0,0 +1,84 @@ +import { atomWithMutation } from "jotai-tanstack-query"; +import { toast } from "sonner"; +import type { + CreateVisionLLMConfigRequest, + CreateVisionLLMConfigResponse, + DeleteVisionLLMConfigResponse, + GetVisionLLMConfigsResponse, + UpdateVisionLLMConfigRequest, + UpdateVisionLLMConfigResponse, +} from "@/contracts/types/new-llm-config.types"; +import { visionLLMConfigApiService } from "@/lib/apis/vision-llm-config-api.service"; +import { cacheKeys } from "@/lib/query-client/cache-keys"; +import { queryClient } from "@/lib/query-client/client"; +import { activeSearchSpaceIdAtom } from "../search-spaces/search-space-query.atoms"; + +export const createVisionLLMConfigMutationAtom = atomWithMutation((get) => { + const searchSpaceId = get(activeSearchSpaceIdAtom); + + return { + mutationKey: ["vision-llm-configs", "create"], + enabled: !!searchSpaceId, + mutationFn: async (request: CreateVisionLLMConfigRequest) => { + return visionLLMConfigApiService.createConfig(request); + }, + onSuccess: (_: CreateVisionLLMConfigResponse, request: CreateVisionLLMConfigRequest) => { + toast.success(`${request.name} created`); + queryClient.invalidateQueries({ + queryKey: cacheKeys.visionLLMConfigs.all(Number(searchSpaceId)), + }); + }, + onError: (error: Error) => { + toast.error(error.message || "Failed to create vision model"); + }, + }; +}); + +export const updateVisionLLMConfigMutationAtom = atomWithMutation((get) => { + const searchSpaceId = get(activeSearchSpaceIdAtom); + + return { + mutationKey: ["vision-llm-configs", "update"], + enabled: !!searchSpaceId, + mutationFn: async (request: UpdateVisionLLMConfigRequest) => { + return visionLLMConfigApiService.updateConfig(request); + }, + onSuccess: (_: UpdateVisionLLMConfigResponse, request: UpdateVisionLLMConfigRequest) => { + toast.success(`${request.data.name ?? "Configuration"} updated`); + queryClient.invalidateQueries({ + queryKey: cacheKeys.visionLLMConfigs.all(Number(searchSpaceId)), + }); + queryClient.invalidateQueries({ + queryKey: cacheKeys.visionLLMConfigs.byId(request.id), + }); + }, + onError: (error: Error) => { + toast.error(error.message || "Failed to update vision model"); + }, + }; +}); + +export const deleteVisionLLMConfigMutationAtom = atomWithMutation((get) => { + const searchSpaceId = get(activeSearchSpaceIdAtom); + + return { + mutationKey: ["vision-llm-configs", "delete"], + enabled: !!searchSpaceId, + mutationFn: async (request: { id: number; name: string }) => { + return visionLLMConfigApiService.deleteConfig(request.id); + }, + onSuccess: (_: DeleteVisionLLMConfigResponse, request: { id: number; name: string }) => { + toast.success(`${request.name} deleted`); + queryClient.setQueryData( + cacheKeys.visionLLMConfigs.all(Number(searchSpaceId)), + (oldData: GetVisionLLMConfigsResponse | undefined) => { + if (!oldData) return oldData; + return oldData.filter((config) => config.id !== request.id); + } + ); + }, + onError: (error: Error) => { + toast.error(error.message || "Failed to delete vision model"); + }, + }; +}); diff --git a/surfsense_web/atoms/vision-llm-config/vision-llm-config-query.atoms.ts b/surfsense_web/atoms/vision-llm-config/vision-llm-config-query.atoms.ts new file mode 100644 index 000000000..53264fb24 --- /dev/null +++ b/surfsense_web/atoms/vision-llm-config/vision-llm-config-query.atoms.ts @@ -0,0 +1,27 @@ +import { atomWithQuery } from "jotai-tanstack-query"; +import { visionLLMConfigApiService } from "@/lib/apis/vision-llm-config-api.service"; +import { cacheKeys } from "@/lib/query-client/cache-keys"; +import { activeSearchSpaceIdAtom } from "../search-spaces/search-space-query.atoms"; + +export const visionLLMConfigsAtom = atomWithQuery((get) => { + const searchSpaceId = get(activeSearchSpaceIdAtom); + + return { + queryKey: cacheKeys.visionLLMConfigs.all(Number(searchSpaceId)), + enabled: !!searchSpaceId, + staleTime: 5 * 60 * 1000, + queryFn: async () => { + return visionLLMConfigApiService.getConfigs(Number(searchSpaceId)); + }, + }; +}); + +export const globalVisionLLMConfigsAtom = atomWithQuery(() => { + return { + queryKey: cacheKeys.visionLLMConfigs.global(), + staleTime: 10 * 60 * 1000, + queryFn: async () => { + return visionLLMConfigApiService.getGlobalConfigs(); + }, + }; +}); diff --git a/surfsense_web/components/settings/llm-role-manager.tsx b/surfsense_web/components/settings/llm-role-manager.tsx index 386845d7d..995159d58 100644 --- a/surfsense_web/components/settings/llm-role-manager.tsx +++ b/surfsense_web/components/settings/llm-role-manager.tsx @@ -24,6 +24,10 @@ import { llmPreferencesAtom, newLLMConfigsAtom, } from "@/atoms/new-llm-config/new-llm-config-query.atoms"; +import { + globalVisionLLMConfigsAtom, + visionLLMConfigsAtom, +} from "@/atoms/vision-llm-config/vision-llm-config-query.atoms"; import { Alert, AlertDescription } from "@/components/ui/alert"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; @@ -77,8 +81,8 @@ const ROLE_DESCRIPTIONS = { description: "Vision-capable model for screenshot analysis and context extraction", color: "text-amber-600 dark:text-amber-400", bgColor: "bg-amber-500/10", - prefKey: "vision_llm_id" as const, - configType: "llm" as const, + prefKey: "vision_llm_config_id" as const, + configType: "vision" as const, }, }; @@ -112,6 +116,18 @@ export function LLMRoleManager({ searchSpaceId }: LLMRoleManagerProps) { error: globalImageConfigsError, } = useAtomValue(globalImageGenConfigsAtom); + // Vision LLM configs + const { + data: userVisionConfigs = [], + isFetching: visionConfigsLoading, + error: visionConfigsError, + } = useAtomValue(visionLLMConfigsAtom); + const { + data: globalVisionConfigs = [], + isFetching: globalVisionConfigsLoading, + error: globalVisionConfigsError, + } = useAtomValue(globalVisionLLMConfigsAtom); + // Preferences const { data: preferences = {}, @@ -125,7 +141,7 @@ export function LLMRoleManager({ searchSpaceId }: LLMRoleManagerProps) { agent_llm_id: preferences.agent_llm_id ?? "", document_summary_llm_id: preferences.document_summary_llm_id ?? "", image_generation_config_id: preferences.image_generation_config_id ?? "", - vision_llm_id: preferences.vision_llm_id ?? "", + vision_llm_config_id: preferences.vision_llm_config_id ?? "", })); const [savingRole, setSavingRole] = useState(null); @@ -137,14 +153,14 @@ export function LLMRoleManager({ searchSpaceId }: LLMRoleManagerProps) { agent_llm_id: preferences.agent_llm_id ?? "", document_summary_llm_id: preferences.document_summary_llm_id ?? "", image_generation_config_id: preferences.image_generation_config_id ?? "", - vision_llm_id: preferences.vision_llm_id ?? "", + vision_llm_config_id: preferences.vision_llm_config_id ?? "", }); } }, [ preferences?.agent_llm_id, preferences?.document_summary_llm_id, preferences?.image_generation_config_id, - preferences?.vision_llm_id, + preferences?.vision_llm_config_id, ]); const handleRoleAssignment = useCallback( @@ -181,6 +197,14 @@ export function LLMRoleManager({ searchSpaceId }: LLMRoleManagerProps) { ...(userImageConfigs ?? []).filter((config) => config.id && config.id.toString().trim() !== ""), ]; + // Combine global and custom vision LLM configs + const allVisionConfigs = [ + ...globalVisionConfigs.map((config) => ({ ...config, is_global: true })), + ...(userVisionConfigs ?? []).filter( + (config) => config.id && config.id.toString().trim() !== "" + ), + ]; + const isAssignmentComplete = allLLMConfigs.some((c) => c.id === assignments.agent_llm_id) && allLLMConfigs.some((c) => c.id === assignments.document_summary_llm_id) && @@ -191,13 +215,17 @@ export function LLMRoleManager({ searchSpaceId }: LLMRoleManagerProps) { preferencesLoading || globalConfigsLoading || imageConfigsLoading || - globalImageConfigsLoading; + globalImageConfigsLoading || + visionConfigsLoading || + globalVisionConfigsLoading; const hasError = configsError || preferencesError || globalConfigsError || imageConfigsError || - globalImageConfigsError; + globalImageConfigsError || + visionConfigsError || + globalVisionConfigsError; const hasAnyConfigs = allLLMConfigs.length > 0 || allImageConfigs.length > 0; return ( @@ -291,15 +319,27 @@ export function LLMRoleManager({ searchSpaceId }: LLMRoleManagerProps) {
{Object.entries(ROLE_DESCRIPTIONS).map(([key, role]) => { const IconComponent = role.icon; - const isImageRole = role.configType === "image"; const currentAssignment = assignments[role.prefKey as keyof typeof assignments]; // Pick the right config lists based on role type - const roleGlobalConfigs = isImageRole ? globalImageConfigs : globalConfigs; - const roleUserConfigs = isImageRole - ? (userImageConfigs ?? []).filter((c) => c.id && c.id.toString().trim() !== "") - : newLLMConfigs.filter((c) => c.id && c.id.toString().trim() !== ""); - const roleAllConfigs = isImageRole ? allImageConfigs : allLLMConfigs; + const roleGlobalConfigs = + role.configType === "image" + ? globalImageConfigs + : role.configType === "vision" + ? globalVisionConfigs + : globalConfigs; + const roleUserConfigs = + role.configType === "image" + ? (userImageConfigs ?? []).filter((c) => c.id && c.id.toString().trim() !== "") + : role.configType === "vision" + ? (userVisionConfigs ?? []).filter((c) => c.id && c.id.toString().trim() !== "") + : newLLMConfigs.filter((c) => c.id && c.id.toString().trim() !== ""); + const roleAllConfigs = + role.configType === "image" + ? allImageConfigs + : role.configType === "vision" + ? allVisionConfigs + : allLLMConfigs; const assignedConfig = roleAllConfigs.find((config) => config.id === currentAssignment); const isAssigned = !!assignedConfig; diff --git a/surfsense_web/contracts/enums/vision-providers.ts b/surfsense_web/contracts/enums/vision-providers.ts new file mode 100644 index 000000000..260b03585 --- /dev/null +++ b/surfsense_web/contracts/enums/vision-providers.ts @@ -0,0 +1,102 @@ +export interface VisionProviderInfo { + value: string; + label: string; + example: string; + description: string; + apiBase?: string; +} + +export const VISION_PROVIDERS: VisionProviderInfo[] = [ + { + value: "OPENAI", + label: "OpenAI", + example: "gpt-4o, gpt-4o-mini", + description: "GPT-4o vision models", + }, + { + value: "ANTHROPIC", + label: "Anthropic", + example: "claude-sonnet-4-20250514", + description: "Claude vision models", + }, + { + value: "GOOGLE", + label: "Google AI Studio", + example: "gemini-2.5-flash, gemini-2.0-flash", + description: "Gemini vision models", + }, + { + value: "AZURE_OPENAI", + label: "Azure OpenAI", + example: "azure/gpt-4o", + description: "OpenAI vision models on Azure", + }, + { + value: "VERTEX_AI", + label: "Google Vertex AI", + example: "vertex_ai/gemini-2.5-flash", + description: "Gemini vision models on Vertex AI", + }, + { + value: "BEDROCK", + label: "AWS Bedrock", + example: "bedrock/anthropic.claude-sonnet-4-20250514-v1:0", + description: "Vision models on AWS Bedrock", + }, + { + value: "XAI", + label: "xAI", + example: "grok-2-vision", + description: "Grok vision models", + }, + { + value: "OPENROUTER", + label: "OpenRouter", + example: "openrouter/openai/gpt-4o", + description: "Vision models via OpenRouter", + }, + { + value: "OLLAMA", + label: "Ollama", + example: "llava, bakllava", + description: "Local vision models via Ollama", + apiBase: "http://localhost:11434", + }, + { + value: "GROQ", + label: "Groq", + example: "llama-4-scout-17b-16e-instruct", + description: "Vision models on Groq", + }, + { + value: "TOGETHER_AI", + label: "Together AI", + example: "meta-llama/Llama-4-Scout-17B-16E-Instruct", + description: "Vision models on Together AI", + }, + { + value: "FIREWORKS_AI", + label: "Fireworks AI", + example: "fireworks_ai/phi-3-vision-128k-instruct", + description: "Vision models on Fireworks AI", + }, + { + value: "DEEPSEEK", + label: "DeepSeek", + example: "deepseek-chat", + description: "DeepSeek vision models", + apiBase: "https://api.deepseek.com", + }, + { + value: "MISTRAL", + label: "Mistral", + example: "pixtral-large-latest", + description: "Pixtral vision models", + }, + { + value: "CUSTOM", + label: "Custom Provider", + example: "custom/my-vision-model", + description: "Custom OpenAI-compatible vision endpoint", + }, +]; diff --git a/surfsense_web/contracts/types/new-llm-config.types.ts b/surfsense_web/contracts/types/new-llm-config.types.ts index 02837cc73..6bef94bac 100644 --- a/surfsense_web/contracts/types/new-llm-config.types.ts +++ b/surfsense_web/contracts/types/new-llm-config.types.ts @@ -252,23 +252,99 @@ export const globalImageGenConfig = z.object({ export const getGlobalImageGenConfigsResponse = z.array(globalImageGenConfig); +// ============================================================================= +// Vision LLM Config (separate table for vision-capable models) +// ============================================================================= + +export const visionProviderEnum = z.enum([ + "OPENAI", + "ANTHROPIC", + "GOOGLE", + "AZURE_OPENAI", + "VERTEX_AI", + "BEDROCK", + "XAI", + "OPENROUTER", + "OLLAMA", + "GROQ", + "TOGETHER_AI", + "FIREWORKS_AI", + "DEEPSEEK", + "MISTRAL", + "CUSTOM", +]); + +export type VisionProvider = z.infer; + +export const visionLLMConfig = z.object({ + id: z.number(), + name: z.string().max(100), + description: z.string().max(500).nullable().optional(), + provider: visionProviderEnum, + custom_provider: z.string().max(100).nullable().optional(), + model_name: z.string().max(100), + api_key: z.string(), + api_base: z.string().max(500).nullable().optional(), + api_version: z.string().max(50).nullable().optional(), + litellm_params: z.record(z.string(), z.any()).nullable().optional(), + created_at: z.string(), + search_space_id: z.number(), + user_id: z.string(), +}); + +export const createVisionLLMConfigRequest = visionLLMConfig.omit({ + id: true, + created_at: true, + user_id: true, +}); + +export const createVisionLLMConfigResponse = visionLLMConfig; + +export const getVisionLLMConfigsResponse = z.array(visionLLMConfig); + +export const updateVisionLLMConfigRequest = z.object({ + id: z.number(), + data: visionLLMConfig + .omit({ id: true, created_at: true, search_space_id: true, user_id: true }) + .partial(), +}); + +export const updateVisionLLMConfigResponse = visionLLMConfig; + +export const deleteVisionLLMConfigResponse = z.object({ + message: z.string(), + id: z.number(), +}); + +export const globalVisionLLMConfig = z.object({ + id: z.number(), + name: z.string(), + description: z.string().nullable().optional(), + provider: z.string(), + custom_provider: z.string().nullable().optional(), + model_name: z.string(), + api_base: z.string().nullable().optional(), + api_version: z.string().nullable().optional(), + litellm_params: z.record(z.string(), z.any()).nullable().optional(), + is_global: z.literal(true), + is_auto_mode: z.boolean().optional().default(false), +}); + +export const getGlobalVisionLLMConfigsResponse = z.array(globalVisionLLMConfig); + // ============================================================================= // LLM Preferences (Role Assignments) // ============================================================================= -/** - * LLM Preferences schemas - for role assignments - * image_generation uses image_generation_config_id (not llm_id) - */ export const llmPreferences = z.object({ agent_llm_id: z.union([z.number(), z.null()]).optional(), document_summary_llm_id: z.union([z.number(), z.null()]).optional(), image_generation_config_id: z.union([z.number(), z.null()]).optional(), - vision_llm_id: z.union([z.number(), z.null()]).optional(), + vision_llm_config_id: z.union([z.number(), z.null()]).optional(), agent_llm: z.union([z.record(z.string(), z.unknown()), z.null()]).optional(), document_summary_llm: z.union([z.record(z.string(), z.unknown()), z.null()]).optional(), image_generation_config: z.union([z.record(z.string(), z.unknown()), z.null()]).optional(), - vision_llm: z.union([z.record(z.string(), z.unknown()), z.null()]).optional(), + vision_llm_config: z.union([z.record(z.string(), z.unknown()), z.null()]).optional(), }); /** @@ -289,7 +365,7 @@ export const updateLLMPreferencesRequest = z.object({ agent_llm_id: true, document_summary_llm_id: true, image_generation_config_id: true, - vision_llm_id: true, + vision_llm_config_id: true, }), }); @@ -341,6 +417,15 @@ export type UpdateImageGenConfigResponse = z.infer; export type GlobalImageGenConfig = z.infer; export type GetGlobalImageGenConfigsResponse = z.infer; +export type VisionLLMConfig = z.infer; +export type CreateVisionLLMConfigRequest = z.infer; +export type CreateVisionLLMConfigResponse = z.infer; +export type GetVisionLLMConfigsResponse = z.infer; +export type UpdateVisionLLMConfigRequest = z.infer; +export type UpdateVisionLLMConfigResponse = z.infer; +export type DeleteVisionLLMConfigResponse = z.infer; +export type GlobalVisionLLMConfig = z.infer; +export type GetGlobalVisionLLMConfigsResponse = z.infer; export type LLMPreferences = z.infer; export type GetLLMPreferencesRequest = z.infer; export type GetLLMPreferencesResponse = z.infer; diff --git a/surfsense_web/lib/apis/vision-llm-config-api.service.ts b/surfsense_web/lib/apis/vision-llm-config-api.service.ts new file mode 100644 index 000000000..4099c6b39 --- /dev/null +++ b/surfsense_web/lib/apis/vision-llm-config-api.service.ts @@ -0,0 +1,58 @@ +import { + type CreateVisionLLMConfigRequest, + createVisionLLMConfigRequest, + createVisionLLMConfigResponse, + deleteVisionLLMConfigResponse, + getGlobalVisionLLMConfigsResponse, + getVisionLLMConfigsResponse, + type UpdateVisionLLMConfigRequest, + updateVisionLLMConfigRequest, + updateVisionLLMConfigResponse, +} from "@/contracts/types/new-llm-config.types"; +import { ValidationError } from "../error"; +import { baseApiService } from "./base-api.service"; + +class VisionLLMConfigApiService { + getGlobalConfigs = async () => { + return baseApiService.get( + `/api/v1/global-vision-llm-configs`, + getGlobalVisionLLMConfigsResponse + ); + }; + + createConfig = async (request: CreateVisionLLMConfigRequest) => { + const parsed = createVisionLLMConfigRequest.safeParse(request); + if (!parsed.success) { + const msg = parsed.error.issues.map((i) => i.message).join(", "); + throw new ValidationError(`Invalid request: ${msg}`); + } + return baseApiService.post(`/api/v1/vision-llm-configs`, createVisionLLMConfigResponse, { + body: parsed.data, + }); + }; + + getConfigs = async (searchSpaceId: number) => { + const params = new URLSearchParams({ + search_space_id: String(searchSpaceId), + }).toString(); + return baseApiService.get(`/api/v1/vision-llm-configs?${params}`, getVisionLLMConfigsResponse); + }; + + updateConfig = async (request: UpdateVisionLLMConfigRequest) => { + const parsed = updateVisionLLMConfigRequest.safeParse(request); + if (!parsed.success) { + const msg = parsed.error.issues.map((i) => i.message).join(", "); + throw new ValidationError(`Invalid request: ${msg}`); + } + const { id, data } = parsed.data; + return baseApiService.put(`/api/v1/vision-llm-configs/${id}`, updateVisionLLMConfigResponse, { + body: data, + }); + }; + + deleteConfig = async (id: number) => { + return baseApiService.delete(`/api/v1/vision-llm-configs/${id}`, deleteVisionLLMConfigResponse); + }; +} + +export const visionLLMConfigApiService = new VisionLLMConfigApiService(); diff --git a/surfsense_web/lib/query-client/cache-keys.ts b/surfsense_web/lib/query-client/cache-keys.ts index 754886618..04f348ff8 100644 --- a/surfsense_web/lib/query-client/cache-keys.ts +++ b/surfsense_web/lib/query-client/cache-keys.ts @@ -39,6 +39,11 @@ export const cacheKeys = { byId: (configId: number) => ["image-gen-configs", "detail", configId] as const, global: () => ["image-gen-configs", "global"] as const, }, + visionLLMConfigs: { + all: (searchSpaceId: number) => ["vision-llm-configs", searchSpaceId] as const, + byId: (configId: number) => ["vision-llm-configs", "detail", configId] as const, + global: () => ["vision-llm-configs", "global"] as const, + }, auth: { user: ["auth", "user"] as const, }, From 3bbe6c303737f63b089ddb5501d67f47d65a37b5 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 19:27:24 +0200 Subject: [PATCH 18/28] Add VisionModelManager and VisionConfigDialog components --- .../settings/vision-model-manager.tsx | 401 ++++++++++++++++++ .../shared/vision-config-dialog.tsx | 381 +++++++++++++++++ 2 files changed, 782 insertions(+) create mode 100644 surfsense_web/components/settings/vision-model-manager.tsx create mode 100644 surfsense_web/components/shared/vision-config-dialog.tsx diff --git a/surfsense_web/components/settings/vision-model-manager.tsx b/surfsense_web/components/settings/vision-model-manager.tsx new file mode 100644 index 000000000..31e6655cb --- /dev/null +++ b/surfsense_web/components/settings/vision-model-manager.tsx @@ -0,0 +1,401 @@ +"use client"; + +import { useAtomValue } from "jotai"; +import { AlertCircle, Dot, Edit3, Info, RefreshCw, Trash2 } from "lucide-react"; +import { useMemo, useState } from "react"; +import { membersAtom, myAccessAtom } from "@/atoms/members/members-query.atoms"; +import { deleteVisionLLMConfigMutationAtom } from "@/atoms/vision-llm-config/vision-llm-config-mutation.atoms"; +import { + globalVisionLLMConfigsAtom, + visionLLMConfigsAtom, +} from "@/atoms/vision-llm-config/vision-llm-config-query.atoms"; +import { VisionConfigDialog } from "@/components/shared/vision-config-dialog"; +import { Alert, AlertDescription } from "@/components/ui/alert"; +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from "@/components/ui/alert-dialog"; +import { Avatar, AvatarFallback, AvatarImage } from "@/components/ui/avatar"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent } from "@/components/ui/card"; +import { Skeleton } from "@/components/ui/skeleton"; +import { Spinner } from "@/components/ui/spinner"; +import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip"; +import type { VisionLLMConfig } from "@/contracts/types/new-llm-config.types"; +import { useMediaQuery } from "@/hooks/use-media-query"; +import { getProviderIcon } from "@/lib/provider-icons"; +import { cn } from "@/lib/utils"; + +interface VisionModelManagerProps { + searchSpaceId: number; +} + +function getInitials(name: string): string { + const parts = name.trim().split(/\s+/); + if (parts.length >= 2) { + return (parts[0][0] + parts[1][0]).toUpperCase(); + } + return name.slice(0, 2).toUpperCase(); +} + +export function VisionModelManager({ searchSpaceId }: VisionModelManagerProps) { + const isDesktop = useMediaQuery("(min-width: 768px)"); + + const { + mutateAsync: deleteConfig, + isPending: isDeleting, + error: deleteError, + } = useAtomValue(deleteVisionLLMConfigMutationAtom); + + const { + data: userConfigs, + isFetching: configsLoading, + error: fetchError, + refetch: refreshConfigs, + } = useAtomValue(visionLLMConfigsAtom); + const { data: globalConfigs = [], isFetching: globalLoading } = useAtomValue( + globalVisionLLMConfigsAtom + ); + + const { data: members } = useAtomValue(membersAtom); + const memberMap = useMemo(() => { + const map = new Map(); + if (members) { + for (const m of members) { + map.set(m.user_id, { + name: m.user_display_name || m.user_email || "Unknown", + email: m.user_email || undefined, + avatarUrl: m.user_avatar_url || undefined, + }); + } + } + return map; + }, [members]); + + const { data: access } = useAtomValue(myAccessAtom); + const canCreate = useMemo(() => { + if (!access) return false; + if (access.is_owner) return true; + return access.permissions?.includes("vision_configs:create") ?? false; + }, [access]); + const canDelete = useMemo(() => { + if (!access) return false; + if (access.is_owner) return true; + return access.permissions?.includes("vision_configs:delete") ?? false; + }, [access]); + const canUpdate = canCreate; + const isReadOnly = !canCreate && !canDelete; + + const [isDialogOpen, setIsDialogOpen] = useState(false); + const [editingConfig, setEditingConfig] = useState(null); + const [configToDelete, setConfigToDelete] = useState(null); + + const isLoading = configsLoading || globalLoading; + const errors = [deleteError, fetchError].filter(Boolean) as Error[]; + + const openEditDialog = (config: VisionLLMConfig) => { + setEditingConfig(config); + setIsDialogOpen(true); + }; + + const openNewDialog = () => { + setEditingConfig(null); + setIsDialogOpen(true); + }; + + const handleDelete = async () => { + if (!configToDelete) return; + try { + await deleteConfig({ id: configToDelete.id, name: configToDelete.name }); + setConfigToDelete(null); + } catch { + // Error handled by mutation + } + }; + + return ( +
+
+ + {canCreate && ( + + )} +
+ + {errors.map((err) => ( +
+ + + {err?.message} + +
+ ))} + + {access && !isLoading && isReadOnly && ( +
+ + + + You have read-only access to vision model + configurations. Contact a space owner to request additional permissions. + + +
+ )} + {access && !isLoading && !isReadOnly && (!canCreate || !canDelete) && ( +
+ + + + You can{" "} + {[canCreate && "create and edit", canDelete && "delete"] + .filter(Boolean) + .join(" and ")}{" "} + vision model configurations + {!canDelete && ", but cannot delete them"}. + + +
+ )} + + {globalConfigs.filter((g) => !("is_auto_mode" in g && g.is_auto_mode)).length > 0 && ( + + + +

+ + {globalConfigs.filter((g) => !("is_auto_mode" in g && g.is_auto_mode)).length}{" "} + global vision{" "} + {globalConfigs.filter((g) => !("is_auto_mode" in g && g.is_auto_mode)).length === 1 + ? "model" + : "models"} + {" "} + available from your administrator. Use the model selector to view and select them. +

+ + + )} + + {isLoading && ( +
+
+
+ + +
+
+ {["skeleton-a", "skeleton-b", "skeleton-c"].map((key) => ( + + +
+
+ + +
+
+
+ + +
+
+ + + +
+ + + ))} +
+
+
+ )} + + {!isLoading && ( +
+ {(userConfigs?.length ?? 0) === 0 ? ( + + +

No Vision Models Yet

+

+ {canCreate + ? "Add your own vision-capable model (GPT-4o, Claude, Gemini, etc.)" + : "No vision models have been added to this space yet. Contact a space owner to add one."} +

+ + + ) : ( +
+ {userConfigs?.map((config) => { + const member = config.user_id ? memberMap.get(config.user_id) : null; + + return ( +
+ + +
+
+

+ {config.name} +

+ {config.description && ( +

+ {config.description} +

+ )} +
+ {(canUpdate || canDelete) && ( +
+ {canUpdate && ( + + + + + + Edit + + + )} + {canDelete && ( + + + + + + Delete + + + )} +
+ )} +
+ +
+ {getProviderIcon(config.provider, { + className: "size-3.5 shrink-0", + })} + + {config.model_name} + +
+ +
+ + {new Date(config.created_at).toLocaleDateString(undefined, { + year: "numeric", + month: "short", + day: "numeric", + })} + + {member && ( + <> + + + + +
+ + {member.avatarUrl && ( + + )} + + {getInitials(member.name)} + + + + {member.name} + +
+ + + {member.email || member.name} + + + + + )} +
+ + +
+ ); + })} +
+ )} +
+ )} + + { + setIsDialogOpen(open); + if (!open) setEditingConfig(null); + }} + config={editingConfig} + isGlobal={false} + searchSpaceId={searchSpaceId} + mode={editingConfig ? "edit" : "create"} + /> + + !open && setConfigToDelete(null)} + > + + + Delete Vision Model + + Are you sure you want to delete{" "} + {configToDelete?.name}? + + + + Cancel + + Delete + {isDeleting && } + + + + +
+ ); +} diff --git a/surfsense_web/components/shared/vision-config-dialog.tsx b/surfsense_web/components/shared/vision-config-dialog.tsx new file mode 100644 index 000000000..d69750316 --- /dev/null +++ b/surfsense_web/components/shared/vision-config-dialog.tsx @@ -0,0 +1,381 @@ +"use client"; + +import { useAtomValue } from "jotai"; +import { AlertCircle } from "lucide-react"; +import { useCallback, useEffect, useRef, useState } from "react"; +import { toast } from "sonner"; +import { updateLLMPreferencesMutationAtom } from "@/atoms/new-llm-config/new-llm-config-mutation.atoms"; +import { + createVisionLLMConfigMutationAtom, + updateVisionLLMConfigMutationAtom, +} from "@/atoms/vision-llm-config/vision-llm-config-mutation.atoms"; +import { Alert, AlertDescription } from "@/components/ui/alert"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Dialog, DialogContent, DialogTitle } from "@/components/ui/dialog"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Separator } from "@/components/ui/separator"; +import { Spinner } from "@/components/ui/spinner"; +import { VISION_PROVIDERS } from "@/contracts/enums/vision-providers"; +import type { + GlobalVisionLLMConfig, + VisionLLMConfig, + VisionProvider, +} from "@/contracts/types/new-llm-config.types"; + +interface VisionConfigDialogProps { + open: boolean; + onOpenChange: (open: boolean) => void; + config: VisionLLMConfig | GlobalVisionLLMConfig | null; + isGlobal: boolean; + searchSpaceId: number; + mode: "create" | "edit" | "view"; +} + +const INITIAL_FORM = { + name: "", + description: "", + provider: "", + model_name: "", + api_key: "", + api_base: "", + api_version: "", +}; + +export function VisionConfigDialog({ + open, + onOpenChange, + config, + isGlobal, + searchSpaceId, + mode, +}: VisionConfigDialogProps) { + const [isSubmitting, setIsSubmitting] = useState(false); + const [formData, setFormData] = useState(INITIAL_FORM); + const [scrollPos, setScrollPos] = useState<"top" | "middle" | "bottom">("top"); + const scrollRef = useRef(null); + + useEffect(() => { + if (open) { + if (mode === "edit" && config && !isGlobal) { + setFormData({ + name: config.name || "", + description: config.description || "", + provider: config.provider || "", + model_name: config.model_name || "", + api_key: (config as VisionLLMConfig).api_key || "", + api_base: config.api_base || "", + api_version: (config as VisionLLMConfig).api_version || "", + }); + } else if (mode === "create") { + setFormData(INITIAL_FORM); + } + setScrollPos("top"); + } + }, [open, mode, config, isGlobal]); + + const { mutateAsync: createConfig } = useAtomValue(createVisionLLMConfigMutationAtom); + const { mutateAsync: updateConfig } = useAtomValue(updateVisionLLMConfigMutationAtom); + const { mutateAsync: updatePreferences } = useAtomValue(updateLLMPreferencesMutationAtom); + + const handleScroll = useCallback((e: React.UIEvent) => { + const el = e.currentTarget; + const atTop = el.scrollTop <= 2; + const atBottom = el.scrollHeight - el.scrollTop - el.clientHeight <= 2; + setScrollPos(atTop ? "top" : atBottom ? "bottom" : "middle"); + }, []); + + const getTitle = () => { + if (mode === "create") return "Add Vision Model"; + if (isGlobal) return "View Global Vision Model"; + return "Edit Vision Model"; + }; + + const getSubtitle = () => { + if (mode === "create") return "Set up a new vision-capable LLM provider"; + if (isGlobal) return "Read-only global configuration"; + return "Update your vision model settings"; + }; + + const handleSubmit = useCallback(async () => { + setIsSubmitting(true); + try { + if (mode === "create") { + const result = await createConfig({ + name: formData.name, + provider: formData.provider as VisionProvider, + model_name: formData.model_name, + api_key: formData.api_key, + api_base: formData.api_base || undefined, + api_version: formData.api_version || undefined, + description: formData.description || undefined, + search_space_id: searchSpaceId, + }); + if (result?.id) { + await updatePreferences({ + search_space_id: searchSpaceId, + data: { vision_llm_config_id: result.id }, + }); + } + onOpenChange(false); + } else if (!isGlobal && config) { + await updateConfig({ + id: config.id, + data: { + name: formData.name, + description: formData.description || undefined, + provider: formData.provider as VisionProvider, + model_name: formData.model_name, + api_key: formData.api_key, + api_base: formData.api_base || undefined, + api_version: formData.api_version || undefined, + }, + }); + onOpenChange(false); + } + } catch (error) { + console.error("Failed to save vision config:", error); + toast.error("Failed to save vision model"); + } finally { + setIsSubmitting(false); + } + }, [ + mode, + isGlobal, + config, + formData, + searchSpaceId, + createConfig, + updateConfig, + updatePreferences, + onOpenChange, + ]); + + const handleUseGlobalConfig = useCallback(async () => { + if (!config || !isGlobal) return; + setIsSubmitting(true); + try { + await updatePreferences({ + search_space_id: searchSpaceId, + data: { vision_llm_config_id: config.id }, + }); + toast.success(`Now using ${config.name}`); + onOpenChange(false); + } catch (error) { + console.error("Failed to set vision model:", error); + toast.error("Failed to set vision model"); + } finally { + setIsSubmitting(false); + } + }, [config, isGlobal, searchSpaceId, updatePreferences, onOpenChange]); + + const isFormValid = formData.name && formData.provider && formData.model_name && formData.api_key; + const selectedProvider = VISION_PROVIDERS.find((p) => p.value === formData.provider); + + return ( + + e.preventDefault()} + > + {getTitle()} + +
+
+
+

{getTitle()}

+ {isGlobal && mode !== "create" && ( + + Global + + )} +
+

{getSubtitle()}

+ {config && mode !== "create" && ( +

{config.model_name}

+ )} +
+
+ +
+ {isGlobal && config && ( + <> + + + + Global configurations are read-only. To customize, create a new model. + + +
+
+
+
+ Name +
+

{config.name}

+
+ {config.description && ( +
+
+ Description +
+

{config.description}

+
+ )} +
+ +
+
+
+ Provider +
+

{config.provider}

+
+
+
+ Model +
+

{config.model_name}

+
+
+
+ + )} + + {(mode === "create" || (mode === "edit" && !isGlobal)) && ( +
+
+ + setFormData((p) => ({ ...p, name: e.target.value }))} + /> +
+ +
+ + setFormData((p) => ({ ...p, description: e.target.value }))} + /> +
+ + + +
+ + +
+ +
+ + setFormData((p) => ({ ...p, model_name: e.target.value }))} + /> +
+ +
+ + setFormData((p) => ({ ...p, api_key: e.target.value }))} + /> +
+ +
+ + setFormData((p) => ({ ...p, api_base: e.target.value }))} + /> +
+ + {formData.provider === "AZURE_OPENAI" && ( +
+ + setFormData((p) => ({ ...p, api_version: e.target.value }))} + /> +
+ )} +
+ )} +
+ +
+ + {mode === "create" || (mode === "edit" && !isGlobal) ? ( + + ) : isGlobal && config ? ( + + ) : null} +
+ + + ); +} From 035a4862f9c78fa75c3e209534e0b553148f12a5 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 19:29:24 +0200 Subject: [PATCH 19/28] Add Vision Models tab to settings dialog with i18n --- .../components/settings/search-space-settings-dialog.tsx | 9 ++++++++- surfsense_web/messages/en.json | 2 ++ surfsense_web/messages/es.json | 2 ++ surfsense_web/messages/hi.json | 2 ++ surfsense_web/messages/pt.json | 2 ++ surfsense_web/messages/zh.json | 2 ++ 6 files changed, 18 insertions(+), 1 deletion(-) diff --git a/surfsense_web/components/settings/search-space-settings-dialog.tsx b/surfsense_web/components/settings/search-space-settings-dialog.tsx index 47094d0c9..6573bc271 100644 --- a/surfsense_web/components/settings/search-space-settings-dialog.tsx +++ b/surfsense_web/components/settings/search-space-settings-dialog.tsx @@ -1,7 +1,7 @@ "use client"; import { useAtom } from "jotai"; -import { Bot, Brain, FileText, Globe, ImageIcon, MessageSquare, Shield } from "lucide-react"; +import { Bot, Brain, Eye, FileText, Globe, ImageIcon, MessageSquare, Shield } from "lucide-react"; import { useTranslations } from "next-intl"; import type React from "react"; import { searchSpaceSettingsDialogAtom } from "@/atoms/settings/settings-dialog.atoms"; @@ -13,6 +13,7 @@ import { ModelConfigManager } from "@/components/settings/model-config-manager"; import { PromptConfigManager } from "@/components/settings/prompt-config-manager"; import { RolesManager } from "@/components/settings/roles-manager"; import { SettingsDialog } from "@/components/settings/settings-dialog"; +import { VisionModelManager } from "@/components/settings/vision-model-manager"; interface SearchSpaceSettingsDialogProps { searchSpaceId: number; @@ -31,6 +32,11 @@ export function SearchSpaceSettingsDialog({ searchSpaceId }: SearchSpaceSettings label: t("nav_image_models"), icon: , }, + { + value: "vision-models", + label: t("nav_vision_models"), + icon: , + }, { value: "team-roles", label: t("nav_team_roles"), icon: }, { value: "prompts", @@ -45,6 +51,7 @@ export function SearchSpaceSettingsDialog({ searchSpaceId }: SearchSpaceSettings models: , roles: , "image-models": , + "vision-models": , "team-roles": , prompts: , "public-links": , diff --git a/surfsense_web/messages/en.json b/surfsense_web/messages/en.json index b67f9db22..a3a4e8853 100644 --- a/surfsense_web/messages/en.json +++ b/surfsense_web/messages/en.json @@ -738,6 +738,8 @@ "nav_role_assignments_desc": "Assign configs to agent roles", "nav_image_models": "Image Models", "nav_image_models_desc": "Configure image generation models", + "nav_vision_models": "Vision Models", + "nav_vision_models_desc": "Configure vision-capable LLM models", "nav_system_instructions": "System Instructions", "nav_system_instructions_desc": "SearchSpace-wide AI instructions", "nav_public_links": "Public Chat Links", diff --git a/surfsense_web/messages/es.json b/surfsense_web/messages/es.json index 5cf248a3a..fa620e271 100644 --- a/surfsense_web/messages/es.json +++ b/surfsense_web/messages/es.json @@ -738,6 +738,8 @@ "nav_role_assignments_desc": "Asignar configuraciones a roles de agente", "nav_image_models": "Modelos de imagen", "nav_image_models_desc": "Configurar modelos de generación de imágenes", + "nav_vision_models": "Modelos de visión", + "nav_vision_models_desc": "Configurar modelos LLM con capacidad de visión", "nav_system_instructions": "Instrucciones del sistema", "nav_system_instructions_desc": "Instrucciones de IA a nivel del espacio de búsqueda", "nav_public_links": "Enlaces de chat públicos", diff --git a/surfsense_web/messages/hi.json b/surfsense_web/messages/hi.json index 0e7194832..faeb4cb94 100644 --- a/surfsense_web/messages/hi.json +++ b/surfsense_web/messages/hi.json @@ -738,6 +738,8 @@ "nav_role_assignments_desc": "एजेंट भूमिकाओं को कॉन्फ़िगरेशन असाइन करें", "nav_image_models": "इमेज मॉडल", "nav_image_models_desc": "इमेज जनरेशन मॉडल कॉन्फ़िगर करें", + "nav_vision_models": "विज़न मॉडल", + "nav_vision_models_desc": "विज़न-सक्षम LLM मॉडल कॉन्फ़िगर करें", "nav_system_instructions": "सिस्टम निर्देश", "nav_system_instructions_desc": "सर्च स्पेस-व्यापी AI निर्देश", "nav_public_links": "सार्वजनिक चैट लिंक", diff --git a/surfsense_web/messages/pt.json b/surfsense_web/messages/pt.json index 00ae18eae..0bed7c6cc 100644 --- a/surfsense_web/messages/pt.json +++ b/surfsense_web/messages/pt.json @@ -738,6 +738,8 @@ "nav_role_assignments_desc": "Atribuir configurações a funções do agente", "nav_image_models": "Modelos de imagem", "nav_image_models_desc": "Configurar modelos de geração de imagens", + "nav_vision_models": "Modelos de visão", + "nav_vision_models_desc": "Configurar modelos LLM com capacidade de visão", "nav_system_instructions": "Instruções do sistema", "nav_system_instructions_desc": "Instruções de IA em nível do espaço de pesquisa", "nav_public_links": "Links de chat públicos", diff --git a/surfsense_web/messages/zh.json b/surfsense_web/messages/zh.json index a6f3b5b84..0d4f7e1c9 100644 --- a/surfsense_web/messages/zh.json +++ b/surfsense_web/messages/zh.json @@ -722,6 +722,8 @@ "nav_role_assignments_desc": "为代理角色分配配置", "nav_image_models": "图像模型", "nav_image_models_desc": "配置图像生成模型", + "nav_vision_models": "视觉模型", + "nav_vision_models_desc": "配置具有视觉能力的LLM模型", "nav_system_instructions": "系统指令", "nav_system_instructions_desc": "搜索空间级别的 AI 指令", "nav_public_links": "公开聊天链接", From e85c355592676d19bd47820981bf62c6ad8bdc1b Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 19:45:30 +0200 Subject: [PATCH 20/28] Add NEXT_PUBLIC_POSTHOG_KEY to desktop release CI and .env.example --- .github/workflows/desktop-release.yml | 1 + surfsense_web/.env.example | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/desktop-release.yml b/.github/workflows/desktop-release.yml index 491df0992..4d217562a 100644 --- a/.github/workflows/desktop-release.yml +++ b/.github/workflows/desktop-release.yml @@ -60,6 +60,7 @@ jobs: NEXT_PUBLIC_ZERO_CACHE_URL: ${{ vars.NEXT_PUBLIC_ZERO_CACHE_URL }} NEXT_PUBLIC_DEPLOYMENT_MODE: ${{ vars.NEXT_PUBLIC_DEPLOYMENT_MODE }} NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE: ${{ vars.NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE }} + NEXT_PUBLIC_POSTHOG_KEY: ${{ secrets.NEXT_PUBLIC_POSTHOG_KEY }} - name: Install desktop dependencies run: pnpm install diff --git a/surfsense_web/.env.example b/surfsense_web/.env.example index b674d8e9b..b448c1f71 100644 --- a/surfsense_web/.env.example +++ b/surfsense_web/.env.example @@ -7,4 +7,7 @@ NEXT_PUBLIC_ZERO_CACHE_URL=http://localhost:4848 DATABASE_URL=postgresql://postgres:[YOUR-PASSWORD]@db.sdsf.supabase.co:5432/postgres # Deployment mode (optional) -NEXT_PUBLIC_DEPLOYMENT_MODE="self-hosted" or "cloud" \ No newline at end of file +NEXT_PUBLIC_DEPLOYMENT_MODE="self-hosted" or "cloud" + +# PostHog analytics (optional, leave empty to disable) +NEXT_PUBLIC_POSTHOG_KEY= \ No newline at end of file From 8566b03c91b5a69efd8359c52889b6bb0118a657 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 20:18:42 +0200 Subject: [PATCH 21/28] Add PostHog analytics to desktop main process --- .github/workflows/desktop-release.yml | 2 + surfsense_desktop/.env | 4 ++ surfsense_desktop/scripts/build-electron.mjs | 6 +++ surfsense_desktop/src/main.ts | 11 ++++- surfsense_desktop/src/modules/analytics.ts | 46 +++++++++++++++++++ .../src/modules/autocomplete/index.ts | 4 ++ surfsense_desktop/src/modules/quick-ask.ts | 3 ++ 7 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 surfsense_desktop/src/modules/analytics.ts diff --git a/.github/workflows/desktop-release.yml b/.github/workflows/desktop-release.yml index 4d217562a..62ba5d445 100644 --- a/.github/workflows/desktop-release.yml +++ b/.github/workflows/desktop-release.yml @@ -71,6 +71,8 @@ jobs: working-directory: surfsense_desktop env: HOSTED_FRONTEND_URL: ${{ vars.HOSTED_FRONTEND_URL }} + POSTHOG_KEY: ${{ secrets.POSTHOG_KEY }} + POSTHOG_HOST: ${{ vars.POSTHOG_HOST }} - name: Package & Publish run: pnpm exec electron-builder ${{ matrix.platform }} --config electron-builder.yml --publish always -c.extraMetadata.version=${{ steps.version.outputs.VERSION }} diff --git a/surfsense_desktop/.env b/surfsense_desktop/.env index d053aac97..a0463a39d 100644 --- a/surfsense_desktop/.env +++ b/surfsense_desktop/.env @@ -4,3 +4,7 @@ # The hosted web frontend URL. Used to intercept OAuth redirects and keep them # inside the desktop app. Set to your production frontend domain. HOSTED_FRONTEND_URL=https://surfsense.net + +# PostHog analytics (leave empty to disable) +POSTHOG_KEY= +POSTHOG_HOST=https://us.i.posthog.com diff --git a/surfsense_desktop/scripts/build-electron.mjs b/surfsense_desktop/scripts/build-electron.mjs index 9f507ea37..bfce6a9ad 100644 --- a/surfsense_desktop/scripts/build-electron.mjs +++ b/surfsense_desktop/scripts/build-electron.mjs @@ -111,6 +111,12 @@ async function buildElectron() { 'process.env.HOSTED_FRONTEND_URL': JSON.stringify( process.env.HOSTED_FRONTEND_URL || desktopEnv.HOSTED_FRONTEND_URL || 'https://surfsense.net' ), + 'process.env.POSTHOG_KEY': JSON.stringify( + process.env.POSTHOG_KEY || desktopEnv.POSTHOG_KEY || '' + ), + 'process.env.POSTHOG_HOST': JSON.stringify( + process.env.POSTHOG_HOST || desktopEnv.POSTHOG_HOST || 'https://us.i.posthog.com' + ), }, }; diff --git a/surfsense_desktop/src/main.ts b/surfsense_desktop/src/main.ts index 95b0359c8..231553f9a 100644 --- a/surfsense_desktop/src/main.ts +++ b/surfsense_desktop/src/main.ts @@ -12,6 +12,7 @@ import { registerAutocomplete, unregisterAutocomplete } from './modules/autocomp import { registerFolderWatcher, unregisterFolderWatcher } from './modules/folder-watcher'; import { registerIpcHandlers } from './ipc/handlers'; import { createTray, destroyTray } from './modules/tray'; +import { initAnalytics, shutdownAnalytics, trackEvent } from './modules/analytics'; registerGlobalErrorHandlers(); @@ -22,6 +23,8 @@ if (!setupDeepLinks()) { registerIpcHandlers(); app.whenReady().then(async () => { + initAnalytics(); + trackEvent('desktop_app_launched'); setupMenu(); try { await startNextServer(); @@ -70,9 +73,15 @@ app.on('before-quit', () => { isQuitting = true; }); -app.on('will-quit', () => { +let didCleanup = false; +app.on('will-quit', async (e) => { + if (didCleanup) return; + didCleanup = true; + e.preventDefault(); unregisterQuickAsk(); unregisterAutocomplete(); unregisterFolderWatcher(); destroyTray(); + await shutdownAnalytics(); + app.exit(); }); diff --git a/surfsense_desktop/src/modules/analytics.ts b/surfsense_desktop/src/modules/analytics.ts new file mode 100644 index 000000000..8f64c1bd8 --- /dev/null +++ b/surfsense_desktop/src/modules/analytics.ts @@ -0,0 +1,46 @@ +import { PostHog } from 'posthog-node'; +import { machineIdSync } from 'node-machine-id'; +import { app } from 'electron'; + +let client: PostHog | null = null; +let distinctId = ''; + +export function initAnalytics(): void { + const key = process.env.POSTHOG_KEY; + if (!key) return; + + try { + distinctId = machineIdSync(true); + } catch { + return; + } + + client = new PostHog(key, { + host: process.env.POSTHOG_HOST || 'https://us.i.posthog.com', + flushAt: 20, + flushInterval: 10000, + }); +} + +export function trackEvent(event: string, properties?: Record): void { + if (!client) return; + + client.capture({ + distinctId, + event, + properties: { + platform: 'desktop', + app_version: app.getVersion(), + os: process.platform, + ...properties, + }, + }); +} + +export async function shutdownAnalytics(): Promise { + if (!client) return; + + const timeout = new Promise((resolve) => setTimeout(resolve, 3000)); + await Promise.race([client.shutdown(), timeout]); + client = null; +} diff --git a/surfsense_desktop/src/modules/autocomplete/index.ts b/surfsense_desktop/src/modules/autocomplete/index.ts index cb09a42e1..d4eb727fd 100644 --- a/surfsense_desktop/src/modules/autocomplete/index.ts +++ b/surfsense_desktop/src/modules/autocomplete/index.ts @@ -6,6 +6,7 @@ import { captureScreen } from './screenshot'; import { createSuggestionWindow, destroySuggestion, getSuggestionWindow } from './suggestion-window'; import { getShortcuts } from '../shortcuts'; import { getActiveSearchSpaceId } from '../active-search-space'; +import { trackEvent } from '../analytics'; let currentShortcut = ''; let autocompleteEnabled = true; @@ -41,6 +42,7 @@ async function triggerAutocomplete(): Promise { console.warn('[autocomplete] No active search space. Select a search space first.'); return; } + trackEvent('desktop_autocomplete_triggered', { search_space_id: searchSpaceId }); const cursor = screen.getCursorScreenPoint(); const win = createSuggestionWindow(cursor.x, cursor.y); @@ -87,9 +89,11 @@ function registerIpcHandlers(): void { ipcRegistered = true; ipcMain.handle(IPC_CHANNELS.ACCEPT_SUGGESTION, async (_event, text: string) => { + trackEvent('desktop_autocomplete_accepted'); await acceptAndInject(text); }); ipcMain.handle(IPC_CHANNELS.DISMISS_SUGGESTION, () => { + trackEvent('desktop_autocomplete_dismissed'); destroySuggestion(); }); ipcMain.handle(IPC_CHANNELS.SET_AUTOCOMPLETE_ENABLED, (_event, enabled: boolean) => { diff --git a/surfsense_desktop/src/modules/quick-ask.ts b/surfsense_desktop/src/modules/quick-ask.ts index d5a2a9c2e..d700b421a 100644 --- a/surfsense_desktop/src/modules/quick-ask.ts +++ b/surfsense_desktop/src/modules/quick-ask.ts @@ -5,6 +5,7 @@ import { checkAccessibilityPermission, getFrontmostApp, simulateCopy, simulatePa import { getServerPort } from './server'; import { getShortcuts } from './shortcuts'; import { getActiveSearchSpaceId } from './active-search-space'; +import { trackEvent } from './analytics'; let currentShortcut = ''; let quickAskWindow: BrowserWindow | null = null; @@ -120,6 +121,7 @@ async function quickAskHandler(): Promise { sourceApp = getFrontmostApp(); console.log('[quick-ask] Source app:', sourceApp, '| Opening Quick Assist with', text.length, 'chars', selected ? '(selected)' : text ? '(clipboard fallback)' : '(empty)'); + trackEvent('desktop_quick_ask_opened', { has_selected_text: !!selected }); openQuickAsk(text); } @@ -151,6 +153,7 @@ function registerIpcHandlers(): void { if (!checkAccessibilityPermission()) return; + trackEvent('desktop_quick_ask_replaced'); clipboard.writeText(text); destroyQuickAsk(); From 556646fe9740c3e8f05aae9722f51b44dc5d55f0 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 20:20:56 +0200 Subject: [PATCH 22/28] Use assets.surfsense.com as PostHog host --- surfsense_desktop/.env | 2 +- surfsense_desktop/scripts/build-electron.mjs | 2 +- surfsense_desktop/src/modules/analytics.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/surfsense_desktop/.env b/surfsense_desktop/.env index a0463a39d..e127b99e0 100644 --- a/surfsense_desktop/.env +++ b/surfsense_desktop/.env @@ -7,4 +7,4 @@ HOSTED_FRONTEND_URL=https://surfsense.net # PostHog analytics (leave empty to disable) POSTHOG_KEY= -POSTHOG_HOST=https://us.i.posthog.com +POSTHOG_HOST=https://assets.surfsense.com diff --git a/surfsense_desktop/scripts/build-electron.mjs b/surfsense_desktop/scripts/build-electron.mjs index bfce6a9ad..90d76ef7a 100644 --- a/surfsense_desktop/scripts/build-electron.mjs +++ b/surfsense_desktop/scripts/build-electron.mjs @@ -115,7 +115,7 @@ async function buildElectron() { process.env.POSTHOG_KEY || desktopEnv.POSTHOG_KEY || '' ), 'process.env.POSTHOG_HOST': JSON.stringify( - process.env.POSTHOG_HOST || desktopEnv.POSTHOG_HOST || 'https://us.i.posthog.com' + process.env.POSTHOG_HOST || desktopEnv.POSTHOG_HOST || 'https://assets.surfsense.com' ), }, }; diff --git a/surfsense_desktop/src/modules/analytics.ts b/surfsense_desktop/src/modules/analytics.ts index 8f64c1bd8..ee6ae8722 100644 --- a/surfsense_desktop/src/modules/analytics.ts +++ b/surfsense_desktop/src/modules/analytics.ts @@ -16,7 +16,7 @@ export function initAnalytics(): void { } client = new PostHog(key, { - host: process.env.POSTHOG_HOST || 'https://us.i.posthog.com', + host: process.env.POSTHOG_HOST || 'https://assets.surfsense.com', flushAt: 20, flushInterval: 10000, }); From 0be3c796354347e8a1e570233a7f560cbd22181c Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 20:22:00 +0200 Subject: [PATCH 23/28] Guard trackEvent with try-catch --- surfsense_desktop/src/modules/analytics.ts | 24 +++++++++++++--------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/surfsense_desktop/src/modules/analytics.ts b/surfsense_desktop/src/modules/analytics.ts index ee6ae8722..0bbcb3026 100644 --- a/surfsense_desktop/src/modules/analytics.ts +++ b/surfsense_desktop/src/modules/analytics.ts @@ -25,16 +25,20 @@ export function initAnalytics(): void { export function trackEvent(event: string, properties?: Record): void { if (!client) return; - client.capture({ - distinctId, - event, - properties: { - platform: 'desktop', - app_version: app.getVersion(), - os: process.platform, - ...properties, - }, - }); + try { + client.capture({ + distinctId, + event, + properties: { + platform: 'desktop', + app_version: app.getVersion(), + os: process.platform, + ...properties, + }, + }); + } catch { + // Analytics should never break the app + } } export async function shutdownAnalytics(): Promise { From c5646eef6620ef673b8dc7bf136068112a619b03 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 20:28:07 +0200 Subject: [PATCH 24/28] Formatting --- .../[search_space_id]/client-layout.tsx | 13 ++-- .../components/DesktopContent.tsx | 78 +++++++++++-------- surfsense_web/app/desktop/login/page.tsx | 4 +- surfsense_web/app/desktop/suggestion/page.tsx | 4 +- surfsense_web/components/TokenHandler.tsx | 2 +- .../components/desktop/shortcut-recorder.tsx | 4 +- 6 files changed, 59 insertions(+), 46 deletions(-) diff --git a/surfsense_web/app/dashboard/[search_space_id]/client-layout.tsx b/surfsense_web/app/dashboard/[search_space_id]/client-layout.tsx index 16af9ac6b..eceb46231 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/client-layout.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/client-layout.tsx @@ -154,11 +154,14 @@ export function DashboardClientLayout({ // Sync to Electron store if stored value is null (first navigation) if (electronAPI?.setActiveSearchSpace) { - electronAPI.getActiveSearchSpace?.().then((stored) => { - if (!stored) { - electronAPI.setActiveSearchSpace!(activeSeacrhSpaceId); - } - }).catch(() => {}); + electronAPI + .getActiveSearchSpace?.() + .then((stored) => { + if (!stored) { + electronAPI.setActiveSearchSpace!(activeSeacrhSpaceId); + } + }) + .catch(() => {}); } }, [search_space_id, setActiveSearchSpaceIdState, electronAPI]); diff --git a/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/DesktopContent.tsx b/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/DesktopContent.tsx index 596ed3e8b..c3f457f96 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/DesktopContent.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/DesktopContent.tsx @@ -6,12 +6,18 @@ import { toast } from "sonner"; import { DEFAULT_SHORTCUTS, ShortcutRecorder } from "@/components/desktop/shortcut-recorder"; import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; import { Label } from "@/components/ui/label"; -import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; import { Spinner } from "@/components/ui/spinner"; import { Switch } from "@/components/ui/switch"; +import type { SearchSpace } from "@/contracts/types/search-space.types"; import { useElectronAPI } from "@/hooks/use-platform"; import { searchSpacesApiService } from "@/lib/apis/search-spaces-api.service"; -import type { SearchSpace } from "@/contracts/types/search-space.types"; export function DesktopContent() { const api = useElectronAPI(); @@ -82,7 +88,10 @@ export function DesktopContent() { await api.setAutocompleteEnabled(checked); }; - const updateShortcut = (key: "generalAssist" | "quickAsk" | "autocomplete", accelerator: string) => { + const updateShortcut = ( + key: "generalAssist" | "quickAsk" | "autocomplete", + accelerator: string + ) => { setShortcuts((prev) => { const updated = { ...prev, [key]: accelerator }; api.setShortcuts?.({ [key]: accelerator }).catch(() => { @@ -110,7 +119,8 @@ export function DesktopContent() { Default Search Space - Choose which search space General Assist, Quick Assist, and Extreme Assist operate against. + Choose which search space General Assist, Quick Assist, and Extreme Assist operate + against. @@ -128,7 +138,9 @@ export function DesktopContent() { ) : ( -

No search spaces found. Create one first.

+

+ No search spaces found. Create one first. +

)} @@ -143,34 +155,34 @@ export function DesktopContent() { {shortcutsLoaded ? ( -
- updateShortcut("generalAssist", accel)} - onReset={() => resetShortcut("generalAssist")} - defaultValue={DEFAULT_SHORTCUTS.generalAssist} - label="General Assist" - description="Launch SurfSense instantly from any application" - icon={Rocket} - /> - updateShortcut("quickAsk", accel)} - onReset={() => resetShortcut("quickAsk")} - defaultValue={DEFAULT_SHORTCUTS.quickAsk} - label="Quick Assist" - description="Select text anywhere, then ask AI to explain, rewrite, or act on it" - icon={Zap} - /> - updateShortcut("autocomplete", accel)} - onReset={() => resetShortcut("autocomplete")} - defaultValue={DEFAULT_SHORTCUTS.autocomplete} - label="Extreme Assist" - description="AI drafts text using your screen context and knowledge base" - icon={BrainCog} - /> +
+ updateShortcut("generalAssist", accel)} + onReset={() => resetShortcut("generalAssist")} + defaultValue={DEFAULT_SHORTCUTS.generalAssist} + label="General Assist" + description="Launch SurfSense instantly from any application" + icon={Rocket} + /> + updateShortcut("quickAsk", accel)} + onReset={() => resetShortcut("quickAsk")} + defaultValue={DEFAULT_SHORTCUTS.quickAsk} + label="Quick Assist" + description="Select text anywhere, then ask AI to explain, rewrite, or act on it" + icon={Zap} + /> + updateShortcut("autocomplete", accel)} + onReset={() => resetShortcut("autocomplete")} + defaultValue={DEFAULT_SHORTCUTS.autocomplete} + label="Extreme Assist" + description="AI drafts text using your screen context and knowledge base" + icon={BrainCog} + />

Click a shortcut and press a new key combination to change it.

diff --git a/surfsense_web/app/desktop/login/page.tsx b/surfsense_web/app/desktop/login/page.tsx index 744680010..8f68d20c1 100644 --- a/surfsense_web/app/desktop/login/page.tsx +++ b/surfsense_web/app/desktop/login/page.tsx @@ -139,9 +139,7 @@ export default function DesktopLoginPage() { height={48} priority /> -

- Welcome to SurfSense Desktop -

+

Welcome to SurfSense Desktop

Configure shortcuts, then sign in to get started.

diff --git a/surfsense_web/app/desktop/suggestion/page.tsx b/surfsense_web/app/desktop/suggestion/page.tsx index e458f6615..0815ba622 100644 --- a/surfsense_web/app/desktop/suggestion/page.tsx +++ b/surfsense_web/app/desktop/suggestion/page.tsx @@ -297,9 +297,7 @@ export default function SuggestionPage() { const isExpanded = expandedOption === index; const needsTruncation = option.length > TRUNCATE_LENGTH; const displayText = - needsTruncation && !isExpanded - ? option.slice(0, TRUNCATE_LENGTH) + "…" - : option; + needsTruncation && !isExpanded ? option.slice(0, TRUNCATE_LENGTH) + "…" : option; return (
{recording ? ( - Press keys… + + Press keys… + ) : ( )} From 13625acdd5ceddeb66a03d7aaeaf1cc9e1dc16b1 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 20:47:17 +0200 Subject: [PATCH 25/28] Add vision model tab to chat page model selector --- .../components/new-chat/chat-header.tsx | 44 +++ .../components/new-chat/model-selector.tsx | 304 +++++++++++++++++- 2 files changed, 339 insertions(+), 9 deletions(-) diff --git a/surfsense_web/components/new-chat/chat-header.tsx b/surfsense_web/components/new-chat/chat-header.tsx index 3263a2b07..0c5253c6c 100644 --- a/surfsense_web/components/new-chat/chat-header.tsx +++ b/surfsense_web/components/new-chat/chat-header.tsx @@ -3,11 +3,14 @@ import { useCallback, useState } from "react"; import { ImageConfigDialog } from "@/components/shared/image-config-dialog"; import { ModelConfigDialog } from "@/components/shared/model-config-dialog"; +import { VisionConfigDialog } from "@/components/shared/vision-config-dialog"; import type { GlobalImageGenConfig, GlobalNewLLMConfig, + GlobalVisionLLMConfig, ImageGenerationConfig, NewLLMConfigPublic, + VisionLLMConfig, } from "@/contracts/types/new-llm-config.types"; import { ModelSelector } from "./model-selector"; @@ -33,6 +36,14 @@ export function ChatHeader({ searchSpaceId, className }: ChatHeaderProps) { const [isImageGlobal, setIsImageGlobal] = useState(false); const [imageDialogMode, setImageDialogMode] = useState<"create" | "edit" | "view">("view"); + // Vision config dialog state + const [visionDialogOpen, setVisionDialogOpen] = useState(false); + const [selectedVisionConfig, setSelectedVisionConfig] = useState< + VisionLLMConfig | GlobalVisionLLMConfig | null + >(null); + const [isVisionGlobal, setIsVisionGlobal] = useState(false); + const [visionDialogMode, setVisionDialogMode] = useState<"create" | "edit" | "view">("view"); + // LLM handlers const handleEditLLMConfig = useCallback( (config: NewLLMConfigPublic | GlobalNewLLMConfig, global: boolean) => { @@ -79,6 +90,29 @@ export function ChatHeader({ searchSpaceId, className }: ChatHeaderProps) { if (!open) setSelectedImageConfig(null); }, []); + // Vision model handlers + const handleAddVisionModel = useCallback(() => { + setSelectedVisionConfig(null); + setIsVisionGlobal(false); + setVisionDialogMode("create"); + setVisionDialogOpen(true); + }, []); + + const handleEditVisionConfig = useCallback( + (config: VisionLLMConfig | GlobalVisionLLMConfig, global: boolean) => { + setSelectedVisionConfig(config); + setIsVisionGlobal(global); + setVisionDialogMode(global ? "view" : "edit"); + setVisionDialogOpen(true); + }, + [] + ); + + const handleVisionDialogClose = useCallback((open: boolean) => { + setVisionDialogOpen(open); + if (!open) setSelectedVisionConfig(null); + }, []); + return (
+
); } diff --git a/surfsense_web/components/new-chat/model-selector.tsx b/surfsense_web/components/new-chat/model-selector.tsx index 39f88f794..46b4a2c3a 100644 --- a/surfsense_web/components/new-chat/model-selector.tsx +++ b/surfsense_web/components/new-chat/model-selector.tsx @@ -1,7 +1,7 @@ "use client"; import { useAtomValue } from "jotai"; -import { Bot, Check, ChevronDown, Edit3, ImageIcon, Plus, Search, Zap } from "lucide-react"; +import { Bot, Check, ChevronDown, Edit3, Eye, ImageIcon, Plus, Search, Zap } from "lucide-react"; import { type UIEvent, useCallback, useMemo, useState } from "react"; import { toast } from "sonner"; import { @@ -15,6 +15,10 @@ import { newLLMConfigsAtom, } from "@/atoms/new-llm-config/new-llm-config-query.atoms"; import { activeSearchSpaceIdAtom } from "@/atoms/search-spaces/search-space-query.atoms"; +import { + globalVisionLLMConfigsAtom, + visionLLMConfigsAtom, +} from "@/atoms/vision-llm-config/vision-llm-config-query.atoms"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { @@ -32,8 +36,10 @@ import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import type { GlobalImageGenConfig, GlobalNewLLMConfig, + GlobalVisionLLMConfig, ImageGenerationConfig, NewLLMConfigPublic, + VisionLLMConfig, } from "@/contracts/types/new-llm-config.types"; import { getProviderIcon } from "@/lib/provider-icons"; import { cn } from "@/lib/utils"; @@ -43,6 +49,8 @@ interface ModelSelectorProps { onAddNewLLM: () => void; onEditImage?: (config: ImageGenerationConfig | GlobalImageGenConfig, isGlobal: boolean) => void; onAddNewImage?: () => void; + onEditVision?: (config: VisionLLMConfig | GlobalVisionLLMConfig, isGlobal: boolean) => void; + onAddNewVision?: () => void; className?: string; } @@ -51,14 +59,18 @@ export function ModelSelector({ onAddNewLLM, onEditImage, onAddNewImage, + onEditVision, + onAddNewVision, className, }: ModelSelectorProps) { const [open, setOpen] = useState(false); - const [activeTab, setActiveTab] = useState<"llm" | "image">("llm"); + const [activeTab, setActiveTab] = useState<"llm" | "image" | "vision">("llm"); const [llmSearchQuery, setLlmSearchQuery] = useState(""); const [imageSearchQuery, setImageSearchQuery] = useState(""); + const [visionSearchQuery, setVisionSearchQuery] = useState(""); const [llmScrollPos, setLlmScrollPos] = useState<"top" | "middle" | "bottom">("top"); const [imageScrollPos, setImageScrollPos] = useState<"top" | "middle" | "bottom">("top"); + const [visionScrollPos, setVisionScrollPos] = useState<"top" | "middle" | "bottom">("top"); const handleListScroll = useCallback( (setter: typeof setLlmScrollPos) => (e: UIEvent) => { const el = e.currentTarget; @@ -82,8 +94,21 @@ export function ModelSelector({ useAtomValue(globalImageGenConfigsAtom); const { data: imageUserConfigs, isLoading: imageUserLoading } = useAtomValue(imageGenConfigsAtom); + // Vision data + const { data: visionGlobalConfigs, isLoading: visionGlobalLoading } = useAtomValue( + globalVisionLLMConfigsAtom + ); + const { data: visionUserConfigs, isLoading: visionUserLoading } = + useAtomValue(visionLLMConfigsAtom); + const isLoading = - llmUserLoading || llmGlobalLoading || prefsLoading || imageGlobalLoading || imageUserLoading; + llmUserLoading || + llmGlobalLoading || + prefsLoading || + imageGlobalLoading || + imageUserLoading || + visionGlobalLoading || + visionUserLoading; // ─── LLM current config ─── const currentLLMConfig = useMemo(() => { @@ -116,6 +141,24 @@ export function ModelSelector({ ); }, [currentImageConfig]); + // ─── Vision current config ─── + const currentVisionConfig = useMemo(() => { + if (!preferences) return null; + const id = preferences.vision_llm_config_id; + if (id === null || id === undefined) return null; + const globalMatch = visionGlobalConfigs?.find((c) => c.id === id); + if (globalMatch) return globalMatch; + return visionUserConfigs?.find((c) => c.id === id) ?? null; + }, [preferences, visionGlobalConfigs, visionUserConfigs]); + + const isVisionAutoMode = useMemo(() => { + return ( + currentVisionConfig && + "is_auto_mode" in currentVisionConfig && + currentVisionConfig.is_auto_mode + ); + }, [currentVisionConfig]); + // ─── LLM filtering ─── const filteredLLMGlobal = useMemo(() => { if (!llmGlobalConfigs) return []; @@ -170,6 +213,33 @@ export function ModelSelector({ const totalImageModels = (imageGlobalConfigs?.length ?? 0) + (imageUserConfigs?.length ?? 0); + // ─── Vision filtering ─── + const filteredVisionGlobal = useMemo(() => { + if (!visionGlobalConfigs) return []; + if (!visionSearchQuery) return visionGlobalConfigs; + const q = visionSearchQuery.toLowerCase(); + return visionGlobalConfigs.filter( + (c) => + c.name.toLowerCase().includes(q) || + c.model_name.toLowerCase().includes(q) || + c.provider.toLowerCase().includes(q) + ); + }, [visionGlobalConfigs, visionSearchQuery]); + + const filteredVisionUser = useMemo(() => { + if (!visionUserConfigs) return []; + if (!visionSearchQuery) return visionUserConfigs; + const q = visionSearchQuery.toLowerCase(); + return visionUserConfigs.filter( + (c) => + c.name.toLowerCase().includes(q) || + c.model_name.toLowerCase().includes(q) || + c.provider.toLowerCase().includes(q) + ); + }, [visionUserConfigs, visionSearchQuery]); + + const totalVisionModels = (visionGlobalConfigs?.length ?? 0) + (visionUserConfigs?.length ?? 0); + // ─── Handlers ─── const handleSelectLLM = useCallback( async (config: NewLLMConfigPublic | GlobalNewLLMConfig) => { @@ -229,6 +299,30 @@ export function ModelSelector({ [currentImageConfig, searchSpaceId, updatePreferences] ); + const handleSelectVision = useCallback( + async (configId: number) => { + if (currentVisionConfig?.id === configId) { + setOpen(false); + return; + } + if (!searchSpaceId) { + toast.error("No search space selected"); + return; + } + try { + await updatePreferences({ + search_space_id: Number(searchSpaceId), + data: { vision_llm_config_id: configId }, + }); + toast.success("Vision model updated"); + setOpen(false); + } catch { + toast.error("Failed to switch vision model"); + } + }, + [currentVisionConfig, searchSpaceId, updatePreferences] + ); + return ( @@ -282,6 +376,23 @@ export function ModelSelector({ ) : ( )} + + {/* Divider */} +
+ + {/* Vision section */} + {currentVisionConfig ? ( + <> + {getProviderIcon(currentVisionConfig.provider, { + isAutoMode: isVisionAutoMode ?? false, + })} + + {currentVisionConfig.name} + + + ) : ( + + )} )} @@ -295,25 +406,32 @@ export function ModelSelector({ > setActiveTab(v as "llm" | "image")} + onValueChange={(v) => setActiveTab(v as "llm" | "image" | "vision")} className="w-full" >
- + - + LLM - + Image + + + Vision +
@@ -676,6 +794,174 @@ export function ModelSelector({ + + {/* ─── Vision Tab ─── */} + + + {totalVisionModels > 3 && ( +
+ +
+ )} + + +
+ +

No vision models found

+

Try a different search term

+
+ + + {filteredVisionGlobal.length > 0 && ( + +
+ Global Vision Models +
+ {filteredVisionGlobal.map((config) => { + const isSelected = currentVisionConfig?.id === config.id; + const isAuto = "is_auto_mode" in config && config.is_auto_mode; + return ( + handleSelectVision(config.id)} + className={cn( + "mx-2 rounded-lg mb-1 cursor-pointer group transition-all hover:bg-accent/50 dark:hover:bg-white/[0.06]", + isSelected && "bg-accent/80 dark:bg-white/[0.06]" + )} + > +
+
+ {getProviderIcon(config.provider, { isAutoMode: isAuto })} +
+
+
+ {config.name} + {isAuto && ( + + Recommended + + )} + {isSelected && } +
+ + {isAuto ? "Auto Mode" : config.model_name} + +
+ {onEditVision && !isAuto && ( + + )} +
+ + ); + })} + + )} + + {filteredVisionUser.length > 0 && ( + <> + {filteredVisionGlobal.length > 0 && ( + + )} + +
+ Your Vision Models +
+ {filteredVisionUser.map((config) => { + const isSelected = currentVisionConfig?.id === config.id; + return ( + handleSelectVision(config.id)} + className={cn( + "mx-2 rounded-lg mb-1 cursor-pointer group transition-all hover:bg-accent/50 dark:hover:bg-white/[0.06]", + isSelected && "bg-accent/80 dark:bg-white/[0.06]" + )} + > +
+
{getProviderIcon(config.provider)}
+
+
+ {config.name} + {isSelected && ( + + )} +
+ + {config.model_name} + +
+ {onEditVision && ( + + )} +
+ + ); + })} + + + )} + + {onAddNewVision && ( +
+ +
+ )} + + + From 087b1498431ec60943a7d680af1f198876dc019b Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 21:06:11 +0200 Subject: [PATCH 26/28] Add setup prompt in overlay when vision model not configured --- surfsense_web/app/desktop/suggestion/page.tsx | 74 +++++++++++++++---- .../app/desktop/suggestion/suggestion.css | 60 +++++++++++++++ 2 files changed, 120 insertions(+), 14 deletions(-) diff --git a/surfsense_web/app/desktop/suggestion/page.tsx b/surfsense_web/app/desktop/suggestion/page.tsx index 0815ba622..46e388568 100644 --- a/surfsense_web/app/desktop/suggestion/page.tsx +++ b/surfsense_web/app/desktop/suggestion/page.tsx @@ -27,24 +27,32 @@ interface AgentStep { items: string[]; } -function friendlyError(raw: string | number): string { +type FriendlyError = { message: string; isSetup?: boolean }; + +function friendlyError(raw: string | number): FriendlyError { if (typeof raw === "number") { - if (raw === 401) return "Please sign in to use suggestions."; - if (raw === 403) return "You don\u2019t have permission for this."; - if (raw === 404) return "Suggestion service not found. Is the backend running?"; - if (raw >= 500) return "Something went wrong on the server. Try again."; - return "Something went wrong. Try again."; + if (raw === 401) return { message: "Please sign in to use suggestions." }; + if (raw === 403) return { message: "You don\u2019t have permission for this." }; + if (raw === 404) return { message: "Suggestion service not found. Is the backend running?" }; + if (raw >= 500) return { message: "Something went wrong on the server. Try again." }; + return { message: "Something went wrong. Try again." }; } const lower = raw.toLowerCase(); if (lower.includes("not authenticated") || lower.includes("unauthorized")) - return "Please sign in to use suggestions."; + return { message: "Please sign in to use suggestions." }; if (lower.includes("no vision llm configured") || lower.includes("no llm configured")) - return "No Vision LLM configured. Set one in search space settings."; + return { + message: "Configure a vision-capable model (e.g. GPT-4o, Gemini) to enable autocomplete.", + isSetup: true, + }; if (lower.includes("does not support vision")) - return "Selected model doesn\u2019t support vision. Set a vision-capable model in settings."; + return { + message: "The selected model doesn\u2019t support vision. Choose a vision-capable model.", + isSetup: true, + }; if (lower.includes("fetch") || lower.includes("network") || lower.includes("econnrefused")) - return "Can\u2019t reach the server. Check your connection."; - return "Something went wrong. Try again."; + return { message: "Can\u2019t reach the server. Check your connection." }; + return { message: "Something went wrong. Try again." }; } const AUTO_DISMISS_MS = 3000; @@ -76,7 +84,7 @@ export default function SuggestionPage() { const api = useElectronAPI(); const [options, setOptions] = useState([]); const [isLoading, setIsLoading] = useState(true); - const [error, setError] = useState(null); + const [error, setError] = useState(null); const [steps, setSteps] = useState([]); const [expandedOption, setExpandedOption] = useState(null); const abortRef = useRef(null); @@ -90,7 +98,7 @@ export default function SuggestionPage() { }, [api]); useEffect(() => { - if (!error) return; + if (!error || error.isSetup) return; const timer = setTimeout(() => { api?.dismissSuggestion?.(); }, AUTO_DISMISS_MS); @@ -233,9 +241,47 @@ export default function SuggestionPage() { } if (error) { + if (error.isSetup) { + return ( +
+
+ +
+
+ Vision Model Required + {error.message} + Settings → Vision Models +
+ +
+ ); + } return (
- {error} + {error.message}
); } diff --git a/surfsense_web/app/desktop/suggestion/suggestion.css b/surfsense_web/app/desktop/suggestion/suggestion.css index f5471cf37..fd5ec5d3b 100644 --- a/surfsense_web/app/desktop/suggestion/suggestion.css +++ b/surfsense_web/app/desktop/suggestion/suggestion.css @@ -117,6 +117,66 @@ body:has(.suggestion-body) { font-size: 12px; } +/* --- Setup prompt (vision model not configured) --- */ + +.suggestion-setup { + display: flex; + flex-direction: row; + align-items: flex-start; + gap: 10px; + border-color: #3b2d6b; + padding: 10px 14px; +} + +.setup-icon { + flex-shrink: 0; + margin-top: 1px; +} + +.setup-content { + display: flex; + flex-direction: column; + gap: 3px; + min-width: 0; +} + +.setup-title { + font-size: 13px; + font-weight: 600; + color: #c4b5fd; +} + +.setup-message { + font-size: 11.5px; + color: #a1a1aa; + line-height: 1.4; +} + +.setup-hint { + font-size: 10.5px; + color: #7c6dac; + margin-top: 2px; +} + +.setup-dismiss { + flex-shrink: 0; + align-self: flex-start; + background: none; + border: none; + color: #6b6b7b; + font-size: 14px; + cursor: pointer; + padding: 2px 4px; + line-height: 1; + border-radius: 4px; + transition: color 0.15s, background 0.15s; +} + +.setup-dismiss:hover { + color: #c4b5fd; + background: rgba(124, 109, 172, 0.15); +} + /* --- Agent activity indicator --- */ .agent-activity { From 36b8a84b0b6353e4bf608bd19fde2ff729faca2f Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 21:55:58 +0200 Subject: [PATCH 27/28] Add vision LLM config examples to global_llm_config.example.yaml --- .../app/config/global_llm_config.example.yaml | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/surfsense_backend/app/config/global_llm_config.example.yaml b/surfsense_backend/app/config/global_llm_config.example.yaml index 49a8d0295..e382fdc74 100644 --- a/surfsense_backend/app/config/global_llm_config.example.yaml +++ b/surfsense_backend/app/config/global_llm_config.example.yaml @@ -263,6 +263,82 @@ global_image_generation_configs: # rpm: 30 # litellm_params: {} +# ============================================================================= +# Vision LLM Configuration +# ============================================================================= +# These configurations power the vision autocomplete feature (screenshot analysis). +# Only vision-capable models should be used here (e.g. GPT-4o, Gemini Pro, Claude 3). +# Supported providers: OpenAI, Anthropic, Google, Azure OpenAI, Vertex AI, Bedrock, +# xAI, OpenRouter, Ollama, Groq, Together AI, Fireworks AI, DeepSeek, Mistral, Custom +# +# Auto mode (ID 0) uses LiteLLM Router for load balancing across all vision configs. + +# Router Settings for Vision LLM Auto Mode +vision_llm_router_settings: + routing_strategy: "usage-based-routing" + num_retries: 3 + allowed_fails: 3 + cooldown_time: 60 + +global_vision_llm_configs: + # Example: OpenAI GPT-4o (recommended for vision) + - id: -1 + name: "Global GPT-4o Vision" + description: "OpenAI's GPT-4o with strong vision capabilities" + provider: "OPENAI" + model_name: "gpt-4o" + api_key: "sk-your-openai-api-key-here" + api_base: "" + rpm: 500 + tpm: 100000 + litellm_params: + temperature: 0.3 + max_tokens: 1000 + + # Example: Google Gemini 2.0 Flash + - id: -2 + name: "Global Gemini 2.0 Flash" + description: "Google's fast vision model with large context" + provider: "GOOGLE" + model_name: "gemini-2.0-flash" + api_key: "your-google-ai-api-key-here" + api_base: "" + rpm: 1000 + tpm: 200000 + litellm_params: + temperature: 0.3 + max_tokens: 1000 + + # Example: Anthropic Claude 3.5 Sonnet + - id: -3 + name: "Global Claude 3.5 Sonnet Vision" + description: "Anthropic's Claude 3.5 Sonnet with vision support" + provider: "ANTHROPIC" + model_name: "claude-3-5-sonnet-20241022" + api_key: "sk-ant-your-anthropic-api-key-here" + api_base: "" + rpm: 1000 + tpm: 100000 + litellm_params: + temperature: 0.3 + max_tokens: 1000 + + # Example: Azure OpenAI GPT-4o + # - id: -4 + # name: "Global Azure GPT-4o Vision" + # description: "Azure-hosted GPT-4o for vision analysis" + # provider: "AZURE_OPENAI" + # model_name: "azure/gpt-4o-deployment" + # api_key: "your-azure-api-key-here" + # api_base: "https://your-resource.openai.azure.com" + # api_version: "2024-02-15-preview" + # rpm: 500 + # tpm: 100000 + # litellm_params: + # temperature: 0.3 + # max_tokens: 1000 + # base_model: "gpt-4o" + # Notes: # - ID 0 is reserved for "Auto" mode - uses LiteLLM Router for load balancing # - Use negative IDs to distinguish global configs from user configs (NewLLMConfig in DB) @@ -283,3 +359,9 @@ global_image_generation_configs: # - The router uses litellm.aimage_generation() for async image generation # - Only RPM (requests per minute) is relevant for image generation rate limiting. # TPM (tokens per minute) does not apply since image APIs are billed/rate-limited per request, not per token. +# +# VISION LLM NOTES: +# - Vision configs use the same ID scheme (negative for global, positive for user DB) +# - Only use vision-capable models (GPT-4o, Gemini, Claude 3, etc.) +# - Lower temperature (0.3) is recommended for accurate screenshot analysis +# - Lower max_tokens (1000) is sufficient since autocomplete produces short suggestions From 4dd1b6c64f19bd840925f1d252450a944cb10d58 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 22:15:35 +0200 Subject: [PATCH 28/28] Add posthog-node and node-machine-id dependencies for desktop analytics --- surfsense_desktop/package.json | 4 +++- surfsense_desktop/pnpm-lock.yaml | 31 +++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/surfsense_desktop/package.json b/surfsense_desktop/package.json index 74f6274cb..7b91d70c9 100644 --- a/surfsense_desktop/package.json +++ b/surfsense_desktop/package.json @@ -34,6 +34,8 @@ "electron-store": "^11.0.2", "electron-updater": "^6.8.3", "get-port-please": "^3.2.0", - "node-mac-permissions": "^2.5.0" + "node-mac-permissions": "^2.5.0", + "node-machine-id": "^1.1.12", + "posthog-node": "^5.29.0" } } diff --git a/surfsense_desktop/pnpm-lock.yaml b/surfsense_desktop/pnpm-lock.yaml index e1df34fb2..e7b84cc01 100644 --- a/surfsense_desktop/pnpm-lock.yaml +++ b/surfsense_desktop/pnpm-lock.yaml @@ -26,6 +26,12 @@ importers: node-mac-permissions: specifier: ^2.5.0 version: 2.5.0 + node-machine-id: + specifier: ^1.1.12 + version: 1.1.12 + posthog-node: + specifier: ^5.29.0 + version: 5.29.0(rxjs@7.8.2) devDependencies: '@electron/rebuild': specifier: ^4.0.3 @@ -308,6 +314,9 @@ packages: resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} engines: {node: '>=14'} + '@posthog/core@1.25.0': + resolution: {integrity: sha512-XKaHvRFIIN7Dw84r1eKimV1rl9DS+9XMCPPZ7P3+l8fE+rDsmumebiTFsY+q40bVXflcGW9wB+57LH0lvcGmhw==} + '@sindresorhus/is@4.6.0': resolution: {integrity: sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw==} engines: {node: '>=10'} @@ -1194,6 +1203,9 @@ packages: resolution: {integrity: sha512-zR8SVCaN3WqV1xwWd04XVAdzm3UTdjbxciLrZtB0Cc7F2Kd34AJfhPD4hm1HU0YH3oGUZO4X9OBLY5ijSTHsGw==} os: [darwin] + node-machine-id@1.1.12: + resolution: {integrity: sha512-QNABxbrPa3qEIfrE6GOJ7BYIuignnJw7iQ2YPbc3Nla1HzRJjXzZOiikfF8m7eAMfichLt3M4VgLOetqgDmgGQ==} + nopt@8.1.0: resolution: {integrity: sha512-ieGu42u/Qsa4TFktmaKEwM6MQH0pOWnaB3htzh0JRtx84+Mebc0cbZYN5bC+6WTZ4+77xrL9Pn5m7CV6VIkV7A==} engines: {node: ^18.17.0 || >=20.5.0} @@ -1263,6 +1275,15 @@ packages: resolution: {integrity: sha512-uysumyrvkUX0rX/dEVqt8gC3sTBzd4zoWfLeS29nb53imdaXVvLINYXTI2GNqzaMuvacNx4uJQ8+b3zXR0pkgQ==} engines: {node: '>=10.4.0'} + posthog-node@5.29.0: + resolution: {integrity: sha512-po7N55haSKxV8VOulkBZJja938yILShl6+fFjoUV3iQgOBCg4Muu615/xRg8mpNiz+UASvL0EEiGvIxdhXfj6Q==} + engines: {node: ^20.20.0 || >=22.22.0} + peerDependencies: + rxjs: ^7.0.0 + peerDependenciesMeta: + rxjs: + optional: true + postject@1.0.0-alpha.6: resolution: {integrity: sha512-b9Eb8h2eVqNE8edvKdwqkrY6O7kAwmI8kcnBv1NScolYJbo59XUF0noFq+lxbC1yN20bmC0WBEbDC5H/7ASb0A==} engines: {node: '>=14.0.0'} @@ -1876,6 +1897,8 @@ snapshots: '@pkgjs/parseargs@0.11.0': optional: true + '@posthog/core@1.25.0': {} + '@sindresorhus/is@4.6.0': {} '@standard-schema/spec@1.1.0': {} @@ -2940,6 +2963,8 @@ snapshots: bindings: 1.5.0 node-addon-api: 7.1.1 + node-machine-id@1.1.12: {} + nopt@8.1.0: dependencies: abbrev: 3.0.1 @@ -3002,6 +3027,12 @@ snapshots: base64-js: 1.5.1 xmlbuilder: 15.1.1 + posthog-node@5.29.0(rxjs@7.8.2): + dependencies: + '@posthog/core': 1.25.0 + optionalDependencies: + rxjs: 7.8.2 + postject@1.0.0-alpha.6: dependencies: commander: 9.5.0