apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-04-25 00:36:31 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge pull request #1162 from CREDO23/feat/vision-autocomplete

Browse source 
[Feat] Multi-suggestion autocomplete, Vision LLM config & Desktop analytics
This commit is contained in:
Rohan Verma 2026-04-07 14:01:44 -07:00 • committed by GitHub
commit e827a3906d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
49 changed files with 3263 additions and 153 deletions
Download patch file Download diff file Expand all files Collapse all files

3
.github/workflows/desktop-release.yml vendored
View file

@ -60,6 +60,7 @@ jobs:
NEXT_PUBLIC_ZERO_CACHE_URL: ${{ vars.NEXT_PUBLIC_ZERO_CACHE_URL }}
NEXT_PUBLIC_DEPLOYMENT_MODE: ${{ vars.NEXT_PUBLIC_DEPLOYMENT_MODE }}
NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE: ${{ vars.NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE }}
NEXT_PUBLIC_POSTHOG_KEY: ${{ secrets.NEXT_PUBLIC_POSTHOG_KEY }}
- name: Install desktop dependencies
run: pnpm install
@ -70,6 +71,8 @@ jobs:
working-directory: surfsense_desktop
env:
HOSTED_FRONTEND_URL: ${{ vars.HOSTED_FRONTEND_URL }}
POSTHOG_KEY: ${{ secrets.POSTHOG_KEY }}
POSTHOG_HOST: ${{ vars.POSTHOG_HOST }}
- name: Package & Publish
run: pnpm exec electron-builder ${{ matrix.platform }} --config electron-builder.yml --publish always -c.extraMetadata.version=${{ steps.version.outputs.VERSION }}

190
surfsense_backend/alembic/versions/120_add_vision_llm_configs_table.py Normal file
View file

@ -0,0 +1,190 @@
"""Add vision LLM configs table and rename preference column
Revision ID: 120
Revises: 119
Changes:
1. Create visionprovider enum type
2. Create vision_llm_configs table
3. Rename vision_llm_id -> vision_llm_config_id on searchspaces
4. Add vision config permissions to existing system roles
"""
from __future__ import annotations
from collections.abc import Sequence
import sqlalchemy as sa
from sqlalchemy.dialects.postgresql import ENUM as PG_ENUM, UUID
from alembic import op
revision: str = "120"
down_revision: str | None = "119"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
VISION_PROVIDER_VALUES = (
"OPENAI",
"ANTHROPIC",
"GOOGLE",
"AZURE_OPENAI",
"VERTEX_AI",
"BEDROCK",
"XAI",
"OPENROUTER",
"OLLAMA",
"GROQ",
"TOGETHER_AI",
"FIREWORKS_AI",
"DEEPSEEK",
"MISTRAL",
"CUSTOM",
)
def upgrade() -> None:
connection = op.get_bind()
# 1. Create visionprovider enum
connection.execute(
sa.text(
"""
DO $$
BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'visionprovider') THEN
CREATE TYPE visionprovider AS ENUM (
'OPENAI', 'ANTHROPIC', 'GOOGLE', 'AZURE_OPENAI', 'VERTEX_AI',
'BEDROCK', 'XAI', 'OPENROUTER', 'OLLAMA', 'GROQ',
'TOGETHER_AI', 'FIREWORKS_AI', 'DEEPSEEK', 'MISTRAL', 'CUSTOM'
);
END IF;
END
$$;
"""
)
)
# 2. Create vision_llm_configs table
result = connection.execute(
sa.text(
"SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = 'vision_llm_configs')"
)
)
if not result.scalar():
op.create_table(
"vision_llm_configs",
sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
sa.Column("name", sa.String(100), nullable=False),
sa.Column("description", sa.String(500), nullable=True),
sa.Column(
"provider",
PG_ENUM(*VISION_PROVIDER_VALUES, name="visionprovider", create_type=False),
nullable=False,
),
sa.Column("custom_provider", sa.String(100), nullable=True),
sa.Column("model_name", sa.String(100), nullable=False),
sa.Column("api_key", sa.String(), nullable=False),
sa.Column("api_base", sa.String(500), nullable=True),
sa.Column("api_version", sa.String(50), nullable=True),
sa.Column("litellm_params", sa.JSON(), nullable=True),
sa.Column("search_space_id", sa.Integer(), nullable=False),
sa.Column("user_id", UUID(as_uuid=True), nullable=False),
sa.Column(
"created_at",
sa.TIMESTAMP(timezone=True),
server_default=sa.text("now()"),
nullable=False,
),
sa.PrimaryKeyConstraint("id"),
sa.ForeignKeyConstraint(
["search_space_id"], ["searchspaces.id"], ondelete="CASCADE"
),
sa.ForeignKeyConstraint(
["user_id"], ["user.id"], ondelete="CASCADE"
),
)
op.execute(
"CREATE INDEX IF NOT EXISTS ix_vision_llm_configs_name "
"ON vision_llm_configs (name)"
)
op.execute(
"CREATE INDEX IF NOT EXISTS ix_vision_llm_configs_search_space_id "
"ON vision_llm_configs (search_space_id)"
)
# 3. Rename vision_llm_id -> vision_llm_config_id on searchspaces
existing_columns = [
col["name"] for col in sa.inspect(connection).get_columns("searchspaces")
]
if "vision_llm_id" in existing_columns and "vision_llm_config_id" not in existing_columns:
op.alter_column("searchspaces", "vision_llm_id", new_column_name="vision_llm_config_id")
elif "vision_llm_config_id" not in existing_columns:
op.add_column(
"searchspaces",
sa.Column("vision_llm_config_id", sa.Integer(), nullable=True, server_default="0"),
)
# 4. Add vision config permissions to existing system roles
connection.execute(
sa.text(
"""
UPDATE search_space_roles
SET permissions = array_cat(
permissions,
ARRAY['vision_configs:create', 'vision_configs:read']
)
WHERE is_system_role = true
AND name = 'Editor'
AND NOT ('vision_configs:create' = ANY(permissions))
"""
)
)
connection.execute(
sa.text(
"""
UPDATE search_space_roles
SET permissions = array_cat(
permissions,
ARRAY['vision_configs:read']
)
WHERE is_system_role = true
AND name = 'Viewer'
AND NOT ('vision_configs:read' = ANY(permissions))
"""
)
)
def downgrade() -> None:
connection = op.get_bind()
# Remove permissions
connection.execute(
sa.text(
"""
UPDATE search_space_roles
SET permissions = array_remove(
array_remove(
array_remove(permissions, 'vision_configs:create'),
'vision_configs:read'
),
'vision_configs:delete'
)
WHERE is_system_role = true
"""
)
)
# Rename column back
existing_columns = [
col["name"] for col in sa.inspect(connection).get_columns("searchspaces")
]
if "vision_llm_config_id" in existing_columns:
op.alter_column("searchspaces", "vision_llm_config_id", new_column_name="vision_llm_id")
# Drop table and enum
op.execute("DROP INDEX IF EXISTS ix_vision_llm_configs_search_space_id")
op.execute("DROP INDEX IF EXISTS ix_vision_llm_configs_name")
op.execute("DROP TABLE IF EXISTS vision_llm_configs")
op.execute("DROP TYPE IF EXISTS visionprovider")

105
surfsense_backend/app/agents/autocomplete/autocomplete_agent.py
View file

@ -14,7 +14,9 @@ LLM call — the window title is used directly as the KB search query.
from __future__ import annotations
import asyncio
import json
import logging
import re
import uuid
from collections.abc import AsyncGenerator
from typing import Any
@ -61,13 +63,21 @@ Key behavior:
- If the text area already has text, continue it naturally — typically just a sentence or two.
Rules:
- Output ONLY the text to be inserted. No quotes, no explanations, no meta-commentary.
- Be CONCISE. Prefer a single paragraph or a few sentences. Autocomplete is a quick assist, not a full draft.
- Match the tone and formality of the surrounding context.
- If the screen shows code, write code. If it shows a casual chat, be casual. If it shows a formal email, be formal.
- Do NOT describe the screenshot or explain your reasoning.
- Do NOT cite or reference documents explicitly — just let the knowledge inform your writing naturally.
- If you cannot determine what to write, output nothing.
- If you cannot determine what to write, output an empty JSON array: []
## Output Format
You MUST provide exactly 3 different suggestion options. Each should be a distinct, plausible completion — vary the tone, detail level, or angle.
Return your suggestions as a JSON array of exactly 3 strings. Output ONLY the JSON array, nothing else — no markdown fences, no explanation, no commentary.
Example format:
["First suggestion text here.", "Second suggestion — a different take.", "Third option with another approach."]
## Filesystem Tools `ls`, `read_file`, `write_file`, `edit_file`, `glob`, `grep`
@ -264,6 +274,50 @@ async def create_autocomplete_agent(
return agent, kb
# ---------------------------------------------------------------------------
# JSON suggestion parsing (with fallback)
# ---------------------------------------------------------------------------
def _parse_suggestions(raw: str) -> list[str]:
"""Extract a list of suggestion strings from the agent's output.
Tries, in order:
1. Direct ``json.loads``
2. Extract content between ```json ... ``` fences
3. Find the first ``[`` … ``]`` span
Falls back to wrapping the raw text as a single suggestion.
"""
text = raw.strip()
if not text:
return []
for candidate in _json_candidates(text):
try:
parsed = json.loads(candidate)
if isinstance(parsed, list) and all(isinstance(s, str) for s in parsed):
return [s for s in parsed if s.strip()]
except (json.JSONDecodeError, ValueError):
continue
return [text]
def _json_candidates(text: str) -> list[str]:
"""Yield candidate JSON strings from raw text."""
candidates = [text]
fence = re.search(r"```(?:json)?\s*\n?(.*?)```", text, re.DOTALL)
if fence:
candidates.append(fence.group(1).strip())
bracket = re.search(r"\[.*]", text, re.DOTALL)
if bracket:
candidates.append(bracket.group(0))
return candidates
# ---------------------------------------------------------------------------
# Streaming helper
# ---------------------------------------------------------------------------
@ -285,7 +339,7 @@ async def stream_autocomplete_agent(
thread_id = uuid.uuid4().hex
config = {"configurable": {"thread_id": thread_id}}
current_text_id: str | None = None
text_buffer: list[str] = []
active_tool_depth = 0
thinking_step_counter = 0
tool_step_ids: dict[str, str] = {}
@ -315,14 +369,12 @@ async def stream_autocomplete_agent(
if emit_message_start:
yield streaming_service.format_message_start()
# Emit an initial "Generating completion" step so the UI immediately
# shows activity once the agent starts its first LLM call.
gen_step_id = next_thinking_step_id()
last_active_step_id = gen_step_id
step_titles[gen_step_id] = "Generating completion"
step_titles[gen_step_id] = "Generating suggestions"
yield streaming_service.format_thinking_step(
step_id=gen_step_id,
title="Generating completion",
title="Generating suggestions",
status="in_progress",
)
@ -331,7 +383,6 @@ async def stream_autocomplete_agent(
input_data, config=config, version="v2"
):
event_type = event.get("event", "")
if event_type == "on_chat_model_stream":
if active_tool_depth > 0:
continue
@ -341,15 +392,20 @@ async def stream_autocomplete_agent(
if chunk and hasattr(chunk, "content"):
content = chunk.content
if content and isinstance(content, str):
if current_text_id is None:
step_event = complete_current_step()
if step_event:
yield step_event
current_text_id = streaming_service.generate_text_id()
yield streaming_service.format_text_start(current_text_id)
yield streaming_service.format_text_delta(
current_text_id, content
)
text_buffer.append(content)
elif event_type == "on_chat_model_end":
if active_tool_depth > 0:
continue
if "surfsense:internal" in event.get("tags", []):
continue
output = event.get("data", {}).get("output")
if output and hasattr(output, "content"):
if getattr(output, "tool_calls", None):
continue
content = output.content
if content and isinstance(content, str) and not text_buffer:
text_buffer.append(content)
elif event_type == "on_tool_start":
active_tool_depth += 1
@ -357,10 +413,6 @@ async def stream_autocomplete_agent(
run_id = event.get("run_id", "")
tool_input = event.get("data", {}).get("input", {})
if current_text_id is not None:
yield streaming_service.format_text_end(current_text_id)
current_text_id = None
step_event = complete_current_step()
if step_event:
yield step_event
@ -393,19 +445,22 @@ async def stream_autocomplete_agent(
if last_active_step_id == step_id:
last_active_step_id = None
if current_text_id is not None:
yield streaming_service.format_text_end(current_text_id)
step_event = complete_current_step()
if step_event:
yield step_event
raw_text = "".join(text_buffer)
suggestions = _parse_suggestions(raw_text)
yield streaming_service.format_data(
"suggestions", {"options": suggestions}
)
yield streaming_service.format_finish()
yield streaming_service.format_done()
except Exception as e:
logger.error(f"Autocomplete agent streaming error: {e}", exc_info=True)
if current_text_id is not None:
yield streaming_service.format_text_end(current_text_id)
yield streaming_service.format_error("Autocomplete failed. Please try again.")
yield streaming_service.format_done()

8
surfsense_backend/app/app.py
View file

@ -25,7 +25,12 @@ from app.agents.new_chat.checkpointer import (
close_checkpointer,
setup_checkpointer_tables,
)
from app.config import config, initialize_image_gen_router, initialize_llm_router
from app.config import (
config,
initialize_image_gen_router,
initialize_llm_router,
initialize_vision_llm_router,
)
from app.db import User, create_db_and_tables, get_async_session
from app.routes import router as crud_router
from app.routes.auth_routes import router as auth_router
@ -223,6 +228,7 @@ async def lifespan(app: FastAPI):
await setup_checkpointer_tables()
initialize_llm_router()
initialize_image_gen_router()
initialize_vision_llm_router()
try:
await asyncio.wait_for(seed_surfsense_docs(), timeout=120)
except TimeoutError:

7
surfsense_backend/app/celery_app.py
View file

@ -18,10 +18,15 @@ def init_worker(**kwargs):
This ensures the Auto mode (LiteLLM Router) is available for background tasks
like document summarization and image generation.
"""
from app.config import initialize_image_gen_router, initialize_llm_router
from app.config import (
initialize_image_gen_router,
initialize_llm_router,
initialize_vision_llm_router,
)
initialize_llm_router()
initialize_image_gen_router()
initialize_vision_llm_router()
# Get Celery configuration from environment

67
surfsense_backend/app/config/__init__.py
View file

@ -102,6 +102,44 @@ def load_global_image_gen_configs():
return []
def load_global_vision_llm_configs():
global_config_file = BASE_DIR / "app" / "config" / "global_llm_config.yaml"
if not global_config_file.exists():
return []
try:
with open(global_config_file, encoding="utf-8") as f:
data = yaml.safe_load(f)
return data.get("global_vision_llm_configs", [])
except Exception as e:
print(f"Warning: Failed to load global vision LLM configs: {e}")
return []
def load_vision_llm_router_settings():
default_settings = {
"routing_strategy": "usage-based-routing",
"num_retries": 3,
"allowed_fails": 3,
"cooldown_time": 60,
}
global_config_file = BASE_DIR / "app" / "config" / "global_llm_config.yaml"
if not global_config_file.exists():
return default_settings
try:
with open(global_config_file, encoding="utf-8") as f:
data = yaml.safe_load(f)
settings = data.get("vision_llm_router_settings", {})
return {**default_settings, **settings}
except Exception as e:
print(f"Warning: Failed to load vision LLM router settings: {e}")
return default_settings
def load_image_gen_router_settings():
"""
Load router settings for image generation Auto mode from YAML file.
@ -182,6 +220,29 @@ def initialize_image_gen_router():
print(f"Warning: Failed to initialize Image Generation Router: {e}")
def initialize_vision_llm_router():
vision_configs = load_global_vision_llm_configs()
router_settings = load_vision_llm_router_settings()
if not vision_configs:
print(
"Info: No global vision LLM configs found, "
"Vision LLM Auto mode will not be available"
)
return
try:
from app.services.vision_llm_router_service import VisionLLMRouterService
VisionLLMRouterService.initialize(vision_configs, router_settings)
print(
f"Info: Vision LLM Router initialized with {len(vision_configs)} models "
f"(strategy: {router_settings.get('routing_strategy', 'usage-based-routing')})"
)
except Exception as e:
print(f"Warning: Failed to initialize Vision LLM Router: {e}")
class Config:
# Check if ffmpeg is installed
if not is_ffmpeg_installed():
@ -335,6 +396,12 @@ class Config:
# Router settings for Image Generation Auto mode
IMAGE_GEN_ROUTER_SETTINGS = load_image_gen_router_settings()
# Global Vision LLM Configurations (optional)
GLOBAL_VISION_LLM_CONFIGS = load_global_vision_llm_configs()
# Router settings for Vision LLM Auto mode
VISION_LLM_ROUTER_SETTINGS = load_vision_llm_router_settings()
# Chonkie Configuration | Edit this to your needs
EMBEDDING_MODEL = os.getenv("EMBEDDING_MODEL")
# Azure OpenAI credentials from environment variables

82
surfsense_backend/app/config/global_llm_config.example.yaml
View file

@ -263,6 +263,82 @@ global_image_generation_configs:
# rpm: 30
# litellm_params: {}
# =============================================================================
# Vision LLM Configuration
# =============================================================================
# These configurations power the vision autocomplete feature (screenshot analysis).
# Only vision-capable models should be used here (e.g. GPT-4o, Gemini Pro, Claude 3).
# Supported providers: OpenAI, Anthropic, Google, Azure OpenAI, Vertex AI, Bedrock,
# xAI, OpenRouter, Ollama, Groq, Together AI, Fireworks AI, DeepSeek, Mistral, Custom
#
# Auto mode (ID 0) uses LiteLLM Router for load balancing across all vision configs.
# Router Settings for Vision LLM Auto Mode
vision_llm_router_settings:
routing_strategy: "usage-based-routing"
num_retries: 3
allowed_fails: 3
cooldown_time: 60
global_vision_llm_configs:
# Example: OpenAI GPT-4o (recommended for vision)
- id: -1
name: "Global GPT-4o Vision"
description: "OpenAI's GPT-4o with strong vision capabilities"
provider: "OPENAI"
model_name: "gpt-4o"
api_key: "sk-your-openai-api-key-here"
api_base: ""
rpm: 500
tpm: 100000
litellm_params:
temperature: 0.3
max_tokens: 1000
# Example: Google Gemini 2.0 Flash
- id: -2
name: "Global Gemini 2.0 Flash"
description: "Google's fast vision model with large context"
provider: "GOOGLE"
model_name: "gemini-2.0-flash"
api_key: "your-google-ai-api-key-here"
api_base: ""
rpm: 1000
tpm: 200000
litellm_params:
temperature: 0.3
max_tokens: 1000
# Example: Anthropic Claude 3.5 Sonnet
- id: -3
name: "Global Claude 3.5 Sonnet Vision"
description: "Anthropic's Claude 3.5 Sonnet with vision support"
provider: "ANTHROPIC"
model_name: "claude-3-5-sonnet-20241022"
api_key: "sk-ant-your-anthropic-api-key-here"
api_base: ""
rpm: 1000
tpm: 100000
litellm_params:
temperature: 0.3
max_tokens: 1000
# Example: Azure OpenAI GPT-4o
# - id: -4
# name: "Global Azure GPT-4o Vision"
# description: "Azure-hosted GPT-4o for vision analysis"
# provider: "AZURE_OPENAI"
# model_name: "azure/gpt-4o-deployment"
# api_key: "your-azure-api-key-here"
# api_base: "https://your-resource.openai.azure.com"
# api_version: "2024-02-15-preview"
# rpm: 500
# tpm: 100000
# litellm_params:
# temperature: 0.3
# max_tokens: 1000
# base_model: "gpt-4o"
# Notes:
# - ID 0 is reserved for "Auto" mode - uses LiteLLM Router for load balancing
# - Use negative IDs to distinguish global configs from user configs (NewLLMConfig in DB)
@ -283,3 +359,9 @@ global_image_generation_configs:
# - The router uses litellm.aimage_generation() for async image generation
# - Only RPM (requests per minute) is relevant for image generation rate limiting.
# TPM (tokens per minute) does not apply since image APIs are billed/rate-limited per request, not per token.
#
# VISION LLM NOTES:
# - Vision configs use the same ID scheme (negative for global, positive for user DB)
# - Only use vision-capable models (GPT-4o, Gemini, Claude 3, etc.)
# - Lower temperature (0.3) is recommended for accurate screenshot analysis
# - Lower max_tokens (1000) is sufficient since autocomplete produces short suggestions

77
surfsense_backend/app/db.py
View file

@ -260,6 +260,24 @@ class ImageGenProvider(StrEnum):
NSCALE = "NSCALE"
class VisionProvider(StrEnum):
OPENAI = "OPENAI"
ANTHROPIC = "ANTHROPIC"
GOOGLE = "GOOGLE"
AZURE_OPENAI = "AZURE_OPENAI"
VERTEX_AI = "VERTEX_AI"
BEDROCK = "BEDROCK"
XAI = "XAI"
OPENROUTER = "OPENROUTER"
OLLAMA = "OLLAMA"
GROQ = "GROQ"
TOGETHER_AI = "TOGETHER_AI"
FIREWORKS_AI = "FIREWORKS_AI"
DEEPSEEK = "DEEPSEEK"
MISTRAL = "MISTRAL"
CUSTOM = "CUSTOM"
class LogLevel(StrEnum):
DEBUG = "DEBUG"
INFO = "INFO"
@ -377,6 +395,11 @@ class Permission(StrEnum):
IMAGE_GENERATIONS_READ = "image_generations:read"
IMAGE_GENERATIONS_DELETE = "image_generations:delete"
# Vision LLM Configs
VISION_CONFIGS_CREATE = "vision_configs:create"
VISION_CONFIGS_READ = "vision_configs:read"
VISION_CONFIGS_DELETE = "vision_configs:delete"
# Connectors
CONNECTORS_CREATE = "connectors:create"
CONNECTORS_READ = "connectors:read"
@ -445,6 +468,9 @@ DEFAULT_ROLE_PERMISSIONS = {
# Image Generations (create and read, no delete)
Permission.IMAGE_GENERATIONS_CREATE.value,
Permission.IMAGE_GENERATIONS_READ.value,
# Vision Configs (create and read, no delete)
Permission.VISION_CONFIGS_CREATE.value,
Permission.VISION_CONFIGS_READ.value,
# Connectors (no delete)
Permission.CONNECTORS_CREATE.value,
Permission.CONNECTORS_READ.value,
@ -478,6 +504,8 @@ DEFAULT_ROLE_PERMISSIONS = {
Permission.VIDEO_PRESENTATIONS_READ.value,
# Image Generations (read only)
Permission.IMAGE_GENERATIONS_READ.value,
# Vision Configs (read only)
Permission.VISION_CONFIGS_READ.value,
# Connectors (read only)
Permission.CONNECTORS_READ.value,
# Logs (read only)
@ -1263,6 +1291,35 @@ class ImageGenerationConfig(BaseModel, TimestampMixin):
user = relationship("User", back_populates="image_generation_configs")
class VisionLLMConfig(BaseModel, TimestampMixin):
__tablename__ = "vision_llm_configs"
name = Column(String(100), nullable=False, index=True)
description = Column(String(500), nullable=True)
provider = Column(SQLAlchemyEnum(VisionProvider), nullable=False)
custom_provider = Column(String(100), nullable=True)
model_name = Column(String(100), nullable=False)
api_key = Column(String, nullable=False)
api_base = Column(String(500), nullable=True)
api_version = Column(String(50), nullable=True)
litellm_params = Column(JSON, nullable=True, default={})
search_space_id = Column(
Integer, ForeignKey("searchspaces.id", ondelete="CASCADE"), nullable=False
)
search_space = relationship(
"SearchSpace", back_populates="vision_llm_configs"
)
user_id = Column(
UUID(as_uuid=True), ForeignKey("user.id", ondelete="CASCADE"), nullable=False
)
user = relationship("User", back_populates="vision_llm_configs")
class ImageGeneration(BaseModel, TimestampMixin):
"""
Stores image generation requests and results using litellm.aimage_generation().
@ -1351,7 +1408,7 @@ class SearchSpace(BaseModel, TimestampMixin):
image_generation_config_id = Column(
Integer, nullable=True, default=0
) # For image generation, defaults to Auto mode
vision_llm_id = Column(
vision_llm_config_id = Column(
Integer, nullable=True, default=0
) # For vision/screenshot analysis, defaults to Auto mode
@ -1432,6 +1489,12 @@ class SearchSpace(BaseModel, TimestampMixin):
order_by="ImageGenerationConfig.id",
cascade="all, delete-orphan",
)
vision_llm_configs = relationship(
"VisionLLMConfig",
back_populates="search_space",
order_by="VisionLLMConfig.id",
cascade="all, delete-orphan",
)
# RBAC relationships
roles = relationship(
@ -1961,6 +2024,12 @@ if config.AUTH_TYPE == "GOOGLE":
passive_deletes=True,
)
vision_llm_configs = relationship(
"VisionLLMConfig",
back_populates="user",
passive_deletes=True,
)
# User memories for personalized AI responses
memories = relationship(
"UserMemory",
@ -2075,6 +2144,12 @@ else:
passive_deletes=True,
)
vision_llm_configs = relationship(
"VisionLLMConfig",
back_populates="user",
passive_deletes=True,
)
# User memories for personalized AI responses
memories = relationship(
"UserMemory",

2
surfsense_backend/app/routes/__init__.py
View file

@ -49,6 +49,7 @@ from .stripe_routes import router as stripe_router
from .surfsense_docs_routes import router as surfsense_docs_router
from .teams_add_connector_route import router as teams_add_connector_router
from .video_presentations_routes import router as video_presentations_router
from .vision_llm_routes import router as vision_llm_router
from .youtube_routes import router as youtube_router
router = APIRouter()
@ -68,6 +69,7 @@ router.include_router(
) # Video presentation status and streaming
router.include_router(reports_router) # Report CRUD and multi-format export
router.include_router(image_generation_router) # Image generation via litellm
router.include_router(vision_llm_router) # Vision LLM configs for screenshot analysis
router.include_router(search_source_connectors_router)
router.include_router(google_calendar_add_connector_router)
router.include_router(google_gmail_add_connector_router)

74
surfsense_backend/app/routes/search_spaces_routes.py
View file

@ -14,6 +14,7 @@ from app.db import (
SearchSpaceMembership,
SearchSpaceRole,
User,
VisionLLMConfig,
get_async_session,
get_default_roles_config,
)
@ -483,6 +484,63 @@ async def _get_image_gen_config_by_id(
return None
async def _get_vision_llm_config_by_id(
session: AsyncSession, config_id: int | None
) -> dict | None:
if config_id is None:
return None
if config_id == 0:
return {
"id": 0,
"name": "Auto (Fastest)",
"description": "Automatically routes requests across available vision LLM providers",
"provider": "AUTO",
"model_name": "auto",
"is_global": True,
"is_auto_mode": True,
}
if config_id < 0:
for cfg in config.GLOBAL_VISION_LLM_CONFIGS:
if cfg.get("id") == config_id:
return {
"id": cfg.get("id"),
"name": cfg.get("name"),
"description": cfg.get("description"),
"provider": cfg.get("provider"),
"custom_provider": cfg.get("custom_provider"),
"model_name": cfg.get("model_name"),
"api_base": cfg.get("api_base") or None,
"api_version": cfg.get("api_version") or None,
"litellm_params": cfg.get("litellm_params", {}),
"is_global": True,
}
return None
result = await session.execute(
select(VisionLLMConfig).filter(VisionLLMConfig.id == config_id)
)
db_config = result.scalars().first()
if db_config:
return {
"id": db_config.id,
"name": db_config.name,
"description": db_config.description,
"provider": db_config.provider.value if db_config.provider else None,
"custom_provider": db_config.custom_provider,
"model_name": db_config.model_name,
"api_base": db_config.api_base,
"api_version": db_config.api_version,
"litellm_params": db_config.litellm_params or {},
"created_at": db_config.created_at.isoformat()
if db_config.created_at
else None,
"search_space_id": db_config.search_space_id,
}
return None
@router.get(
"/search-spaces/{search_space_id}/llm-preferences",
response_model=LLMPreferencesRead,
@ -522,17 +580,19 @@ async def get_llm_preferences(
image_generation_config = await _get_image_gen_config_by_id(
session, search_space.image_generation_config_id
)
vision_llm = await _get_llm_config_by_id(session, search_space.vision_llm_id)
vision_llm_config = await _get_vision_llm_config_by_id(
session, search_space.vision_llm_config_id
)
return LLMPreferencesRead(
agent_llm_id=search_space.agent_llm_id,
document_summary_llm_id=search_space.document_summary_llm_id,
image_generation_config_id=search_space.image_generation_config_id,
vision_llm_id=search_space.vision_llm_id,
vision_llm_config_id=search_space.vision_llm_config_id,
agent_llm=agent_llm,
document_summary_llm=document_summary_llm,
image_generation_config=image_generation_config,
vision_llm=vision_llm,
vision_llm_config=vision_llm_config,
)
except HTTPException:
@ -592,17 +652,19 @@ async def update_llm_preferences(
image_generation_config = await _get_image_gen_config_by_id(
session, search_space.image_generation_config_id
)
vision_llm = await _get_llm_config_by_id(session, search_space.vision_llm_id)
vision_llm_config = await _get_vision_llm_config_by_id(
session, search_space.vision_llm_config_id
)
return LLMPreferencesRead(
agent_llm_id=search_space.agent_llm_id,
document_summary_llm_id=search_space.document_summary_llm_id,
image_generation_config_id=search_space.image_generation_config_id,
vision_llm_id=search_space.vision_llm_id,
vision_llm_config_id=search_space.vision_llm_config_id,
agent_llm=agent_llm,
document_summary_llm=document_summary_llm,
image_generation_config=image_generation_config,
vision_llm=vision_llm,
vision_llm_config=vision_llm_config,
)
except HTTPException:

267
surfsense_backend/app/routes/vision_llm_routes.py Normal file
View file

@ -0,0 +1,267 @@
import logging
from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.config import config
from app.db import (
Permission,
User,
VisionLLMConfig,
get_async_session,
)
from app.schemas import (
GlobalVisionLLMConfigRead,
VisionLLMConfigCreate,
VisionLLMConfigRead,
VisionLLMConfigUpdate,
)
from app.users import current_active_user
from app.utils.rbac import check_permission
router = APIRouter()
logger = logging.getLogger(__name__)
# =============================================================================
# Global Vision LLM Configs (from YAML)
# =============================================================================
@router.get(
"/global-vision-llm-configs",
response_model=list[GlobalVisionLLMConfigRead],
)
async def get_global_vision_llm_configs(
user: User = Depends(current_active_user),
):
try:
global_configs = config.GLOBAL_VISION_LLM_CONFIGS
safe_configs = []
if global_configs and len(global_configs) > 0:
safe_configs.append(
{
"id": 0,
"name": "Auto (Fastest)",
"description": "Automatically routes across available vision LLM providers.",
"provider": "AUTO",
"custom_provider": None,
"model_name": "auto",
"api_base": None,
"api_version": None,
"litellm_params": {},
"is_global": True,
"is_auto_mode": True,
}
)
for cfg in global_configs:
safe_configs.append(
{
"id": cfg.get("id"),
"name": cfg.get("name"),
"description": cfg.get("description"),
"provider": cfg.get("provider"),
"custom_provider": cfg.get("custom_provider"),
"model_name": cfg.get("model_name"),
"api_base": cfg.get("api_base") or None,
"api_version": cfg.get("api_version") or None,
"litellm_params": cfg.get("litellm_params", {}),
"is_global": True,
}
)
return safe_configs
except Exception as e:
logger.exception("Failed to fetch global vision LLM configs")
raise HTTPException(
status_code=500, detail=f"Failed to fetch configs: {e!s}"
) from e
# =============================================================================
# VisionLLMConfig CRUD
# =============================================================================
@router.post("/vision-llm-configs", response_model=VisionLLMConfigRead)
async def create_vision_llm_config(
config_data: VisionLLMConfigCreate,
session: AsyncSession = Depends(get_async_session),
user: User = Depends(current_active_user),
):
try:
await check_permission(
session,
user,
config_data.search_space_id,
Permission.VISION_CONFIGS_CREATE.value,
"You don't have permission to create vision LLM configs in this search space",
)
db_config = VisionLLMConfig(**config_data.model_dump(), user_id=user.id)
session.add(db_config)
await session.commit()
await session.refresh(db_config)
return db_config
except HTTPException:
raise
except Exception as e:
await session.rollback()
logger.exception("Failed to create VisionLLMConfig")
raise HTTPException(
status_code=500, detail=f"Failed to create config: {e!s}"
) from e
@router.get("/vision-llm-configs", response_model=list[VisionLLMConfigRead])
async def list_vision_llm_configs(
search_space_id: int,
skip: int = 0,
limit: int = 100,
session: AsyncSession = Depends(get_async_session),
user: User = Depends(current_active_user),
):
try:
await check_permission(
session,
user,
search_space_id,
Permission.VISION_CONFIGS_READ.value,
"You don't have permission to view vision LLM configs in this search space",
)
result = await session.execute(
select(VisionLLMConfig)
.filter(VisionLLMConfig.search_space_id == search_space_id)
.order_by(VisionLLMConfig.created_at.desc())
.offset(skip)
.limit(limit)
)
return result.scalars().all()
except HTTPException:
raise
except Exception as e:
logger.exception("Failed to list VisionLLMConfigs")
raise HTTPException(
status_code=500, detail=f"Failed to fetch configs: {e!s}"
) from e
@router.get(
"/vision-llm-configs/{config_id}", response_model=VisionLLMConfigRead
)
async def get_vision_llm_config(
config_id: int,
session: AsyncSession = Depends(get_async_session),
user: User = Depends(current_active_user),
):
try:
result = await session.execute(
select(VisionLLMConfig).filter(VisionLLMConfig.id == config_id)
)
db_config = result.scalars().first()
if not db_config:
raise HTTPException(status_code=404, detail="Config not found")
await check_permission(
session,
user,
db_config.search_space_id,
Permission.VISION_CONFIGS_READ.value,
"You don't have permission to view vision LLM configs in this search space",
)
return db_config
except HTTPException:
raise
except Exception as e:
logger.exception("Failed to get VisionLLMConfig")
raise HTTPException(
status_code=500, detail=f"Failed to fetch config: {e!s}"
) from e
@router.put(
"/vision-llm-configs/{config_id}", response_model=VisionLLMConfigRead
)
async def update_vision_llm_config(
config_id: int,
update_data: VisionLLMConfigUpdate,
session: AsyncSession = Depends(get_async_session),
user: User = Depends(current_active_user),
):
try:
result = await session.execute(
select(VisionLLMConfig).filter(VisionLLMConfig.id == config_id)
)
db_config = result.scalars().first()
if not db_config:
raise HTTPException(status_code=404, detail="Config not found")
await check_permission(
session,
user,
db_config.search_space_id,
Permission.VISION_CONFIGS_CREATE.value,
"You don't have permission to update vision LLM configs in this search space",
)
for key, value in update_data.model_dump(exclude_unset=True).items():
setattr(db_config, key, value)
await session.commit()
await session.refresh(db_config)
return db_config
except HTTPException:
raise
except Exception as e:
await session.rollback()
logger.exception("Failed to update VisionLLMConfig")
raise HTTPException(
status_code=500, detail=f"Failed to update config: {e!s}"
) from e
@router.delete("/vision-llm-configs/{config_id}", response_model=dict)
async def delete_vision_llm_config(
config_id: int,
session: AsyncSession = Depends(get_async_session),
user: User = Depends(current_active_user),
):
try:
result = await session.execute(
select(VisionLLMConfig).filter(VisionLLMConfig.id == config_id)
)
db_config = result.scalars().first()
if not db_config:
raise HTTPException(status_code=404, detail="Config not found")
await check_permission(
session,
user,
db_config.search_space_id,
Permission.VISION_CONFIGS_DELETE.value,
"You don't have permission to delete vision LLM configs in this search space",
)
await session.delete(db_config)
await session.commit()
return {
"message": "Vision LLM config deleted successfully",
"id": config_id,
}
except HTTPException:
raise
except Exception as e:
await session.rollback()
logger.exception("Failed to delete VisionLLMConfig")
raise HTTPException(
status_code=500, detail=f"Failed to delete config: {e!s}"
) from e

13
surfsense_backend/app/schemas/__init__.py
View file

@ -125,6 +125,13 @@ from .video_presentations import (
VideoPresentationRead,
VideoPresentationUpdate,
)
from .vision_llm import (
GlobalVisionLLMConfigRead,
VisionLLMConfigCreate,
VisionLLMConfigPublic,
VisionLLMConfigRead,
VisionLLMConfigUpdate,
)
__all__ = [
# Folder schemas
@ -163,6 +170,8 @@ __all__ = [
"FolderUpdate",
"GlobalImageGenConfigRead",
"GlobalNewLLMConfigRead",
# Vision LLM Config schemas
"GlobalVisionLLMConfigRead",
"GoogleDriveIndexRequest",
"GoogleDriveIndexingOptions",
# Base schemas
@ -264,4 +273,8 @@ __all__ = [
"VideoPresentationCreate",
"VideoPresentationRead",
"VideoPresentationUpdate",
"VisionLLMConfigCreate",
"VisionLLMConfigPublic",
"VisionLLMConfigRead",
"VisionLLMConfigUpdate",
]

10
surfsense_backend/app/schemas/new_llm_config.py
View file

@ -182,8 +182,8 @@ class LLMPreferencesRead(BaseModel):
image_generation_config_id: int | None = Field(
None, description="ID of the image generation config to use"
)
vision_llm_id: int | None = Field(
None, description="ID of the LLM config to use for vision/screenshot analysis"
vision_llm_config_id: int | None = Field(
None, description="ID of the vision LLM config to use for vision/screenshot analysis"
)
agent_llm: dict[str, Any] | None = Field(
None, description="Full config for agent LLM"
@ -194,7 +194,7 @@ class LLMPreferencesRead(BaseModel):
image_generation_config: dict[str, Any] | None = Field(
None, description="Full config for image generation"
)
vision_llm: dict[str, Any] | None = Field(
vision_llm_config: dict[str, Any] | None = Field(
None, description="Full config for vision LLM"
)
@ -213,6 +213,6 @@ class LLMPreferencesUpdate(BaseModel):
image_generation_config_id: int | None = Field(
None, description="ID of the image generation config to use"
)
vision_llm_id: int | None = Field(
None, description="ID of the LLM config to use for vision/screenshot analysis"
vision_llm_config_id: int | None = Field(
None, description="ID of the vision LLM config to use for vision/screenshot analysis"
)

75
surfsense_backend/app/schemas/vision_llm.py Normal file
View file

@ -0,0 +1,75 @@
import uuid
from datetime import datetime
from typing import Any
from pydantic import BaseModel, ConfigDict, Field
from app.db import VisionProvider
class VisionLLMConfigBase(BaseModel):
name: str = Field(..., max_length=100)
description: str | None = Field(None, max_length=500)
provider: VisionProvider = Field(...)
custom_provider: str | None = Field(None, max_length=100)
model_name: str = Field(..., max_length=100)
api_key: str = Field(...)
api_base: str | None = Field(None, max_length=500)
api_version: str | None = Field(None, max_length=50)
litellm_params: dict[str, Any] | None = Field(default=None)
class VisionLLMConfigCreate(VisionLLMConfigBase):
search_space_id: int = Field(...)
class VisionLLMConfigUpdate(BaseModel):
name: str | None = Field(None, max_length=100)
description: str | None = Field(None, max_length=500)
provider: VisionProvider | None = None
custom_provider: str | None = Field(None, max_length=100)
model_name: str | None = Field(None, max_length=100)
api_key: str | None = None
api_base: str | None = Field(None, max_length=500)
api_version: str | None = Field(None, max_length=50)
litellm_params: dict[str, Any] | None = None
class VisionLLMConfigRead(VisionLLMConfigBase):
id: int
created_at: datetime
search_space_id: int
user_id: uuid.UUID
model_config = ConfigDict(from_attributes=True)
class VisionLLMConfigPublic(BaseModel):
id: int
name: str
description: str | None = None
provider: VisionProvider
custom_provider: str | None = None
model_name: str
api_base: str | None = None
api_version: str | None = None
litellm_params: dict[str, Any] | None = None
created_at: datetime
search_space_id: int
user_id: uuid.UUID
model_config = ConfigDict(from_attributes=True)
class GlobalVisionLLMConfigRead(BaseModel):
id: int = Field(...)
name: str
description: str | None = None
provider: str
custom_provider: str | None = None
model_name: str
api_base: str | None = None
api_version: str | None = None
litellm_params: dict[str, Any] | None = None
is_global: bool = True
is_auto_mode: bool = False

119
surfsense_backend/app/services/llm_service.py
View file

@ -32,7 +32,6 @@ logger = logging.getLogger(__name__)
class LLMRole:
AGENT = "agent" # For agent/chat operations
DOCUMENT_SUMMARY = "document_summary" # For document summarization
VISION = "vision" # For vision/screenshot analysis
def get_global_llm_config(llm_config_id: int) -> dict | None:
@ -188,7 +187,7 @@ async def get_search_space_llm_instance(
Args:
session: Database session
search_space_id: Search Space ID
role: LLM role ('agent', 'document_summary', or 'vision')
role: LLM role ('agent' or 'document_summary')
Returns:
ChatLiteLLM or ChatLiteLLMRouter instance, or None if not found
@ -210,8 +209,6 @@ async def get_search_space_llm_instance(
llm_config_id = search_space.agent_llm_id
elif role == LLMRole.DOCUMENT_SUMMARY:
llm_config_id = search_space.document_summary_llm_id
elif role == LLMRole.VISION:
llm_config_id = search_space.vision_llm_id
else:
logger.error(f"Invalid LLM role: {role}")
return None
@ -411,8 +408,118 @@ async def get_document_summary_llm(
async def get_vision_llm(
session: AsyncSession, search_space_id: int
) -> ChatLiteLLM | ChatLiteLLMRouter | None:
"""Get the search space's vision LLM instance for screenshot analysis."""
return await get_search_space_llm_instance(session, search_space_id, LLMRole.VISION)
"""Get the search space's vision LLM instance for screenshot analysis.
Resolves from the dedicated VisionLLMConfig system:
- Auto mode (ID 0): VisionLLMRouterService
- Global (negative ID): YAML configs
- DB (positive ID): VisionLLMConfig table
"""
from app.db import VisionLLMConfig
from app.services.vision_llm_router_service import (
VISION_PROVIDER_MAP,
VisionLLMRouterService,
get_global_vision_llm_config,
is_vision_auto_mode,
)
try:
result = await session.execute(
select(SearchSpace).where(SearchSpace.id == search_space_id)
)
search_space = result.scalars().first()
if not search_space:
logger.error(f"Search space {search_space_id} not found")
return None
config_id = search_space.vision_llm_config_id
if config_id is None:
logger.error(
f"No vision LLM configured for search space {search_space_id}"
)
return None
if is_vision_auto_mode(config_id):
if not VisionLLMRouterService.is_initialized():
logger.error(
"Vision Auto mode requested but Vision LLM Router not initialized"
)
return None
try:
return ChatLiteLLMRouter(
router=VisionLLMRouterService.get_router(),
streaming=True,
)
except Exception as e:
logger.error(f"Failed to create vision ChatLiteLLMRouter: {e}")
return None
if config_id < 0:
global_cfg = get_global_vision_llm_config(config_id)
if not global_cfg:
logger.error(f"Global vision LLM config {config_id} not found")
return None
if global_cfg.get("custom_provider"):
model_string = (
f"{global_cfg['custom_provider']}/{global_cfg['model_name']}"
)
else:
prefix = VISION_PROVIDER_MAP.get(
global_cfg["provider"].upper(),
global_cfg["provider"].lower(),
)
model_string = f"{prefix}/{global_cfg['model_name']}"
litellm_kwargs = {
"model": model_string,
"api_key": global_cfg["api_key"],
}
if global_cfg.get("api_base"):
litellm_kwargs["api_base"] = global_cfg["api_base"]
if global_cfg.get("litellm_params"):
litellm_kwargs.update(global_cfg["litellm_params"])
return ChatLiteLLM(**litellm_kwargs)
result = await session.execute(
select(VisionLLMConfig).where(
VisionLLMConfig.id == config_id,
VisionLLMConfig.search_space_id == search_space_id,
)
)
vision_cfg = result.scalars().first()
if not vision_cfg:
logger.error(
f"Vision LLM config {config_id} not found in search space {search_space_id}"
)
return None
if vision_cfg.custom_provider:
model_string = f"{vision_cfg.custom_provider}/{vision_cfg.model_name}"
else:
prefix = VISION_PROVIDER_MAP.get(
vision_cfg.provider.value.upper(),
vision_cfg.provider.value.lower(),
)
model_string = f"{prefix}/{vision_cfg.model_name}"
litellm_kwargs = {
"model": model_string,
"api_key": vision_cfg.api_key,
}
if vision_cfg.api_base:
litellm_kwargs["api_base"] = vision_cfg.api_base
if vision_cfg.litellm_params:
litellm_kwargs.update(vision_cfg.litellm_params)
return ChatLiteLLM(**litellm_kwargs)
except Exception as e:
logger.error(
f"Error getting vision LLM for search space {search_space_id}: {e!s}"
)
return None
# Backward-compatible alias (LLM preferences are now per-search-space, not per-user)

193
surfsense_backend/app/services/vision_llm_router_service.py Normal file
View file

@ -0,0 +1,193 @@
import logging
from typing import Any
from litellm import Router
logger = logging.getLogger(__name__)
VISION_AUTO_MODE_ID = 0
VISION_PROVIDER_MAP = {
"OPENAI": "openai",
"ANTHROPIC": "anthropic",
"GOOGLE": "gemini",
"AZURE_OPENAI": "azure",
"VERTEX_AI": "vertex_ai",
"BEDROCK": "bedrock",
"XAI": "xai",
"OPENROUTER": "openrouter",
"OLLAMA": "ollama_chat",
"GROQ": "groq",
"TOGETHER_AI": "together_ai",
"FIREWORKS_AI": "fireworks_ai",
"DEEPSEEK": "openai",
"MISTRAL": "mistral",
"CUSTOM": "custom",
}
class VisionLLMRouterService:
_instance = None
_router: Router | None = None
_model_list: list[dict] = []
_router_settings: dict = {}
_initialized: bool = False
def __new__(cls):
if cls._instance is None:
cls._instance = super().__new__(cls)
return cls._instance
@classmethod
def get_instance(cls) -> "VisionLLMRouterService":
if cls._instance is None:
cls._instance = cls()
return cls._instance
@classmethod
def initialize(
cls,
global_configs: list[dict],
router_settings: dict | None = None,
) -> None:
instance = cls.get_instance()
if instance._initialized:
logger.debug("Vision LLM Router already initialized, skipping")
return
model_list = []
for config in global_configs:
deployment = cls._config_to_deployment(config)
if deployment:
model_list.append(deployment)
if not model_list:
logger.warning(
"No valid vision LLM configs found for router initialization"
)
return
instance._model_list = model_list
instance._router_settings = router_settings or {}
default_settings = {
"routing_strategy": "usage-based-routing",
"num_retries": 3,
"allowed_fails": 3,
"cooldown_time": 60,
"retry_after": 5,
}
final_settings = {**default_settings, **instance._router_settings}
try:
instance._router = Router(
model_list=model_list,
routing_strategy=final_settings.get(
"routing_strategy", "usage-based-routing"
),
num_retries=final_settings.get("num_retries", 3),
allowed_fails=final_settings.get("allowed_fails", 3),
cooldown_time=final_settings.get("cooldown_time", 60),
set_verbose=False,
)
instance._initialized = True
logger.info(
"Vision LLM Router initialized with %d deployments, strategy: %s",
len(model_list),
final_settings.get("routing_strategy"),
)
except Exception as e:
logger.error(f"Failed to initialize Vision LLM Router: {e}")
instance._router = None
@classmethod
def _config_to_deployment(cls, config: dict) -> dict | None:
try:
if not config.get("model_name") or not config.get("api_key"):
return None
if config.get("custom_provider"):
model_string = f"{config['custom_provider']}/{config['model_name']}"
else:
provider = config.get("provider", "").upper()
provider_prefix = VISION_PROVIDER_MAP.get(provider, provider.lower())
model_string = f"{provider_prefix}/{config['model_name']}"
litellm_params: dict[str, Any] = {
"model": model_string,
"api_key": config.get("api_key"),
}
if config.get("api_base"):
litellm_params["api_base"] = config["api_base"]
if config.get("api_version"):
litellm_params["api_version"] = config["api_version"]
if config.get("litellm_params"):
litellm_params.update(config["litellm_params"])
deployment: dict[str, Any] = {
"model_name": "auto",
"litellm_params": litellm_params,
}
if config.get("rpm"):
deployment["rpm"] = config["rpm"]
if config.get("tpm"):
deployment["tpm"] = config["tpm"]
return deployment
except Exception as e:
logger.warning(f"Failed to convert vision config to deployment: {e}")
return None
@classmethod
def get_router(cls) -> Router | None:
instance = cls.get_instance()
return instance._router
@classmethod
def is_initialized(cls) -> bool:
instance = cls.get_instance()
return instance._initialized and instance._router is not None
@classmethod
def get_model_count(cls) -> int:
instance = cls.get_instance()
return len(instance._model_list)
def is_vision_auto_mode(config_id: int | None) -> bool:
return config_id == VISION_AUTO_MODE_ID
def build_vision_model_string(
provider: str, model_name: str, custom_provider: str | None
) -> str:
if custom_provider:
return f"{custom_provider}/{model_name}"
prefix = VISION_PROVIDER_MAP.get(provider.upper(), provider.lower())
return f"{prefix}/{model_name}"
def get_global_vision_llm_config(config_id: int) -> dict | None:
from app.config import config
if config_id == VISION_AUTO_MODE_ID:
return {
"id": VISION_AUTO_MODE_ID,
"name": "Auto (Fastest)",
"provider": "AUTO",
"model_name": "auto",
"is_auto_mode": True,
}
if config_id > 0:
return None
for cfg in config.GLOBAL_VISION_LLM_CONFIGS:
if cfg.get("id") == config_id:
return cfg
return None

4
surfsense_desktop/.env
View file

@ -4,3 +4,7 @@
# The hosted web frontend URL. Used to intercept OAuth redirects and keep them
# inside the desktop app. Set to your production frontend domain.
HOSTED_FRONTEND_URL=https://surfsense.net
# PostHog analytics (leave empty to disable)
POSTHOG_KEY=
POSTHOG_HOST=https://assets.surfsense.com

4
surfsense_desktop/package.json
View file

@ -34,6 +34,8 @@
"electron-store": "^11.0.2",
"electron-updater": "^6.8.3",
"get-port-please": "^3.2.0",
"node-mac-permissions": "^2.5.0"
"node-mac-permissions": "^2.5.0",
"node-machine-id": "^1.1.12",
"posthog-node": "^5.29.0"
}
}

31
surfsense_desktop/pnpm-lock.yaml generated
View file

@ -26,6 +26,12 @@ importers:
node-mac-permissions:
specifier: ^2.5.0
version: 2.5.0
node-machine-id:
specifier: ^1.1.12
version: 1.1.12
posthog-node:
specifier: ^5.29.0
version: 5.29.0(rxjs@7.8.2)
devDependencies:
'@electron/rebuild':
specifier: ^4.0.3
@ -308,6 +314,9 @@ packages:
resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==}
engines: {node: '>=14'}
'@posthog/core@1.25.0':
resolution: {integrity: sha512-XKaHvRFIIN7Dw84r1eKimV1rl9DS+9XMCPPZ7P3+l8fE+rDsmumebiTFsY+q40bVXflcGW9wB+57LH0lvcGmhw==}
'@sindresorhus/is@4.6.0':
resolution: {integrity: sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw==}
engines: {node: '>=10'}
@ -1194,6 +1203,9 @@ packages:
resolution: {integrity: sha512-zR8SVCaN3WqV1xwWd04XVAdzm3UTdjbxciLrZtB0Cc7F2Kd34AJfhPD4hm1HU0YH3oGUZO4X9OBLY5ijSTHsGw==}
os: [darwin]
node-machine-id@1.1.12:
resolution: {integrity: sha512-QNABxbrPa3qEIfrE6GOJ7BYIuignnJw7iQ2YPbc3Nla1HzRJjXzZOiikfF8m7eAMfichLt3M4VgLOetqgDmgGQ==}
nopt@8.1.0:
resolution: {integrity: sha512-ieGu42u/Qsa4TFktmaKEwM6MQH0pOWnaB3htzh0JRtx84+Mebc0cbZYN5bC+6WTZ4+77xrL9Pn5m7CV6VIkV7A==}
engines: {node: ^18.17.0 || >=20.5.0}
@ -1263,6 +1275,15 @@ packages:
resolution: {integrity: sha512-uysumyrvkUX0rX/dEVqt8gC3sTBzd4zoWfLeS29nb53imdaXVvLINYXTI2GNqzaMuvacNx4uJQ8+b3zXR0pkgQ==}
engines: {node: '>=10.4.0'}
posthog-node@5.29.0:
resolution: {integrity: sha512-po7N55haSKxV8VOulkBZJja938yILShl6+fFjoUV3iQgOBCg4Muu615/xRg8mpNiz+UASvL0EEiGvIxdhXfj6Q==}
engines: {node: ^20.20.0 || >=22.22.0}
peerDependencies:
rxjs: ^7.0.0
peerDependenciesMeta:
rxjs:
optional: true
postject@1.0.0-alpha.6:
resolution: {integrity: sha512-b9Eb8h2eVqNE8edvKdwqkrY6O7kAwmI8kcnBv1NScolYJbo59XUF0noFq+lxbC1yN20bmC0WBEbDC5H/7ASb0A==}
engines: {node: '>=14.0.0'}
@ -1876,6 +1897,8 @@ snapshots:
'@pkgjs/parseargs@0.11.0':
optional: true
'@posthog/core@1.25.0': {}
'@sindresorhus/is@4.6.0': {}
'@standard-schema/spec@1.1.0': {}
@ -2940,6 +2963,8 @@ snapshots:
bindings: 1.5.0
node-addon-api: 7.1.1
node-machine-id@1.1.12: {}
nopt@8.1.0:
dependencies:
abbrev: 3.0.1
@ -3002,6 +3027,12 @@ snapshots:
base64-js: 1.5.1
xmlbuilder: 15.1.1
posthog-node@5.29.0(rxjs@7.8.2):
dependencies:
'@posthog/core': 1.25.0
optionalDependencies:
rxjs: 7.8.2
postject@1.0.0-alpha.6:
dependencies:
commander: 9.5.0

6
surfsense_desktop/scripts/build-electron.mjs
View file

@ -111,6 +111,12 @@ async function buildElectron() {
'process.env.HOSTED_FRONTEND_URL': JSON.stringify(
process.env.HOSTED_FRONTEND_URL || desktopEnv.HOSTED_FRONTEND_URL || 'https://surfsense.net'
),
'process.env.POSTHOG_KEY': JSON.stringify(
process.env.POSTHOG_KEY || desktopEnv.POSTHOG_KEY || ''
),
'process.env.POSTHOG_HOST': JSON.stringify(
process.env.POSTHOG_HOST || desktopEnv.POSTHOG_HOST || 'https://assets.surfsense.com'
),
},
};

11
surfsense_desktop/src/main.ts
View file

@ -12,6 +12,7 @@ import { registerAutocomplete, unregisterAutocomplete } from './modules/autocomp
import { registerFolderWatcher, unregisterFolderWatcher } from './modules/folder-watcher';
import { registerIpcHandlers } from './ipc/handlers';
import { createTray, destroyTray } from './modules/tray';
import { initAnalytics, shutdownAnalytics, trackEvent } from './modules/analytics';
registerGlobalErrorHandlers();
@ -22,6 +23,8 @@ if (!setupDeepLinks()) {
registerIpcHandlers();
app.whenReady().then(async () => {
initAnalytics();
trackEvent('desktop_app_launched');
setupMenu();
try {
await startNextServer();
@ -70,9 +73,15 @@ app.on('before-quit', () => {
isQuitting = true;
});
app.on('will-quit', () => {
let didCleanup = false;
app.on('will-quit', async (e) => {
if (didCleanup) return;
didCleanup = true;
e.preventDefault();
unregisterQuickAsk();
unregisterAutocomplete();
unregisterFolderWatcher();
destroyTray();
await shutdownAnalytics();
app.exit();
});

50
surfsense_desktop/src/modules/analytics.ts Normal file
View file

@ -0,0 +1,50 @@
import { PostHog } from 'posthog-node';
import { machineIdSync } from 'node-machine-id';
import { app } from 'electron';
let client: PostHog | null = null;
let distinctId = '';
export function initAnalytics(): void {
const key = process.env.POSTHOG_KEY;
if (!key) return;
try {
distinctId = machineIdSync(true);
} catch {
return;
}
client = new PostHog(key, {
host: process.env.POSTHOG_HOST || 'https://assets.surfsense.com',
flushAt: 20,
flushInterval: 10000,
});
}
export function trackEvent(event: string, properties?: Record<string, unknown>): void {
if (!client) return;
try {
client.capture({
distinctId,
event,
properties: {
platform: 'desktop',
app_version: app.getVersion(),
os: process.platform,
...properties,
},
});
} catch {
// Analytics should never break the app
}
}
export async function shutdownAnalytics(): Promise<void> {
if (!client) return;
const timeout = new Promise<void>((resolve) => setTimeout(resolve, 3000));
await Promise.race([client.shutdown(), timeout]);
client = null;
}

4
surfsense_desktop/src/modules/autocomplete/index.ts
View file

@ -6,6 +6,7 @@ import { captureScreen } from './screenshot';
import { createSuggestionWindow, destroySuggestion, getSuggestionWindow } from './suggestion-window';
import { getShortcuts } from '../shortcuts';
import { getActiveSearchSpaceId } from '../active-search-space';
import { trackEvent } from '../analytics';
let currentShortcut = '';
let autocompleteEnabled = true;
@ -41,6 +42,7 @@ async function triggerAutocomplete(): Promise<void> {
console.warn('[autocomplete] No active search space. Select a search space first.');
return;
}
trackEvent('desktop_autocomplete_triggered', { search_space_id: searchSpaceId });
const cursor = screen.getCursorScreenPoint();
const win = createSuggestionWindow(cursor.x, cursor.y);
@ -87,9 +89,11 @@ function registerIpcHandlers(): void {
ipcRegistered = true;
ipcMain.handle(IPC_CHANNELS.ACCEPT_SUGGESTION, async (_event, text: string) => {
trackEvent('desktop_autocomplete_accepted');
await acceptAndInject(text);
});
ipcMain.handle(IPC_CHANNELS.DISMISS_SUGGESTION, () => {
trackEvent('desktop_autocomplete_dismissed');
destroySuggestion();
});
ipcMain.handle(IPC_CHANNELS.SET_AUTOCOMPLETE_ENABLED, (_event, enabled: boolean) => {

3
surfsense_desktop/src/modules/quick-ask.ts
View file

@ -5,6 +5,7 @@ import { checkAccessibilityPermission, getFrontmostApp, simulateCopy, simulatePa
import { getServerPort } from './server';
import { getShortcuts } from './shortcuts';
import { getActiveSearchSpaceId } from './active-search-space';
import { trackEvent } from './analytics';
let currentShortcut = '';
let quickAskWindow: BrowserWindow | null = null;
@ -121,6 +122,7 @@ async function quickAskHandler(): Promise<void> {
sourceApp = getFrontmostApp();
console.log('[quick-ask] Source app:', sourceApp, '| Opening Quick Assist with', text.length, 'chars', selected ? '(selected)' : text ? '(clipboard fallback)' : '(empty)');
trackEvent('desktop_quick_ask_opened', { has_selected_text: !!selected });
openQuickAsk(text);
}
@ -152,6 +154,7 @@ function registerIpcHandlers(): void {
if (!checkAccessibilityPermission()) return;
trackEvent('desktop_quick_ask_replaced');
clipboard.writeText(text);
destroyQuickAsk();

3
surfsense_web/.env.example
View file

@ -8,3 +8,6 @@ DATABASE_URL=postgresql://postgres:[YOUR-PASSWORD]@db.sdsf.supabase.co:5432/post
# Deployment mode (optional)
NEXT_PUBLIC_DEPLOYMENT_MODE="self-hosted" or "cloud"
# PostHog analytics (optional, leave empty to disable)
NEXT_PUBLIC_POSTHOG_KEY=

13
surfsense_web/app/dashboard/[search_space_id]/client-layout.tsx
View file

@ -154,11 +154,14 @@ export function DashboardClientLayout({
// Sync to Electron store if stored value is null (first navigation)
if (electronAPI?.setActiveSearchSpace) {
electronAPI.getActiveSearchSpace?.().then((stored) => {
if (!stored) {
electronAPI.setActiveSearchSpace!(activeSeacrhSpaceId);
}
}).catch(() => {});
electronAPI
.getActiveSearchSpace?.()
.then((stored) => {
if (!stored) {
electronAPI.setActiveSearchSpace!(activeSeacrhSpaceId);
}
})
.catch(() => {});
}
}, [search_space_id, setActiveSearchSpaceIdState, electronAPI]);

78
surfsense_web/app/dashboard/[search_space_id]/user-settings/components/DesktopContent.tsx
View file

@ -6,12 +6,18 @@ import { toast } from "sonner";
import { DEFAULT_SHORTCUTS, ShortcutRecorder } from "@/components/desktop/shortcut-recorder";
import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";
import { Label } from "@/components/ui/label";
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from "@/components/ui/select";
import { Spinner } from "@/components/ui/spinner";
import { Switch } from "@/components/ui/switch";
import type { SearchSpace } from "@/contracts/types/search-space.types";
import { useElectronAPI } from "@/hooks/use-platform";
import { searchSpacesApiService } from "@/lib/apis/search-spaces-api.service";
import type { SearchSpace } from "@/contracts/types/search-space.types";
export function DesktopContent() {
const api = useElectronAPI();
@ -82,7 +88,10 @@ export function DesktopContent() {
await api.setAutocompleteEnabled(checked);
};
const updateShortcut = (key: "generalAssist" | "quickAsk" | "autocomplete", accelerator: string) => {
const updateShortcut = (
key: "generalAssist" | "quickAsk" | "autocomplete",
accelerator: string
) => {
setShortcuts((prev) => {
const updated = { ...prev, [key]: accelerator };
api.setShortcuts?.({ [key]: accelerator }).catch(() => {
@ -110,7 +119,8 @@ export function DesktopContent() {
<CardHeader className="px-3 md:px-6 pt-3 md:pt-6 pb-2 md:pb-3">
<CardTitle className="text-base md:text-lg">Default Search Space</CardTitle>
<CardDescription className="text-xs md:text-sm">
Choose which search space General Assist, Quick Assist, and Extreme Assist operate against.
Choose which search space General Assist, Quick Assist, and Extreme Assist operate
against.
</CardDescription>
</CardHeader>
<CardContent className="px-3 md:px-6 pb-3 md:pb-6">
@ -128,7 +138,9 @@ export function DesktopContent() {
</SelectContent>
</Select>
) : (
<p className="text-sm text-muted-foreground">No search spaces found. Create one first.</p>
<p className="text-sm text-muted-foreground">
No search spaces found. Create one first.
</p>
)}
</CardContent>
</Card>
@ -143,34 +155,34 @@ export function DesktopContent() {
</CardHeader>
<CardContent className="px-3 md:px-6 pb-3 md:pb-6">
{shortcutsLoaded ? (
<div className="flex flex-col gap-3">
<ShortcutRecorder
value={shortcuts.generalAssist}
onChange={(accel) => updateShortcut("generalAssist", accel)}
onReset={() => resetShortcut("generalAssist")}
defaultValue={DEFAULT_SHORTCUTS.generalAssist}
label="General Assist"
description="Launch SurfSense instantly from any application"
icon={Rocket}
/>
<ShortcutRecorder
value={shortcuts.quickAsk}
onChange={(accel) => updateShortcut("quickAsk", accel)}
onReset={() => resetShortcut("quickAsk")}
defaultValue={DEFAULT_SHORTCUTS.quickAsk}
label="Quick Assist"
description="Select text anywhere, then ask AI to explain, rewrite, or act on it"
icon={Zap}
/>
<ShortcutRecorder
value={shortcuts.autocomplete}
onChange={(accel) => updateShortcut("autocomplete", accel)}
onReset={() => resetShortcut("autocomplete")}
defaultValue={DEFAULT_SHORTCUTS.autocomplete}
label="Extreme Assist"
description="AI drafts text using your screen context and knowledge base"
icon={BrainCog}
/>
<div className="flex flex-col gap-3">
<ShortcutRecorder
value={shortcuts.generalAssist}
onChange={(accel) => updateShortcut("generalAssist", accel)}
onReset={() => resetShortcut("generalAssist")}
defaultValue={DEFAULT_SHORTCUTS.generalAssist}
label="General Assist"
description="Launch SurfSense instantly from any application"
icon={Rocket}
/>
<ShortcutRecorder
value={shortcuts.quickAsk}
onChange={(accel) => updateShortcut("quickAsk", accel)}
onReset={() => resetShortcut("quickAsk")}
defaultValue={DEFAULT_SHORTCUTS.quickAsk}
label="Quick Assist"
description="Select text anywhere, then ask AI to explain, rewrite, or act on it"
icon={Zap}
/>
<ShortcutRecorder
value={shortcuts.autocomplete}
onChange={(accel) => updateShortcut("autocomplete", accel)}
onReset={() => resetShortcut("autocomplete")}
defaultValue={DEFAULT_SHORTCUTS.autocomplete}
label="Extreme Assist"
description="AI drafts text using your screen context and knowledge base"
icon={BrainCog}
/>
<p className="text-[11px] text-muted-foreground">
Click a shortcut and press a new key combination to change it.
</p>

4
surfsense_web/app/desktop/login/page.tsx
View file

@ -139,9 +139,7 @@ export default function DesktopLoginPage() {
height={48}
priority
/>
<h1 className="text-lg font-semibold tracking-tight">
Welcome to SurfSense Desktop
</h1>
<h1 className="text-lg font-semibold tracking-tight">Welcome to SurfSense Desktop</h1>
<p className="mt-1 text-sm text-muted-foreground">
Configure shortcuts, then sign in to get started.
</p>

158
surfsense_web/app/desktop/suggestion/page.tsx
View file

@ -14,6 +14,10 @@ type SSEEvent =
| {
type: "data-thinking-step";
data: { id: string; title: string; status: string; items: string[] };
}
| {
type: "data-suggestions";
data: { options: string[] };
};
interface AgentStep {
@ -23,24 +27,32 @@ interface AgentStep {
items: string[];
}
function friendlyError(raw: string | number): string {
type FriendlyError = { message: string; isSetup?: boolean };
function friendlyError(raw: string | number): FriendlyError {
if (typeof raw === "number") {
if (raw === 401) return "Please sign in to use suggestions.";
if (raw === 403) return "You don\u2019t have permission for this.";
if (raw === 404) return "Suggestion service not found. Is the backend running?";
if (raw >= 500) return "Something went wrong on the server. Try again.";
return "Something went wrong. Try again.";
if (raw === 401) return { message: "Please sign in to use suggestions." };
if (raw === 403) return { message: "You don\u2019t have permission for this." };
if (raw === 404) return { message: "Suggestion service not found. Is the backend running?" };
if (raw >= 500) return { message: "Something went wrong on the server. Try again." };
return { message: "Something went wrong. Try again." };
}
const lower = raw.toLowerCase();
if (lower.includes("not authenticated") || lower.includes("unauthorized"))
return "Please sign in to use suggestions.";
return { message: "Please sign in to use suggestions." };
if (lower.includes("no vision llm configured") || lower.includes("no llm configured"))
return "No Vision LLM configured. Set one in search space settings.";
return {
message: "Configure a vision-capable model (e.g. GPT-4o, Gemini) to enable autocomplete.",
isSetup: true,
};
if (lower.includes("does not support vision"))
return "Selected model doesn\u2019t support vision. Set a vision-capable model in settings.";
return {
message: "The selected model doesn\u2019t support vision. Choose a vision-capable model.",
isSetup: true,
};
if (lower.includes("fetch") || lower.includes("network") || lower.includes("econnrefused"))
return "Can\u2019t reach the server. Check your connection.";
return "Something went wrong. Try again.";
return { message: "Can\u2019t reach the server. Check your connection." };
return { message: "Something went wrong. Try again." };
}
const AUTO_DISMISS_MS = 3000;
@ -70,10 +82,11 @@ function StepIcon({ status }: { status: string }) {
export default function SuggestionPage() {
const api = useElectronAPI();
const [suggestion, setSuggestion] = useState("");
const [options, setOptions] = useState<string[]>([]);
const [isLoading, setIsLoading] = useState(true);
const [error, setError] = useState<string | null>(null);
const [error, setError] = useState<FriendlyError | null>(null);
const [steps, setSteps] = useState<AgentStep[]>([]);
const [expandedOption, setExpandedOption] = useState<number | null>(null);
const abortRef = useRef<AbortController | null>(null);
const isDesktop = !!api?.onAutocompleteContext;
@ -85,13 +98,21 @@ export default function SuggestionPage() {
}, [api]);
useEffect(() => {
if (!error) return;
if (!error || error.isSetup) return;
const timer = setTimeout(() => {
api?.dismissSuggestion?.();
}, AUTO_DISMISS_MS);
return () => clearTimeout(timer);
}, [error, api]);
useEffect(() => {
if (isLoading || error || options.length > 0) return;
const timer = setTimeout(() => {
api?.dismissSuggestion?.();
}, AUTO_DISMISS_MS);
return () => clearTimeout(timer);
}, [isLoading, error, options, api]);
const fetchSuggestion = useCallback(
async (screenshot: string, searchSpaceId: string, appName?: string, windowTitle?: string) => {
abortRef.current?.abort();
@ -99,9 +120,10 @@ export default function SuggestionPage() {
abortRef.current = controller;
setIsLoading(true);
setSuggestion("");
setOptions([]);
setError(null);
setSteps([]);
setExpandedOption(null);
let token = getBearerToken();
if (!token) {
@ -165,8 +187,8 @@ export default function SuggestionPage() {
try {
const parsed: SSEEvent = JSON.parse(data);
if (parsed.type === "text-delta") {
setSuggestion((prev) => prev + parsed.delta);
if (parsed.type === "data-suggestions") {
setOptions(parsed.data.options);
} else if (parsed.type === "error") {
setError(friendlyError(parsed.errorText));
} else if (parsed.type === "data-thinking-step") {
@ -219,14 +241,52 @@ export default function SuggestionPage() {
}
if (error) {
if (error.isSetup) {
return (
<div className="suggestion-tooltip suggestion-setup">
<div className="setup-icon">
<svg viewBox="0 0 24 24" fill="none" width="28" height="28" aria-hidden="true">
<path
d="M1 12C1 12 5 4 12 4C19 4 23 12 23 12C23 12 19 20 12 20C5 20 1 12 1 12Z"
stroke="#a78bfa"
strokeWidth="1.5"
strokeLinecap="round"
strokeLinejoin="round"
/>
<circle
cx="12"
cy="12"
r="3"
stroke="#a78bfa"
strokeWidth="1.5"
strokeLinecap="round"
strokeLinejoin="round"
/>
</svg>
</div>
<div className="setup-content">
<span className="setup-title">Vision Model Required</span>
<span className="setup-message">{error.message}</span>
<span className="setup-hint">Settings → Vision Models</span>
</div>
<button
type="button"
className="setup-dismiss"
onClick={() => api?.dismissSuggestion?.()}
>
✕
</button>
</div>
);
}
return (
<div className="suggestion-tooltip suggestion-error">
<span className="suggestion-error-text">{error}</span>
<span className="suggestion-error-text">{error.message}</span>
</div>
);
}
const showLoading = isLoading && !suggestion;
const showLoading = isLoading && options.length === 0;
if (showLoading) {
return (
@ -258,29 +318,63 @@ export default function SuggestionPage() {
);
}
const handleAccept = () => {
if (suggestion) {
api?.acceptSuggestion?.(suggestion);
}
const handleSelect = (text: string) => {
api?.acceptSuggestion?.(text);
};
const handleDismiss = () => {
api?.dismissSuggestion?.();
};
if (!suggestion) return null;
const TRUNCATE_LENGTH = 120;
if (options.length === 0) {
return (
<div className="suggestion-tooltip suggestion-error">
<span className="suggestion-error-text">No suggestions available.</span>
</div>
);
}
return (
<div className="suggestion-tooltip">
<p className="suggestion-text">{suggestion}</p>
<div className="suggestion-options">
{options.map((option, index) => {
const isExpanded = expandedOption === index;
const needsTruncation = option.length > TRUNCATE_LENGTH;
const displayText =
needsTruncation && !isExpanded ? option.slice(0, TRUNCATE_LENGTH) + "…" : option;
return (
<div
key={index}
role="button"
tabIndex={0}
className="suggestion-option"
onClick={() => handleSelect(option)}
onKeyDown={(e) => {
if (e.key === "Enter") handleSelect(option);
}}
>
<span className="option-number">{index + 1}</span>
<span className="option-text">{displayText}</span>
{needsTruncation && (
<button
type="button"
className="option-expand"
onClick={(e) => {
e.stopPropagation();
setExpandedOption(isExpanded ? null : index);
}}
>
{isExpanded ? "less" : "more"}
</button>
)}
</div>
);
})}
</div>
<div className="suggestion-actions">
<button
type="button"
className="suggestion-btn suggestion-btn-accept"
onClick={handleAccept}
>
Accept
</button>
<button
type="button"
className="suggestion-btn suggestion-btn-dismiss"

157
surfsense_web/app/desktop/suggestion/suggestion.css
View file

@ -117,6 +117,66 @@ body:has(.suggestion-body) {
font-size: 12px;
}
/* --- Setup prompt (vision model not configured) --- */
.suggestion-setup {
display: flex;
flex-direction: row;
align-items: flex-start;
gap: 10px;
border-color: #3b2d6b;
padding: 10px 14px;
}
.setup-icon {
flex-shrink: 0;
margin-top: 1px;
}
.setup-content {
display: flex;
flex-direction: column;
gap: 3px;
min-width: 0;
}
.setup-title {
font-size: 13px;
font-weight: 600;
color: #c4b5fd;
}
.setup-message {
font-size: 11.5px;
color: #a1a1aa;
line-height: 1.4;
}
.setup-hint {
font-size: 10.5px;
color: #7c6dac;
margin-top: 2px;
}
.setup-dismiss {
flex-shrink: 0;
align-self: flex-start;
background: none;
border: none;
color: #6b6b7b;
font-size: 14px;
cursor: pointer;
padding: 2px 4px;
line-height: 1;
border-radius: 4px;
transition: color 0.15s, background 0.15s;
}
.setup-dismiss:hover {
color: #c4b5fd;
background: rgba(124, 109, 172, 0.15);
}
/* --- Agent activity indicator --- */
.agent-activity {
@ -127,6 +187,10 @@ body:has(.suggestion-body) {
max-height: 340px;
}
.agent-activity::-webkit-scrollbar {
display: none;
}
.activity-initial {
display: flex;
align-items: center;
@ -191,3 +255,96 @@ body:has(.suggestion-body) {
transform: rotate(360deg);
}
}
/* --- Suggestion option cards --- */
.suggestion-options {
display: flex;
flex-direction: column;
gap: 4px;
overflow-y: auto;
flex: 1 1 auto;
min-height: 0;
margin-bottom: 6px;
}
.suggestion-options::-webkit-scrollbar {
width: 5px;
}
.suggestion-options::-webkit-scrollbar-track {
background: transparent;
}
.suggestion-options::-webkit-scrollbar-thumb {
background: #555;
border-radius: 3px;
}
.suggestion-option {
display: flex;
align-items: flex-start;
gap: 8px;
padding: 6px 8px;
border-radius: 5px;
border: 1px solid #333;
background: #262626;
cursor: pointer;
text-align: left;
font-family: inherit;
transition:
background 0.15s,
border-color 0.15s;
width: 100%;
}
.suggestion-option:hover {
background: #2a2d3a;
border-color: #3b82f6;
}
.option-number {
flex-shrink: 0;
width: 18px;
height: 18px;
border-radius: 50%;
background: #3f3f46;
color: #d4d4d4;
font-size: 10px;
font-weight: 600;
display: flex;
align-items: center;
justify-content: center;
margin-top: 1px;
}
.suggestion-option:hover .option-number {
background: #2563eb;
color: #fff;
}
.option-text {
color: #d4d4d4;
font-size: 12px;
line-height: 1.45;
word-wrap: break-word;
white-space: pre-wrap;
flex: 1 1 auto;
min-width: 0;
}
.option-expand {
flex-shrink: 0;
background: none;
border: none;
color: #71717a;
font-size: 10px;
cursor: pointer;
padding: 0 2px;
font-family: inherit;
margin-top: 1px;
}
.option-expand:hover {
color: #a1a1aa;
}

84
surfsense_web/atoms/vision-llm-config/vision-llm-config-mutation.atoms.ts Normal file
View file

@ -0,0 +1,84 @@
import { atomWithMutation } from "jotai-tanstack-query";
import { toast } from "sonner";
import type {
CreateVisionLLMConfigRequest,
CreateVisionLLMConfigResponse,
DeleteVisionLLMConfigResponse,
GetVisionLLMConfigsResponse,
UpdateVisionLLMConfigRequest,
UpdateVisionLLMConfigResponse,
} from "@/contracts/types/new-llm-config.types";
import { visionLLMConfigApiService } from "@/lib/apis/vision-llm-config-api.service";
import { cacheKeys } from "@/lib/query-client/cache-keys";
import { queryClient } from "@/lib/query-client/client";
import { activeSearchSpaceIdAtom } from "../search-spaces/search-space-query.atoms";
export const createVisionLLMConfigMutationAtom = atomWithMutation((get) => {
const searchSpaceId = get(activeSearchSpaceIdAtom);
return {
mutationKey: ["vision-llm-configs", "create"],
enabled: !!searchSpaceId,
mutationFn: async (request: CreateVisionLLMConfigRequest) => {
return visionLLMConfigApiService.createConfig(request);
},
onSuccess: (_: CreateVisionLLMConfigResponse, request: CreateVisionLLMConfigRequest) => {
toast.success(`${request.name} created`);
queryClient.invalidateQueries({
queryKey: cacheKeys.visionLLMConfigs.all(Number(searchSpaceId)),
});
},
onError: (error: Error) => {
toast.error(error.message || "Failed to create vision model");
},
};
});
export const updateVisionLLMConfigMutationAtom = atomWithMutation((get) => {
const searchSpaceId = get(activeSearchSpaceIdAtom);
return {
mutationKey: ["vision-llm-configs", "update"],
enabled: !!searchSpaceId,
mutationFn: async (request: UpdateVisionLLMConfigRequest) => {
return visionLLMConfigApiService.updateConfig(request);
},
onSuccess: (_: UpdateVisionLLMConfigResponse, request: UpdateVisionLLMConfigRequest) => {
toast.success(`${request.data.name ?? "Configuration"} updated`);
queryClient.invalidateQueries({
queryKey: cacheKeys.visionLLMConfigs.all(Number(searchSpaceId)),
});
queryClient.invalidateQueries({
queryKey: cacheKeys.visionLLMConfigs.byId(request.id),
});
},
onError: (error: Error) => {
toast.error(error.message || "Failed to update vision model");
},
};
});
export const deleteVisionLLMConfigMutationAtom = atomWithMutation((get) => {
const searchSpaceId = get(activeSearchSpaceIdAtom);
return {
mutationKey: ["vision-llm-configs", "delete"],
enabled: !!searchSpaceId,
mutationFn: async (request: { id: number; name: string }) => {
return visionLLMConfigApiService.deleteConfig(request.id);
},
onSuccess: (_: DeleteVisionLLMConfigResponse, request: { id: number; name: string }) => {
toast.success(`${request.name} deleted`);
queryClient.setQueryData(
cacheKeys.visionLLMConfigs.all(Number(searchSpaceId)),
(oldData: GetVisionLLMConfigsResponse | undefined) => {
if (!oldData) return oldData;
return oldData.filter((config) => config.id !== request.id);
}
);
},
onError: (error: Error) => {
toast.error(error.message || "Failed to delete vision model");
},
};
});

27
surfsense_web/atoms/vision-llm-config/vision-llm-config-query.atoms.ts Normal file
View file

@ -0,0 +1,27 @@
import { atomWithQuery } from "jotai-tanstack-query";
import { visionLLMConfigApiService } from "@/lib/apis/vision-llm-config-api.service";
import { cacheKeys } from "@/lib/query-client/cache-keys";
import { activeSearchSpaceIdAtom } from "../search-spaces/search-space-query.atoms";
export const visionLLMConfigsAtom = atomWithQuery((get) => {
const searchSpaceId = get(activeSearchSpaceIdAtom);
return {
queryKey: cacheKeys.visionLLMConfigs.all(Number(searchSpaceId)),
enabled: !!searchSpaceId,
staleTime: 5 * 60 * 1000,
queryFn: async () => {
return visionLLMConfigApiService.getConfigs(Number(searchSpaceId));
},
};
});
export const globalVisionLLMConfigsAtom = atomWithQuery(() => {
return {
queryKey: cacheKeys.visionLLMConfigs.global(),
staleTime: 10 * 60 * 1000,
queryFn: async () => {
return visionLLMConfigApiService.getGlobalConfigs();
},
};
});

2
surfsense_web/components/TokenHandler.tsx
View file

@ -2,8 +2,8 @@
import { useEffect } from "react";
import { useGlobalLoadingEffect } from "@/hooks/use-global-loading";
import { getAndClearRedirectPath, setBearerToken, setRefreshToken } from "@/lib/auth-utils";
import { searchSpacesApiService } from "@/lib/apis/search-spaces-api.service";
import { getAndClearRedirectPath, setBearerToken, setRefreshToken } from "@/lib/auth-utils";
import { trackLoginSuccess } from "@/lib/posthog/events";
interface TokenHandlerProps {

4
surfsense_web/components/desktop/shortcut-recorder.tsx
View file

@ -150,7 +150,9 @@ export function ShortcutRecorder({
)}
>
{recording ? (
<span className="text-[11px] text-primary animate-pulse whitespace-nowrap">Press keys…</span>
<span className="text-[11px] text-primary animate-pulse whitespace-nowrap">
Press keys…
</span>
) : (
<Kbd keys={displayKeys} />
)}

44
surfsense_web/components/new-chat/chat-header.tsx
View file

@ -3,11 +3,14 @@
import { useCallback, useState } from "react";
import { ImageConfigDialog } from "@/components/shared/image-config-dialog";
import { ModelConfigDialog } from "@/components/shared/model-config-dialog";
import { VisionConfigDialog } from "@/components/shared/vision-config-dialog";
import type {
GlobalImageGenConfig,
GlobalNewLLMConfig,
GlobalVisionLLMConfig,
ImageGenerationConfig,
NewLLMConfigPublic,
VisionLLMConfig,
} from "@/contracts/types/new-llm-config.types";
import { ModelSelector } from "./model-selector";
@ -33,6 +36,14 @@ export function ChatHeader({ searchSpaceId, className }: ChatHeaderProps) {
const [isImageGlobal, setIsImageGlobal] = useState(false);
const [imageDialogMode, setImageDialogMode] = useState<"create" | "edit" | "view">("view");
// Vision config dialog state
const [visionDialogOpen, setVisionDialogOpen] = useState(false);
const [selectedVisionConfig, setSelectedVisionConfig] = useState<
VisionLLMConfig | GlobalVisionLLMConfig | null
>(null);
const [isVisionGlobal, setIsVisionGlobal] = useState(false);
const [visionDialogMode, setVisionDialogMode] = useState<"create" | "edit" | "view">("view");
// LLM handlers
const handleEditLLMConfig = useCallback(
(config: NewLLMConfigPublic | GlobalNewLLMConfig, global: boolean) => {
@ -79,6 +90,29 @@ export function ChatHeader({ searchSpaceId, className }: ChatHeaderProps) {
if (!open) setSelectedImageConfig(null);
}, []);
// Vision model handlers
const handleAddVisionModel = useCallback(() => {
setSelectedVisionConfig(null);
setIsVisionGlobal(false);
setVisionDialogMode("create");
setVisionDialogOpen(true);
}, []);
const handleEditVisionConfig = useCallback(
(config: VisionLLMConfig | GlobalVisionLLMConfig, global: boolean) => {
setSelectedVisionConfig(config);
setIsVisionGlobal(global);
setVisionDialogMode(global ? "view" : "edit");
setVisionDialogOpen(true);
},
[]
);
const handleVisionDialogClose = useCallback((open: boolean) => {
setVisionDialogOpen(open);
if (!open) setSelectedVisionConfig(null);
}, []);
return (
<div className="flex items-center gap-2">
<ModelSelector
@ -86,6 +120,8 @@ export function ChatHeader({ searchSpaceId, className }: ChatHeaderProps) {
onAddNewLLM={handleAddNewLLM}
onEditImage={handleEditImageConfig}
onAddNewImage={handleAddImageModel}
onEditVision={handleEditVisionConfig}
onAddNewVision={handleAddVisionModel}
className={className}
/>
<ModelConfigDialog
@ -104,6 +140,14 @@ export function ChatHeader({ searchSpaceId, className }: ChatHeaderProps) {
searchSpaceId={searchSpaceId}
mode={imageDialogMode}
/>
<VisionConfigDialog
open={visionDialogOpen}
onOpenChange={handleVisionDialogClose}
config={selectedVisionConfig}
isGlobal={isVisionGlobal}
searchSpaceId={searchSpaceId}
mode={visionDialogMode}
/>
</div>
);
}

304
surfsense_web/components/new-chat/model-selector.tsx
View file

@ -1,7 +1,7 @@
"use client";
import { useAtomValue } from "jotai";
import { Bot, Check, ChevronDown, Edit3, ImageIcon, Plus, Search, Zap } from "lucide-react";
import { Bot, Check, ChevronDown, Edit3, Eye, ImageIcon, Plus, Search, Zap } from "lucide-react";
import { type UIEvent, useCallback, useMemo, useState } from "react";
import { toast } from "sonner";
import {
@ -15,6 +15,10 @@ import {
newLLMConfigsAtom,
} from "@/atoms/new-llm-config/new-llm-config-query.atoms";
import { activeSearchSpaceIdAtom } from "@/atoms/search-spaces/search-space-query.atoms";
import {
globalVisionLLMConfigsAtom,
visionLLMConfigsAtom,
} from "@/atoms/vision-llm-config/vision-llm-config-query.atoms";
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
import {
@ -32,8 +36,10 @@ import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
import type {
GlobalImageGenConfig,
GlobalNewLLMConfig,
GlobalVisionLLMConfig,
ImageGenerationConfig,
NewLLMConfigPublic,
VisionLLMConfig,
} from "@/contracts/types/new-llm-config.types";
import { getProviderIcon } from "@/lib/provider-icons";
import { cn } from "@/lib/utils";
@ -43,6 +49,8 @@ interface ModelSelectorProps {
onAddNewLLM: () => void;
onEditImage?: (config: ImageGenerationConfig | GlobalImageGenConfig, isGlobal: boolean) => void;
onAddNewImage?: () => void;
onEditVision?: (config: VisionLLMConfig | GlobalVisionLLMConfig, isGlobal: boolean) => void;
onAddNewVision?: () => void;
className?: string;
}
@ -51,14 +59,18 @@ export function ModelSelector({
onAddNewLLM,
onEditImage,
onAddNewImage,
onEditVision,
onAddNewVision,
className,
}: ModelSelectorProps) {
const [open, setOpen] = useState(false);
const [activeTab, setActiveTab] = useState<"llm" | "image">("llm");
const [activeTab, setActiveTab] = useState<"llm" | "image" | "vision">("llm");
const [llmSearchQuery, setLlmSearchQuery] = useState("");
const [imageSearchQuery, setImageSearchQuery] = useState("");
const [visionSearchQuery, setVisionSearchQuery] = useState("");
const [llmScrollPos, setLlmScrollPos] = useState<"top" | "middle" | "bottom">("top");
const [imageScrollPos, setImageScrollPos] = useState<"top" | "middle" | "bottom">("top");
const [visionScrollPos, setVisionScrollPos] = useState<"top" | "middle" | "bottom">("top");
const handleListScroll = useCallback(
(setter: typeof setLlmScrollPos) => (e: UIEvent<HTMLDivElement>) => {
const el = e.currentTarget;
@ -82,8 +94,21 @@ export function ModelSelector({
useAtomValue(globalImageGenConfigsAtom);
const { data: imageUserConfigs, isLoading: imageUserLoading } = useAtomValue(imageGenConfigsAtom);
// Vision data
const { data: visionGlobalConfigs, isLoading: visionGlobalLoading } = useAtomValue(
globalVisionLLMConfigsAtom
);
const { data: visionUserConfigs, isLoading: visionUserLoading } =
useAtomValue(visionLLMConfigsAtom);
const isLoading =
llmUserLoading || llmGlobalLoading || prefsLoading || imageGlobalLoading || imageUserLoading;
llmUserLoading ||
llmGlobalLoading ||
prefsLoading ||
imageGlobalLoading ||
imageUserLoading ||
visionGlobalLoading ||
visionUserLoading;
// ─── LLM current config ───
const currentLLMConfig = useMemo(() => {
@ -116,6 +141,24 @@ export function ModelSelector({
);
}, [currentImageConfig]);
// ─── Vision current config ───
const currentVisionConfig = useMemo(() => {
if (!preferences) return null;
const id = preferences.vision_llm_config_id;
if (id === null || id === undefined) return null;
const globalMatch = visionGlobalConfigs?.find((c) => c.id === id);
if (globalMatch) return globalMatch;
return visionUserConfigs?.find((c) => c.id === id) ?? null;
}, [preferences, visionGlobalConfigs, visionUserConfigs]);
const isVisionAutoMode = useMemo(() => {
return (
currentVisionConfig &&
"is_auto_mode" in currentVisionConfig &&
currentVisionConfig.is_auto_mode
);
}, [currentVisionConfig]);
// ─── LLM filtering ───
const filteredLLMGlobal = useMemo(() => {
if (!llmGlobalConfigs) return [];
@ -170,6 +213,33 @@ export function ModelSelector({
const totalImageModels = (imageGlobalConfigs?.length ?? 0) + (imageUserConfigs?.length ?? 0);
// ─── Vision filtering ───
const filteredVisionGlobal = useMemo(() => {
if (!visionGlobalConfigs) return [];
if (!visionSearchQuery) return visionGlobalConfigs;
const q = visionSearchQuery.toLowerCase();
return visionGlobalConfigs.filter(
(c) =>
c.name.toLowerCase().includes(q) ||
c.model_name.toLowerCase().includes(q) ||
c.provider.toLowerCase().includes(q)
);
}, [visionGlobalConfigs, visionSearchQuery]);
const filteredVisionUser = useMemo(() => {
if (!visionUserConfigs) return [];
if (!visionSearchQuery) return visionUserConfigs;
const q = visionSearchQuery.toLowerCase();
return visionUserConfigs.filter(
(c) =>
c.name.toLowerCase().includes(q) ||
c.model_name.toLowerCase().includes(q) ||
c.provider.toLowerCase().includes(q)
);
}, [visionUserConfigs, visionSearchQuery]);
const totalVisionModels = (visionGlobalConfigs?.length ?? 0) + (visionUserConfigs?.length ?? 0);
// ─── Handlers ───
const handleSelectLLM = useCallback(
async (config: NewLLMConfigPublic | GlobalNewLLMConfig) => {
@ -229,6 +299,30 @@ export function ModelSelector({
[currentImageConfig, searchSpaceId, updatePreferences]
);
const handleSelectVision = useCallback(
async (configId: number) => {
if (currentVisionConfig?.id === configId) {
setOpen(false);
return;
}
if (!searchSpaceId) {
toast.error("No search space selected");
return;
}
try {
await updatePreferences({
search_space_id: Number(searchSpaceId),
data: { vision_llm_config_id: configId },
});
toast.success("Vision model updated");
setOpen(false);
} catch {
toast.error("Failed to switch vision model");
}
},
[currentVisionConfig, searchSpaceId, updatePreferences]
);
return (
<Popover open={open} onOpenChange={setOpen}>
<PopoverTrigger asChild>
@ -282,6 +376,23 @@ export function ModelSelector({
) : (
<ImageIcon className="size-4 text-muted-foreground" />
)}
{/* Divider */}
<div className="h-4 w-px bg-border/60 dark:bg-white/10 mx-0.5" />
{/* Vision section */}
{currentVisionConfig ? (
<>
{getProviderIcon(currentVisionConfig.provider, {
isAutoMode: isVisionAutoMode ?? false,
})}
<span className="max-w-[80px] md:max-w-[100px] truncate hidden md:inline">
{currentVisionConfig.name}
</span>
</>
) : (
<Eye className="size-4 text-muted-foreground" />
)}
</>
)}
<ChevronDown className="h-3.5 w-3.5 text-muted-foreground ml-1 shrink-0" />
@ -295,25 +406,32 @@ export function ModelSelector({
>
<Tabs
value={activeTab}
onValueChange={(v) => setActiveTab(v as "llm" | "image")}
onValueChange={(v) => setActiveTab(v as "llm" | "image" | "vision")}
className="w-full"
>
<div className="border-b border-border/80 dark:border-neutral-800">
<TabsList className="w-full grid grid-cols-2 rounded-none rounded-t-lg bg-transparent h-11 p-0 gap-0">
<TabsList className="w-full grid grid-cols-3 rounded-none rounded-t-lg bg-transparent h-11 p-0 gap-0">
<TabsTrigger
value="llm"
className="gap-2 text-sm font-medium rounded-none text-muted-foreground transition-all duration-200 h-full bg-transparent data-[state=active]:bg-transparent shadow-none data-[state=active]:shadow-none border-b-[1.5px] border-transparent data-[state=active]:border-foreground dark:data-[state=active]:border-white data-[state=active]:text-foreground"
className="gap-1.5 text-sm font-medium rounded-none text-muted-foreground transition-all duration-200 h-full bg-transparent data-[state=active]:bg-transparent shadow-none data-[state=active]:shadow-none border-b-[1.5px] border-transparent data-[state=active]:border-foreground dark:data-[state=active]:border-white data-[state=active]:text-foreground"
>
<Zap className="size-4" />
<Zap className="size-3.5" />
LLM
</TabsTrigger>
<TabsTrigger
value="image"
className="gap-2 text-sm font-medium rounded-none text-muted-foreground transition-all duration-200 h-full bg-transparent data-[state=active]:bg-transparent shadow-none data-[state=active]:shadow-none border-b-[1.5px] border-transparent data-[state=active]:border-foreground dark:data-[state=active]:border-white data-[state=active]:text-foreground"
className="gap-1.5 text-sm font-medium rounded-none text-muted-foreground transition-all duration-200 h-full bg-transparent data-[state=active]:bg-transparent shadow-none data-[state=active]:shadow-none border-b-[1.5px] border-transparent data-[state=active]:border-foreground dark:data-[state=active]:border-white data-[state=active]:text-foreground"
>
<ImageIcon className="size-4" />
<ImageIcon className="size-3.5" />
Image
</TabsTrigger>
<TabsTrigger
value="vision"
className="gap-1.5 text-sm font-medium rounded-none text-muted-foreground transition-all duration-200 h-full bg-transparent data-[state=active]:bg-transparent shadow-none data-[state=active]:shadow-none border-b-[1.5px] border-transparent data-[state=active]:border-foreground dark:data-[state=active]:border-white data-[state=active]:text-foreground"
>
<Eye className="size-3.5" />
Vision
</TabsTrigger>
</TabsList>
</div>
@ -676,6 +794,174 @@ export function ModelSelector({
</CommandList>
</Command>
</TabsContent>
{/* ─── Vision Tab ─── */}
<TabsContent value="vision" className="mt-0">
<Command
shouldFilter={false}
className="rounded-none rounded-b-lg dark:bg-neutral-900 [&_[data-slot=command-input-wrapper]]:border-0 [&_[data-slot=command-input-wrapper]]:px-0 [&_[data-slot=command-input-wrapper]]:gap-2"
>
{totalVisionModels > 3 && (
<div className="px-2 md:px-3 py-1.5 md:py-2">
<CommandInput
placeholder="Search vision models"
value={visionSearchQuery}
onValueChange={setVisionSearchQuery}
className="h-7 md:h-8 w-full text-xs md:text-sm border-0 bg-transparent focus:ring-0 placeholder:text-muted-foreground/60"
/>
</div>
)}
<CommandList
className="max-h-[300px] md:max-h-[400px] overflow-y-auto"
onScroll={handleListScroll(setVisionScrollPos)}
style={{
maskImage: `linear-gradient(to bottom, ${visionScrollPos === "top" ? "black" : "transparent"}, black 16px, black calc(100% - 16px), ${visionScrollPos === "bottom" ? "black" : "transparent"})`,
WebkitMaskImage: `linear-gradient(to bottom, ${visionScrollPos === "top" ? "black" : "transparent"}, black 16px, black calc(100% - 16px), ${visionScrollPos === "bottom" ? "black" : "transparent"})`,
}}
>
<CommandEmpty className="py-8 text-center">
<div className="flex flex-col items-center gap-2">
<Search className="size-8 text-muted-foreground" />
<p className="text-sm text-muted-foreground">No vision models found</p>
<p className="text-xs text-muted-foreground/60">Try a different search term</p>
</div>
</CommandEmpty>
{filteredVisionGlobal.length > 0 && (
<CommandGroup>
<div className="flex items-center gap-2 px-3 py-2 text-xs font-semibold text-muted-foreground tracking-wider">
Global Vision Models
</div>
{filteredVisionGlobal.map((config) => {
const isSelected = currentVisionConfig?.id === config.id;
const isAuto = "is_auto_mode" in config && config.is_auto_mode;
return (
<CommandItem
key={`vis-g-${config.id}`}
value={`vis-g-${config.id}`}
onSelect={() => handleSelectVision(config.id)}
className={cn(
"mx-2 rounded-lg mb-1 cursor-pointer group transition-all hover:bg-accent/50 dark:hover:bg-white/[0.06]",
isSelected && "bg-accent/80 dark:bg-white/[0.06]"
)}
>
<div className="flex items-center gap-3 min-w-0 flex-1">
<div className="shrink-0">
{getProviderIcon(config.provider, { isAutoMode: isAuto })}
</div>
<div className="min-w-0 flex-1">
<div className="flex items-center gap-2">
<span className="font-medium truncate">{config.name}</span>
{isAuto && (
<Badge
variant="secondary"
className="text-[9px] px-1 py-0 h-3.5 bg-violet-800 text-white dark:bg-violet-800 dark:text-white border-0"
>
Recommended
</Badge>
)}
{isSelected && <Check className="size-3.5 text-primary shrink-0" />}
</div>
<span className="text-xs text-muted-foreground truncate block">
{isAuto ? "Auto Mode" : config.model_name}
</span>
</div>
{onEditVision && !isAuto && (
<Button
variant="ghost"
size="icon"
className="size-7 shrink-0 rounded-md hover:bg-muted opacity-0 group-hover:opacity-100 transition-opacity"
onClick={(e) => {
e.stopPropagation();
setOpen(false);
onEditVision(config as VisionLLMConfig, true);
}}
>
<Edit3 className="size-3.5 text-muted-foreground" />
</Button>
)}
</div>
</CommandItem>
);
})}
</CommandGroup>
)}
{filteredVisionUser.length > 0 && (
<>
{filteredVisionGlobal.length > 0 && (
<CommandSeparator className="my-1 mx-4 bg-border/60" />
)}
<CommandGroup>
<div className="flex items-center gap-2 px-3 py-2 text-xs font-semibold text-muted-foreground tracking-wider">
Your Vision Models
</div>
{filteredVisionUser.map((config) => {
const isSelected = currentVisionConfig?.id === config.id;
return (
<CommandItem
key={`vis-u-${config.id}`}
value={`vis-u-${config.id}`}
onSelect={() => handleSelectVision(config.id)}
className={cn(
"mx-2 rounded-lg mb-1 cursor-pointer group transition-all hover:bg-accent/50 dark:hover:bg-white/[0.06]",
isSelected && "bg-accent/80 dark:bg-white/[0.06]"
)}
>
<div className="flex items-center gap-3 min-w-0 flex-1">
<div className="shrink-0">{getProviderIcon(config.provider)}</div>
<div className="min-w-0 flex-1">
<div className="flex items-center gap-2">
<span className="font-medium truncate">{config.name}</span>
{isSelected && (
<Check className="size-3.5 text-primary shrink-0" />
)}
</div>
<span className="text-xs text-muted-foreground truncate block">
{config.model_name}
</span>
</div>
{onEditVision && (
<Button
variant="ghost"
size="icon"
className="h-7 w-7 shrink-0 opacity-0 group-hover:opacity-100 transition-opacity"
onClick={(e) => {
e.stopPropagation();
setOpen(false);
onEditVision(config, false);
}}
>
<Edit3 className="size-3.5 text-muted-foreground" />
</Button>
)}
</div>
</CommandItem>
);
})}
</CommandGroup>
</>
)}
{onAddNewVision && (
<div className="p-2 bg-muted/20 dark:bg-neutral-900">
<Button
variant="ghost"
size="sm"
className="w-full justify-start gap-2 h-9 rounded-lg hover:bg-accent/50 dark:hover:bg-white/[0.06]"
onClick={() => {
setOpen(false);
onAddNewVision();
}}
>
<Plus className="size-4 text-primary" />
<span className="text-sm font-medium">Add Vision Model</span>
</Button>
</div>
)}
</CommandList>
</Command>
</TabsContent>
</Tabs>
</PopoverContent>
</Popover>

66
surfsense_web/components/settings/llm-role-manager.tsx
View file

@ -24,6 +24,10 @@ import {
llmPreferencesAtom,
newLLMConfigsAtom,
} from "@/atoms/new-llm-config/new-llm-config-query.atoms";
import {
globalVisionLLMConfigsAtom,
visionLLMConfigsAtom,
} from "@/atoms/vision-llm-config/vision-llm-config-query.atoms";
import { Alert, AlertDescription } from "@/components/ui/alert";
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
@ -77,8 +81,8 @@ const ROLE_DESCRIPTIONS = {
description: "Vision-capable model for screenshot analysis and context extraction",
color: "text-amber-600 dark:text-amber-400",
bgColor: "bg-amber-500/10",
prefKey: "vision_llm_id" as const,
configType: "llm" as const,
prefKey: "vision_llm_config_id" as const,
configType: "vision" as const,
},
};
@ -112,6 +116,18 @@ export function LLMRoleManager({ searchSpaceId }: LLMRoleManagerProps) {
error: globalImageConfigsError,
} = useAtomValue(globalImageGenConfigsAtom);
// Vision LLM configs
const {
data: userVisionConfigs = [],
isFetching: visionConfigsLoading,
error: visionConfigsError,
} = useAtomValue(visionLLMConfigsAtom);
const {
data: globalVisionConfigs = [],
isFetching: globalVisionConfigsLoading,
error: globalVisionConfigsError,
} = useAtomValue(globalVisionLLMConfigsAtom);
// Preferences
const {
data: preferences = {},
@ -125,7 +141,7 @@ export function LLMRoleManager({ searchSpaceId }: LLMRoleManagerProps) {
agent_llm_id: preferences.agent_llm_id ?? "",
document_summary_llm_id: preferences.document_summary_llm_id ?? "",
image_generation_config_id: preferences.image_generation_config_id ?? "",
vision_llm_id: preferences.vision_llm_id ?? "",
vision_llm_config_id: preferences.vision_llm_config_id ?? "",
}));
const [savingRole, setSavingRole] = useState<string | null>(null);
@ -137,14 +153,14 @@ export function LLMRoleManager({ searchSpaceId }: LLMRoleManagerProps) {
agent_llm_id: preferences.agent_llm_id ?? "",
document_summary_llm_id: preferences.document_summary_llm_id ?? "",
image_generation_config_id: preferences.image_generation_config_id ?? "",
vision_llm_id: preferences.vision_llm_id ?? "",
vision_llm_config_id: preferences.vision_llm_config_id ?? "",
});
}
}, [
preferences?.agent_llm_id,
preferences?.document_summary_llm_id,
preferences?.image_generation_config_id,
preferences?.vision_llm_id,
preferences?.vision_llm_config_id,
]);
const handleRoleAssignment = useCallback(
@ -181,6 +197,14 @@ export function LLMRoleManager({ searchSpaceId }: LLMRoleManagerProps) {
...(userImageConfigs ?? []).filter((config) => config.id && config.id.toString().trim() !== ""),
];
// Combine global and custom vision LLM configs
const allVisionConfigs = [
...globalVisionConfigs.map((config) => ({ ...config, is_global: true })),
...(userVisionConfigs ?? []).filter(
(config) => config.id && config.id.toString().trim() !== ""
),
];
const isAssignmentComplete =
allLLMConfigs.some((c) => c.id === assignments.agent_llm_id) &&
allLLMConfigs.some((c) => c.id === assignments.document_summary_llm_id) &&
@ -191,13 +215,17 @@ export function LLMRoleManager({ searchSpaceId }: LLMRoleManagerProps) {
preferencesLoading ||
globalConfigsLoading ||
imageConfigsLoading ||
globalImageConfigsLoading;
globalImageConfigsLoading ||
visionConfigsLoading ||
globalVisionConfigsLoading;
const hasError =
configsError ||
preferencesError ||
globalConfigsError ||
imageConfigsError ||
globalImageConfigsError;
globalImageConfigsError ||
visionConfigsError ||
globalVisionConfigsError;
const hasAnyConfigs = allLLMConfigs.length > 0 || allImageConfigs.length > 0;
return (
@ -291,15 +319,27 @@ export function LLMRoleManager({ searchSpaceId }: LLMRoleManagerProps) {
<div className="grid gap-4 grid-cols-1 lg:grid-cols-2">
{Object.entries(ROLE_DESCRIPTIONS).map(([key, role]) => {
const IconComponent = role.icon;
const isImageRole = role.configType === "image";
const currentAssignment = assignments[role.prefKey as keyof typeof assignments];
// Pick the right config lists based on role type
const roleGlobalConfigs = isImageRole ? globalImageConfigs : globalConfigs;
const roleUserConfigs = isImageRole
? (userImageConfigs ?? []).filter((c) => c.id && c.id.toString().trim() !== "")
: newLLMConfigs.filter((c) => c.id && c.id.toString().trim() !== "");
const roleAllConfigs = isImageRole ? allImageConfigs : allLLMConfigs;
const roleGlobalConfigs =
role.configType === "image"
? globalImageConfigs
: role.configType === "vision"
? globalVisionConfigs
: globalConfigs;
const roleUserConfigs =
role.configType === "image"
? (userImageConfigs ?? []).filter((c) => c.id && c.id.toString().trim() !== "")
: role.configType === "vision"
? (userVisionConfigs ?? []).filter((c) => c.id && c.id.toString().trim() !== "")
: newLLMConfigs.filter((c) => c.id && c.id.toString().trim() !== "");
const roleAllConfigs =
role.configType === "image"
? allImageConfigs
: role.configType === "vision"
? allVisionConfigs
: allLLMConfigs;
const assignedConfig = roleAllConfigs.find((config) => config.id === currentAssignment);
const isAssigned = !!assignedConfig;

9
surfsense_web/components/settings/search-space-settings-dialog.tsx
View file

@ -1,7 +1,7 @@
"use client";
import { useAtom } from "jotai";
import { Bot, Brain, FileText, Globe, ImageIcon, MessageSquare, Shield } from "lucide-react";
import { Bot, Brain, Eye, FileText, Globe, ImageIcon, MessageSquare, Shield } from "lucide-react";
import { useTranslations } from "next-intl";
import type React from "react";
import { searchSpaceSettingsDialogAtom } from "@/atoms/settings/settings-dialog.atoms";
@ -13,6 +13,7 @@ import { ModelConfigManager } from "@/components/settings/model-config-manager";
import { PromptConfigManager } from "@/components/settings/prompt-config-manager";
import { RolesManager } from "@/components/settings/roles-manager";
import { SettingsDialog } from "@/components/settings/settings-dialog";
import { VisionModelManager } from "@/components/settings/vision-model-manager";
interface SearchSpaceSettingsDialogProps {
searchSpaceId: number;
@ -31,6 +32,11 @@ export function SearchSpaceSettingsDialog({ searchSpaceId }: SearchSpaceSettings
label: t("nav_image_models"),
icon: <ImageIcon className="h-4 w-4" />,
},
{
value: "vision-models",
label: t("nav_vision_models"),
icon: <Eye className="h-4 w-4" />,
},
{ value: "team-roles", label: t("nav_team_roles"), icon: <Shield className="h-4 w-4" /> },
{
value: "prompts",
@ -45,6 +51,7 @@ export function SearchSpaceSettingsDialog({ searchSpaceId }: SearchSpaceSettings
models: <ModelConfigManager searchSpaceId={searchSpaceId} />,
roles: <LLMRoleManager searchSpaceId={searchSpaceId} />,
"image-models": <ImageModelManager searchSpaceId={searchSpaceId} />,
"vision-models": <VisionModelManager searchSpaceId={searchSpaceId} />,
"team-roles": <RolesManager searchSpaceId={searchSpaceId} />,
prompts: <PromptConfigManager searchSpaceId={searchSpaceId} />,
"public-links": <PublicChatSnapshotsManager searchSpaceId={searchSpaceId} />,

401
surfsense_web/components/settings/vision-model-manager.tsx Normal file
View file

@ -0,0 +1,401 @@
"use client";
import { useAtomValue } from "jotai";
import { AlertCircle, Dot, Edit3, Info, RefreshCw, Trash2 } from "lucide-react";
import { useMemo, useState } from "react";
import { membersAtom, myAccessAtom } from "@/atoms/members/members-query.atoms";
import { deleteVisionLLMConfigMutationAtom } from "@/atoms/vision-llm-config/vision-llm-config-mutation.atoms";
import {
globalVisionLLMConfigsAtom,
visionLLMConfigsAtom,
} from "@/atoms/vision-llm-config/vision-llm-config-query.atoms";
import { VisionConfigDialog } from "@/components/shared/vision-config-dialog";
import { Alert, AlertDescription } from "@/components/ui/alert";
import {
AlertDialog,
AlertDialogAction,
AlertDialogCancel,
AlertDialogContent,
AlertDialogDescription,
AlertDialogFooter,
AlertDialogHeader,
AlertDialogTitle,
} from "@/components/ui/alert-dialog";
import { Avatar, AvatarFallback, AvatarImage } from "@/components/ui/avatar";
import { Button } from "@/components/ui/button";
import { Card, CardContent } from "@/components/ui/card";
import { Skeleton } from "@/components/ui/skeleton";
import { Spinner } from "@/components/ui/spinner";
import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip";
import type { VisionLLMConfig } from "@/contracts/types/new-llm-config.types";
import { useMediaQuery } from "@/hooks/use-media-query";
import { getProviderIcon } from "@/lib/provider-icons";
import { cn } from "@/lib/utils";
interface VisionModelManagerProps {
searchSpaceId: number;
}
function getInitials(name: string): string {
const parts = name.trim().split(/\s+/);
if (parts.length >= 2) {
return (parts[0][0] + parts[1][0]).toUpperCase();
}
return name.slice(0, 2).toUpperCase();
}
export function VisionModelManager({ searchSpaceId }: VisionModelManagerProps) {
const isDesktop = useMediaQuery("(min-width: 768px)");
const {
mutateAsync: deleteConfig,
isPending: isDeleting,
error: deleteError,
} = useAtomValue(deleteVisionLLMConfigMutationAtom);
const {
data: userConfigs,
isFetching: configsLoading,
error: fetchError,
refetch: refreshConfigs,
} = useAtomValue(visionLLMConfigsAtom);
const { data: globalConfigs = [], isFetching: globalLoading } = useAtomValue(
globalVisionLLMConfigsAtom
);
const { data: members } = useAtomValue(membersAtom);
const memberMap = useMemo(() => {
const map = new Map<string, { name: string; email?: string; avatarUrl?: string }>();
if (members) {
for (const m of members) {
map.set(m.user_id, {
name: m.user_display_name || m.user_email || "Unknown",
email: m.user_email || undefined,
avatarUrl: m.user_avatar_url || undefined,
});
}
}
return map;
}, [members]);
const { data: access } = useAtomValue(myAccessAtom);
const canCreate = useMemo(() => {
if (!access) return false;
if (access.is_owner) return true;
return access.permissions?.includes("vision_configs:create") ?? false;
}, [access]);
const canDelete = useMemo(() => {
if (!access) return false;
if (access.is_owner) return true;
return access.permissions?.includes("vision_configs:delete") ?? false;
}, [access]);
const canUpdate = canCreate;
const isReadOnly = !canCreate && !canDelete;
const [isDialogOpen, setIsDialogOpen] = useState(false);
const [editingConfig, setEditingConfig] = useState<VisionLLMConfig | null>(null);
const [configToDelete, setConfigToDelete] = useState<VisionLLMConfig | null>(null);
const isLoading = configsLoading || globalLoading;
const errors = [deleteError, fetchError].filter(Boolean) as Error[];
const openEditDialog = (config: VisionLLMConfig) => {
setEditingConfig(config);
setIsDialogOpen(true);
};
const openNewDialog = () => {
setEditingConfig(null);
setIsDialogOpen(true);
};
const handleDelete = async () => {
if (!configToDelete) return;
try {
await deleteConfig({ id: configToDelete.id, name: configToDelete.name });
setConfigToDelete(null);
} catch {
// Error handled by mutation
}
};
return (
<div className="space-y-4 md:space-y-6">
<div className="flex flex-col space-y-4 sm:flex-row sm:items-center sm:justify-between sm:space-y-0">
<Button
variant="secondary"
size="sm"
onClick={() => refreshConfigs()}
disabled={isLoading}
className="gap-2"
>
<RefreshCw className={cn("h-3.5 w-3.5", configsLoading && "animate-spin")} />
Refresh
</Button>
{canCreate && (
<Button
variant="outline"
onClick={openNewDialog}
className="gap-2 bg-white text-black hover:bg-neutral-100 dark:bg-white dark:text-black dark:hover:bg-neutral-200"
>
Add Vision Model
</Button>
)}
</div>
{errors.map((err) => (
<div key={err?.message}>
<Alert variant="destructive" className="py-3">
<AlertCircle className="h-3 w-3 md:h-4 md:w-4 shrink-0" />
<AlertDescription className="text-xs md:text-sm">{err?.message}</AlertDescription>
</Alert>
</div>
))}
{access && !isLoading && isReadOnly && (
<div>
<Alert className="bg-muted/50 py-3 md:py-4">
<Info className="h-3 w-3 md:h-4 md:w-4 shrink-0" />
<AlertDescription className="text-xs md:text-sm">
You have <span className="font-medium">read-only</span> access to vision model
configurations. Contact a space owner to request additional permissions.
</AlertDescription>
</Alert>
</div>
)}
{access && !isLoading && !isReadOnly && (!canCreate || !canDelete) && (
<div>
<Alert className="bg-muted/50 py-3 md:py-4">
<Info className="h-3 w-3 md:h-4 md:w-4 shrink-0" />
<AlertDescription className="text-xs md:text-sm">
You can{" "}
{[canCreate && "create and edit", canDelete && "delete"]
.filter(Boolean)
.join(" and ")}{" "}
vision model configurations
{!canDelete && ", but cannot delete them"}.
</AlertDescription>
</Alert>
</div>
)}
{globalConfigs.filter((g) => !("is_auto_mode" in g && g.is_auto_mode)).length > 0 && (
<Alert className="bg-muted/50 py-3">
<Info className="h-3 w-3 md:h-4 md:w-4 shrink-0" />
<AlertDescription className="text-xs md:text-sm">
<p>
<span className="font-medium">
{globalConfigs.filter((g) => !("is_auto_mode" in g && g.is_auto_mode)).length}{" "}
global vision{" "}
{globalConfigs.filter((g) => !("is_auto_mode" in g && g.is_auto_mode)).length === 1
? "model"
: "models"}
</span>{" "}
available from your administrator. Use the model selector to view and select them.
</p>
</AlertDescription>
</Alert>
)}
{isLoading && (
<div className="space-y-4 md:space-y-6">
<div className="space-y-4">
<div className="flex items-center justify-between">
<Skeleton className="h-6 md:h-7 w-40 md:w-48" />
<Skeleton className="h-8 md:h-9 w-32 md:w-36 rounded-md" />
</div>
<div className="grid gap-3 grid-cols-1 sm:grid-cols-2 xl:grid-cols-3">
{["skeleton-a", "skeleton-b", "skeleton-c"].map((key) => (
<Card key={key} className="border-border/60">
<CardContent className="p-4 flex flex-col gap-3">
<div className="flex items-start justify-between gap-2">
<div className="space-y-1.5 flex-1 min-w-0">
<Skeleton className="h-4 w-28 md:w-32" />
<Skeleton className="h-3 w-40 md:w-48" />
</div>
</div>
<div className="flex items-center gap-2">
<Skeleton className="h-5 w-16 rounded-full" />
<Skeleton className="h-5 w-24 rounded-md" />
</div>
<div className="flex items-center gap-2 pt-2 border-t border-border/40">
<Skeleton className="h-3 w-20" />
<Skeleton className="h-4 w-4 rounded-full" />
<Skeleton className="h-3 w-16" />
</div>
</CardContent>
</Card>
))}
</div>
</div>
</div>
)}
{!isLoading && (
<div className="space-y-4 md:space-y-6">
{(userConfigs?.length ?? 0) === 0 ? (
<Card className="border-0 bg-transparent shadow-none">
<CardContent className="flex flex-col items-center justify-center py-10 md:py-16 text-center">
<h3 className="text-sm md:text-base font-semibold mb-2">No Vision Models Yet</h3>
<p className="text-[11px] md:text-xs text-muted-foreground max-w-sm mb-4">
{canCreate
? "Add your own vision-capable model (GPT-4o, Claude, Gemini, etc.)"
: "No vision models have been added to this space yet. Contact a space owner to add one."}
</p>
</CardContent>
</Card>
) : (
<div className="grid gap-3 grid-cols-1 sm:grid-cols-2 xl:grid-cols-3">
{userConfigs?.map((config) => {
const member = config.user_id ? memberMap.get(config.user_id) : null;
return (
<div key={config.id}>
<Card className="group relative overflow-hidden transition-all duration-200 border-border/60 hover:shadow-md h-full">
<CardContent className="p-4 flex flex-col gap-3 h-full">
<div className="flex items-start justify-between gap-2">
<div className="min-w-0 flex-1">
<h4 className="text-sm font-semibold tracking-tight truncate">
{config.name}
</h4>
{config.description && (
<p className="text-[11px] text-muted-foreground/70 truncate mt-0.5">
{config.description}
</p>
)}
</div>
{(canUpdate || canDelete) && (
<div className="flex items-center gap-0.5 shrink-0 sm:opacity-0 sm:group-hover:opacity-100 transition-opacity duration-150">
{canUpdate && (
<TooltipProvider>
<Tooltip open={isDesktop ? undefined : false}>
<TooltipTrigger asChild>
<Button
variant="ghost"
size="icon"
onClick={() => openEditDialog(config)}
className="h-7 w-7 text-muted-foreground hover:text-foreground"
>
<Edit3 className="h-3 w-3" />
</Button>
</TooltipTrigger>
<TooltipContent>Edit</TooltipContent>
</Tooltip>
</TooltipProvider>
)}
{canDelete && (
<TooltipProvider>
<Tooltip open={isDesktop ? undefined : false}>
<TooltipTrigger asChild>
<Button
variant="ghost"
size="icon"
onClick={() => setConfigToDelete(config)}
className="h-7 w-7 text-muted-foreground hover:text-destructive"
>
<Trash2 className="h-3 w-3" />
</Button>
</TooltipTrigger>
<TooltipContent>Delete</TooltipContent>
</Tooltip>
</TooltipProvider>
)}
</div>
)}
</div>
<div className="flex items-center gap-2 flex-wrap">
{getProviderIcon(config.provider, {
className: "size-3.5 shrink-0",
})}
<code className="text-[11px] font-mono text-muted-foreground bg-muted/60 px-2 py-0.5 rounded-md truncate max-w-[160px]">
{config.model_name}
</code>
</div>
<div className="flex items-center gap-2 pt-2 border-t border-border/40 mt-auto">
<span className="text-[11px] text-muted-foreground/60">
{new Date(config.created_at).toLocaleDateString(undefined, {
year: "numeric",
month: "short",
day: "numeric",
})}
</span>
{member && (
<>
<Dot className="h-4 w-4 text-muted-foreground/30" />
<TooltipProvider>
<Tooltip open={isDesktop ? undefined : false}>
<TooltipTrigger asChild>
<div className="flex items-center gap-1.5 cursor-default">
<Avatar className="size-4.5 shrink-0">
{member.avatarUrl && (
<AvatarImage src={member.avatarUrl} alt={member.name} />
)}
<AvatarFallback className="text-[9px]">
{getInitials(member.name)}
</AvatarFallback>
</Avatar>
<span className="text-[11px] text-muted-foreground/60 truncate max-w-[120px]">
{member.name}
</span>
</div>
</TooltipTrigger>
<TooltipContent side="bottom">
{member.email || member.name}
</TooltipContent>
</Tooltip>
</TooltipProvider>
</>
)}
</div>
</CardContent>
</Card>
</div>
);
})}
</div>
)}
</div>
)}
<VisionConfigDialog
open={isDialogOpen}
onOpenChange={(open) => {
setIsDialogOpen(open);
if (!open) setEditingConfig(null);
}}
config={editingConfig}
isGlobal={false}
searchSpaceId={searchSpaceId}
mode={editingConfig ? "edit" : "create"}
/>
<AlertDialog
open={!!configToDelete}
onOpenChange={(open) => !open && setConfigToDelete(null)}
>
<AlertDialogContent className="select-none">
<AlertDialogHeader>
<AlertDialogTitle>Delete Vision Model</AlertDialogTitle>
<AlertDialogDescription>
Are you sure you want to delete{" "}
<span className="font-semibold text-foreground">{configToDelete?.name}</span>?
</AlertDialogDescription>
</AlertDialogHeader>
<AlertDialogFooter>
<AlertDialogCancel disabled={isDeleting}>Cancel</AlertDialogCancel>
<AlertDialogAction
onClick={handleDelete}
disabled={isDeleting}
className="relative bg-destructive text-destructive-foreground hover:bg-destructive/90"
>
<span className={isDeleting ? "opacity-0" : ""}>Delete</span>
{isDeleting && <Spinner size="sm" className="absolute" />}
</AlertDialogAction>
</AlertDialogFooter>
</AlertDialogContent>
</AlertDialog>
</div>
);
}

381
surfsense_web/components/shared/vision-config-dialog.tsx Normal file
View file

@ -0,0 +1,381 @@
"use client";
import { useAtomValue } from "jotai";
import { AlertCircle } from "lucide-react";
import { useCallback, useEffect, useRef, useState } from "react";
import { toast } from "sonner";
import { updateLLMPreferencesMutationAtom } from "@/atoms/new-llm-config/new-llm-config-mutation.atoms";
import {
createVisionLLMConfigMutationAtom,
updateVisionLLMConfigMutationAtom,
} from "@/atoms/vision-llm-config/vision-llm-config-mutation.atoms";
import { Alert, AlertDescription } from "@/components/ui/alert";
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
import { Dialog, DialogContent, DialogTitle } from "@/components/ui/dialog";
import { Input } from "@/components/ui/input";
import { Label } from "@/components/ui/label";
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from "@/components/ui/select";
import { Separator } from "@/components/ui/separator";
import { Spinner } from "@/components/ui/spinner";
import { VISION_PROVIDERS } from "@/contracts/enums/vision-providers";
import type {
GlobalVisionLLMConfig,
VisionLLMConfig,
VisionProvider,
} from "@/contracts/types/new-llm-config.types";
interface VisionConfigDialogProps {
open: boolean;
onOpenChange: (open: boolean) => void;
config: VisionLLMConfig | GlobalVisionLLMConfig | null;
isGlobal: boolean;
searchSpaceId: number;
mode: "create" | "edit" | "view";
}
const INITIAL_FORM = {
name: "",
description: "",
provider: "",
model_name: "",
api_key: "",
api_base: "",
api_version: "",
};
export function VisionConfigDialog({
open,
onOpenChange,
config,
isGlobal,
searchSpaceId,
mode,
}: VisionConfigDialogProps) {
const [isSubmitting, setIsSubmitting] = useState(false);
const [formData, setFormData] = useState(INITIAL_FORM);
const [scrollPos, setScrollPos] = useState<"top" | "middle" | "bottom">("top");
const scrollRef = useRef<HTMLDivElement>(null);
useEffect(() => {
if (open) {
if (mode === "edit" && config && !isGlobal) {
setFormData({
name: config.name || "",
description: config.description || "",
provider: config.provider || "",
model_name: config.model_name || "",
api_key: (config as VisionLLMConfig).api_key || "",
api_base: config.api_base || "",
api_version: (config as VisionLLMConfig).api_version || "",
});
} else if (mode === "create") {
setFormData(INITIAL_FORM);
}
setScrollPos("top");
}
}, [open, mode, config, isGlobal]);
const { mutateAsync: createConfig } = useAtomValue(createVisionLLMConfigMutationAtom);
const { mutateAsync: updateConfig } = useAtomValue(updateVisionLLMConfigMutationAtom);
const { mutateAsync: updatePreferences } = useAtomValue(updateLLMPreferencesMutationAtom);
const handleScroll = useCallback((e: React.UIEvent<HTMLDivElement>) => {
const el = e.currentTarget;
const atTop = el.scrollTop <= 2;
const atBottom = el.scrollHeight - el.scrollTop - el.clientHeight <= 2;
setScrollPos(atTop ? "top" : atBottom ? "bottom" : "middle");
}, []);
const getTitle = () => {
if (mode === "create") return "Add Vision Model";
if (isGlobal) return "View Global Vision Model";
return "Edit Vision Model";
};
const getSubtitle = () => {
if (mode === "create") return "Set up a new vision-capable LLM provider";
if (isGlobal) return "Read-only global configuration";
return "Update your vision model settings";
};
const handleSubmit = useCallback(async () => {
setIsSubmitting(true);
try {
if (mode === "create") {
const result = await createConfig({
name: formData.name,
provider: formData.provider as VisionProvider,
model_name: formData.model_name,
api_key: formData.api_key,
api_base: formData.api_base || undefined,
api_version: formData.api_version || undefined,
description: formData.description || undefined,
search_space_id: searchSpaceId,
});
if (result?.id) {
await updatePreferences({
search_space_id: searchSpaceId,
data: { vision_llm_config_id: result.id },
});
}
onOpenChange(false);
} else if (!isGlobal && config) {
await updateConfig({
id: config.id,
data: {
name: formData.name,
description: formData.description || undefined,
provider: formData.provider as VisionProvider,
model_name: formData.model_name,
api_key: formData.api_key,
api_base: formData.api_base || undefined,
api_version: formData.api_version || undefined,
},
});
onOpenChange(false);
}
} catch (error) {
console.error("Failed to save vision config:", error);
toast.error("Failed to save vision model");
} finally {
setIsSubmitting(false);
}
}, [
mode,
isGlobal,
config,
formData,
searchSpaceId,
createConfig,
updateConfig,
updatePreferences,
onOpenChange,
]);
const handleUseGlobalConfig = useCallback(async () => {
if (!config || !isGlobal) return;
setIsSubmitting(true);
try {
await updatePreferences({
search_space_id: searchSpaceId,
data: { vision_llm_config_id: config.id },
});
toast.success(`Now using ${config.name}`);
onOpenChange(false);
} catch (error) {
console.error("Failed to set vision model:", error);
toast.error("Failed to set vision model");
} finally {
setIsSubmitting(false);
}
}, [config, isGlobal, searchSpaceId, updatePreferences, onOpenChange]);
const isFormValid = formData.name && formData.provider && formData.model_name && formData.api_key;
const selectedProvider = VISION_PROVIDERS.find((p) => p.value === formData.provider);
return (
<Dialog open={open} onOpenChange={onOpenChange}>
<DialogContent
className="max-w-lg h-[85vh] flex flex-col p-0 gap-0 overflow-hidden"
onOpenAutoFocus={(e) => e.preventDefault()}
>
<DialogTitle className="sr-only">{getTitle()}</DialogTitle>
<div className="flex items-start justify-between px-6 pt-6 pb-4 pr-14">
<div className="space-y-1">
<div className="flex items-center gap-2">
<h2 className="text-lg font-semibold tracking-tight">{getTitle()}</h2>
{isGlobal && mode !== "create" && (
<Badge variant="secondary" className="text-[10px]">
Global
</Badge>
)}
</div>
<p className="text-sm text-muted-foreground">{getSubtitle()}</p>
{config && mode !== "create" && (
<p className="text-xs font-mono text-muted-foreground/70">{config.model_name}</p>
)}
</div>
</div>
<div
ref={scrollRef}
onScroll={handleScroll}
className="flex-1 overflow-y-auto px-6 py-5"
style={{
maskImage: `linear-gradient(to bottom, ${scrollPos === "top" ? "black" : "transparent"}, black 16px, black calc(100% - 16px), ${scrollPos === "bottom" ? "black" : "transparent"})`,
WebkitMaskImage: `linear-gradient(to bottom, ${scrollPos === "top" ? "black" : "transparent"}, black 16px, black calc(100% - 16px), ${scrollPos === "bottom" ? "black" : "transparent"})`,
}}
>
{isGlobal && config && (
<>
<Alert className="mb-5 border-amber-500/30 bg-amber-500/5">
<AlertCircle className="size-4 text-amber-500" />
<AlertDescription className="text-sm text-amber-700 dark:text-amber-400">
Global configurations are read-only. To customize, create a new model.
</AlertDescription>
</Alert>
<div className="space-y-4">
<div className="grid gap-4 sm:grid-cols-2">
<div className="space-y-1.5">
<div className="text-xs font-medium text-muted-foreground uppercase tracking-wider">
Name
</div>
<p className="text-sm font-medium">{config.name}</p>
</div>
{config.description && (
<div className="space-y-1.5">
<div className="text-xs font-medium text-muted-foreground uppercase tracking-wider">
Description
</div>
<p className="text-sm text-muted-foreground">{config.description}</p>
</div>
)}
</div>
<Separator />
<div className="grid gap-4 sm:grid-cols-2">
<div className="space-y-1.5">
<div className="text-xs font-medium text-muted-foreground uppercase tracking-wider">
Provider
</div>
<p className="text-sm font-medium">{config.provider}</p>
</div>
<div className="space-y-1.5">
<div className="text-xs font-medium text-muted-foreground uppercase tracking-wider">
Model
</div>
<p className="text-sm font-medium font-mono">{config.model_name}</p>
</div>
</div>
</div>
</>
)}
{(mode === "create" || (mode === "edit" && !isGlobal)) && (
<div className="space-y-4">
<div className="space-y-2">
<Label className="text-sm font-medium">Name *</Label>
<Input
placeholder="e.g., My GPT-4o Vision"
value={formData.name}
onChange={(e) => setFormData((p) => ({ ...p, name: e.target.value }))}
/>
</div>
<div className="space-y-2">
<Label className="text-sm font-medium">Description</Label>
<Input
placeholder="Optional description"
value={formData.description}
onChange={(e) => setFormData((p) => ({ ...p, description: e.target.value }))}
/>
</div>
<Separator />
<div className="space-y-2">
<Label className="text-sm font-medium">Provider *</Label>
<Select
value={formData.provider}
onValueChange={(val) =>
setFormData((p) => ({ ...p, provider: val, model_name: "" }))
}
>
<SelectTrigger>
<SelectValue placeholder="Select a provider" />
</SelectTrigger>
<SelectContent>
{VISION_PROVIDERS.map((p) => (
<SelectItem key={p.value} value={p.value} description={p.example}>
{p.label}
</SelectItem>
))}
</SelectContent>
</Select>
</div>
<div className="space-y-2">
<Label className="text-sm font-medium">Model Name *</Label>
<Input
placeholder={selectedProvider?.example?.split(",")[0]?.trim() || "e.g., gpt-4o"}
value={formData.model_name}
onChange={(e) => setFormData((p) => ({ ...p, model_name: e.target.value }))}
/>
</div>
<div className="space-y-2">
<Label className="text-sm font-medium">API Key *</Label>
<Input
type="password"
placeholder="sk-..."
value={formData.api_key}
onChange={(e) => setFormData((p) => ({ ...p, api_key: e.target.value }))}
/>
</div>
<div className="space-y-2">
<Label className="text-sm font-medium">API Base URL</Label>
<Input
placeholder={selectedProvider?.apiBase || "Optional"}
value={formData.api_base}
onChange={(e) => setFormData((p) => ({ ...p, api_base: e.target.value }))}
/>
</div>
{formData.provider === "AZURE_OPENAI" && (
<div className="space-y-2">
<Label className="text-sm font-medium">API Version (Azure)</Label>
<Input
placeholder="2024-02-15-preview"
value={formData.api_version}
onChange={(e) => setFormData((p) => ({ ...p, api_version: e.target.value }))}
/>
</div>
)}
</div>
)}
</div>
<div className="shrink-0 px-6 py-4 flex items-center justify-end gap-3">
<Button
type="button"
variant="secondary"
onClick={() => onOpenChange(false)}
disabled={isSubmitting}
className="text-sm h-9"
>
Cancel
</Button>
{mode === "create" || (mode === "edit" && !isGlobal) ? (
<Button
onClick={handleSubmit}
disabled={isSubmitting || !isFormValid}
className="relative text-sm h-9 min-w-[120px]"
>
<span className={isSubmitting ? "opacity-0" : ""}>
{mode === "edit" ? "Save Changes" : "Add Model"}
</span>
{isSubmitting && <Spinner size="sm" className="absolute" />}
</Button>
) : isGlobal && config ? (
<Button
className="relative text-sm h-9"
onClick={handleUseGlobalConfig}
disabled={isSubmitting}
>
<span className={isSubmitting ? "opacity-0" : ""}>Use This Model</span>
{isSubmitting && <Spinner size="sm" className="absolute" />}
</Button>
) : null}
</div>
</DialogContent>
</Dialog>
);
}

102
surfsense_web/contracts/enums/vision-providers.ts Normal file
View file

@ -0,0 +1,102 @@
export interface VisionProviderInfo {
value: string;
label: string;
example: string;
description: string;
apiBase?: string;
}
export const VISION_PROVIDERS: VisionProviderInfo[] = [
{
value: "OPENAI",
label: "OpenAI",
example: "gpt-4o, gpt-4o-mini",
description: "GPT-4o vision models",
},
{
value: "ANTHROPIC",
label: "Anthropic",
example: "claude-sonnet-4-20250514",
description: "Claude vision models",
},
{
value: "GOOGLE",
label: "Google AI Studio",
example: "gemini-2.5-flash, gemini-2.0-flash",
description: "Gemini vision models",
},
{
value: "AZURE_OPENAI",
label: "Azure OpenAI",
example: "azure/gpt-4o",
description: "OpenAI vision models on Azure",
},
{
value: "VERTEX_AI",
label: "Google Vertex AI",
example: "vertex_ai/gemini-2.5-flash",
description: "Gemini vision models on Vertex AI",
},
{
value: "BEDROCK",
label: "AWS Bedrock",
example: "bedrock/anthropic.claude-sonnet-4-20250514-v1:0",
description: "Vision models on AWS Bedrock",
},
{
value: "XAI",
label: "xAI",
example: "grok-2-vision",
description: "Grok vision models",
},
{
value: "OPENROUTER",
label: "OpenRouter",
example: "openrouter/openai/gpt-4o",
description: "Vision models via OpenRouter",
},
{
value: "OLLAMA",
label: "Ollama",
example: "llava, bakllava",
description: "Local vision models via Ollama",
apiBase: "http://localhost:11434",
},
{
value: "GROQ",
label: "Groq",
example: "llama-4-scout-17b-16e-instruct",
description: "Vision models on Groq",
},
{
value: "TOGETHER_AI",
label: "Together AI",
example: "meta-llama/Llama-4-Scout-17B-16E-Instruct",
description: "Vision models on Together AI",
},
{
value: "FIREWORKS_AI",
label: "Fireworks AI",
example: "fireworks_ai/phi-3-vision-128k-instruct",
description: "Vision models on Fireworks AI",
},
{
value: "DEEPSEEK",
label: "DeepSeek",
example: "deepseek-chat",
description: "DeepSeek vision models",
apiBase: "https://api.deepseek.com",
},
{
value: "MISTRAL",
label: "Mistral",
example: "pixtral-large-latest",
description: "Pixtral vision models",
},
{
value: "CUSTOM",
label: "Custom Provider",
example: "custom/my-vision-model",
description: "Custom OpenAI-compatible vision endpoint",
},
];

99
surfsense_web/contracts/types/new-llm-config.types.ts
View file

@ -252,23 +252,99 @@ export const globalImageGenConfig = z.object({
export const getGlobalImageGenConfigsResponse = z.array(globalImageGenConfig);
// =============================================================================
// Vision LLM Config (separate table for vision-capable models)
// =============================================================================
export const visionProviderEnum = z.enum([
"OPENAI",
"ANTHROPIC",
"GOOGLE",
"AZURE_OPENAI",
"VERTEX_AI",
"BEDROCK",
"XAI",
"OPENROUTER",
"OLLAMA",
"GROQ",
"TOGETHER_AI",
"FIREWORKS_AI",
"DEEPSEEK",
"MISTRAL",
"CUSTOM",
]);
export type VisionProvider = z.infer<typeof visionProviderEnum>;
export const visionLLMConfig = z.object({
id: z.number(),
name: z.string().max(100),
description: z.string().max(500).nullable().optional(),
provider: visionProviderEnum,
custom_provider: z.string().max(100).nullable().optional(),
model_name: z.string().max(100),
api_key: z.string(),
api_base: z.string().max(500).nullable().optional(),
api_version: z.string().max(50).nullable().optional(),
litellm_params: z.record(z.string(), z.any()).nullable().optional(),
created_at: z.string(),
search_space_id: z.number(),
user_id: z.string(),
});
export const createVisionLLMConfigRequest = visionLLMConfig.omit({
id: true,
created_at: true,
user_id: true,
});
export const createVisionLLMConfigResponse = visionLLMConfig;
export const getVisionLLMConfigsResponse = z.array(visionLLMConfig);
export const updateVisionLLMConfigRequest = z.object({
id: z.number(),
data: visionLLMConfig
.omit({ id: true, created_at: true, search_space_id: true, user_id: true })
.partial(),
});
export const updateVisionLLMConfigResponse = visionLLMConfig;
export const deleteVisionLLMConfigResponse = z.object({
message: z.string(),
id: z.number(),
});
export const globalVisionLLMConfig = z.object({
id: z.number(),
name: z.string(),
description: z.string().nullable().optional(),
provider: z.string(),
custom_provider: z.string().nullable().optional(),
model_name: z.string(),
api_base: z.string().nullable().optional(),
api_version: z.string().nullable().optional(),
litellm_params: z.record(z.string(), z.any()).nullable().optional(),
is_global: z.literal(true),
is_auto_mode: z.boolean().optional().default(false),
});
export const getGlobalVisionLLMConfigsResponse = z.array(globalVisionLLMConfig);
// =============================================================================
// LLM Preferences (Role Assignments)
// =============================================================================
/**
* LLM Preferences schemas - for role assignments
* image_generation uses image_generation_config_id (not llm_id)
*/
export const llmPreferences = z.object({
agent_llm_id: z.union([z.number(), z.null()]).optional(),
document_summary_llm_id: z.union([z.number(), z.null()]).optional(),
image_generation_config_id: z.union([z.number(), z.null()]).optional(),
vision_llm_id: z.union([z.number(), z.null()]).optional(),
vision_llm_config_id: z.union([z.number(), z.null()]).optional(),
agent_llm: z.union([z.record(z.string(), z.unknown()), z.null()]).optional(),
document_summary_llm: z.union([z.record(z.string(), z.unknown()), z.null()]).optional(),
image_generation_config: z.union([z.record(z.string(), z.unknown()), z.null()]).optional(),
vision_llm: z.union([z.record(z.string(), z.unknown()), z.null()]).optional(),
vision_llm_config: z.union([z.record(z.string(), z.unknown()), z.null()]).optional(),
});
/**
@ -289,7 +365,7 @@ export const updateLLMPreferencesRequest = z.object({
agent_llm_id: true,
document_summary_llm_id: true,
image_generation_config_id: true,
vision_llm_id: true,
vision_llm_config_id: true,
}),
});
@ -341,6 +417,15 @@ export type UpdateImageGenConfigResponse = z.infer<typeof updateImageGenConfigRe
export type DeleteImageGenConfigResponse = z.infer<typeof deleteImageGenConfigResponse>;
export type GlobalImageGenConfig = z.infer<typeof globalImageGenConfig>;
export type GetGlobalImageGenConfigsResponse = z.infer<typeof getGlobalImageGenConfigsResponse>;
export type VisionLLMConfig = z.infer<typeof visionLLMConfig>;
export type CreateVisionLLMConfigRequest = z.infer<typeof createVisionLLMConfigRequest>;
export type CreateVisionLLMConfigResponse = z.infer<typeof createVisionLLMConfigResponse>;
export type GetVisionLLMConfigsResponse = z.infer<typeof getVisionLLMConfigsResponse>;
export type UpdateVisionLLMConfigRequest = z.infer<typeof updateVisionLLMConfigRequest>;
export type UpdateVisionLLMConfigResponse = z.infer<typeof updateVisionLLMConfigResponse>;
export type DeleteVisionLLMConfigResponse = z.infer<typeof deleteVisionLLMConfigResponse>;
export type GlobalVisionLLMConfig = z.infer<typeof globalVisionLLMConfig>;
export type GetGlobalVisionLLMConfigsResponse = z.infer<typeof getGlobalVisionLLMConfigsResponse>;
export type LLMPreferences = z.infer<typeof llmPreferences>;
export type GetLLMPreferencesRequest = z.infer<typeof getLLMPreferencesRequest>;
export type GetLLMPreferencesResponse = z.infer<typeof getLLMPreferencesResponse>;

58
surfsense_web/lib/apis/vision-llm-config-api.service.ts Normal file
View file

@ -0,0 +1,58 @@
import {
type CreateVisionLLMConfigRequest,
createVisionLLMConfigRequest,
createVisionLLMConfigResponse,
deleteVisionLLMConfigResponse,
getGlobalVisionLLMConfigsResponse,
getVisionLLMConfigsResponse,
type UpdateVisionLLMConfigRequest,
updateVisionLLMConfigRequest,
updateVisionLLMConfigResponse,
} from "@/contracts/types/new-llm-config.types";
import { ValidationError } from "../error";
import { baseApiService } from "./base-api.service";
class VisionLLMConfigApiService {
getGlobalConfigs = async () => {
return baseApiService.get(
`/api/v1/global-vision-llm-configs`,
getGlobalVisionLLMConfigsResponse
);
};
createConfig = async (request: CreateVisionLLMConfigRequest) => {
const parsed = createVisionLLMConfigRequest.safeParse(request);
if (!parsed.success) {
const msg = parsed.error.issues.map((i) => i.message).join(", ");
throw new ValidationError(`Invalid request: ${msg}`);
}
return baseApiService.post(`/api/v1/vision-llm-configs`, createVisionLLMConfigResponse, {
body: parsed.data,
});
};
getConfigs = async (searchSpaceId: number) => {
const params = new URLSearchParams({
search_space_id: String(searchSpaceId),
}).toString();
return baseApiService.get(`/api/v1/vision-llm-configs?${params}`, getVisionLLMConfigsResponse);
};
updateConfig = async (request: UpdateVisionLLMConfigRequest) => {
const parsed = updateVisionLLMConfigRequest.safeParse(request);
if (!parsed.success) {
const msg = parsed.error.issues.map((i) => i.message).join(", ");
throw new ValidationError(`Invalid request: ${msg}`);
}
const { id, data } = parsed.data;
return baseApiService.put(`/api/v1/vision-llm-configs/${id}`, updateVisionLLMConfigResponse, {
body: data,
});
};
deleteConfig = async (id: number) => {
return baseApiService.delete(`/api/v1/vision-llm-configs/${id}`, deleteVisionLLMConfigResponse);
};
}
export const visionLLMConfigApiService = new VisionLLMConfigApiService();

5
surfsense_web/lib/query-client/cache-keys.ts
View file

@ -39,6 +39,11 @@ export const cacheKeys = {
byId: (configId: number) => ["image-gen-configs", "detail", configId] as const,
global: () => ["image-gen-configs", "global"] as const,
},
visionLLMConfigs: {
all: (searchSpaceId: number) => ["vision-llm-configs", searchSpaceId] as const,
byId: (configId: number) => ["vision-llm-configs", "detail", configId] as const,
global: () => ["vision-llm-configs", "global"] as const,
},
auth: {
user: ["auth", "user"] as const,
},

2
surfsense_web/messages/en.json
View file

@ -738,6 +738,8 @@
"nav_role_assignments_desc": "Assign configs to agent roles",
"nav_image_models": "Image Models",
"nav_image_models_desc": "Configure image generation models",
"nav_vision_models": "Vision Models",
"nav_vision_models_desc": "Configure vision-capable LLM models",
"nav_system_instructions": "System Instructions",
"nav_system_instructions_desc": "SearchSpace-wide AI instructions",
"nav_public_links": "Public Chat Links",

2
surfsense_web/messages/es.json
View file

@ -738,6 +738,8 @@
"nav_role_assignments_desc": "Asignar configuraciones a roles de agente",
"nav_image_models": "Modelos de imagen",
"nav_image_models_desc": "Configurar modelos de generación de imágenes",
"nav_vision_models": "Modelos de visión",
"nav_vision_models_desc": "Configurar modelos LLM con capacidad de visión",
"nav_system_instructions": "Instrucciones del sistema",
"nav_system_instructions_desc": "Instrucciones de IA a nivel del espacio de búsqueda",
"nav_public_links": "Enlaces de chat públicos",

2
surfsense_web/messages/hi.json
View file

@ -738,6 +738,8 @@
"nav_role_assignments_desc": "एजेंट भूमिकाओं को कॉन्फ़िगरेशन असाइन करें",
"nav_image_models": "इमेज मॉडल",
"nav_image_models_desc": "इमेज जनरेशन मॉडल कॉन्फ़िगर करें",
"nav_vision_models": "विज़न मॉडल",
"nav_vision_models_desc": "विज़न-सक्षम LLM मॉडल कॉन्फ़िगर करें",
"nav_system_instructions": "सिस्टम निर्देश",
"nav_system_instructions_desc": "सर्च स्पेस-व्यापी AI निर्देश",
"nav_public_links": "सार्वजनिक चैट लिंक",

2
surfsense_web/messages/pt.json
View file

@ -738,6 +738,8 @@
"nav_role_assignments_desc": "Atribuir configurações a funções do agente",
"nav_image_models": "Modelos de imagem",
"nav_image_models_desc": "Configurar modelos de geração de imagens",
"nav_vision_models": "Modelos de visão",
"nav_vision_models_desc": "Configurar modelos LLM com capacidade de visão",
"nav_system_instructions": "Instruções do sistema",
"nav_system_instructions_desc": "Instruções de IA em nível do espaço de pesquisa",
"nav_public_links": "Links de chat públicos",

2
surfsense_web/messages/zh.json
View file

@ -722,6 +722,8 @@
"nav_role_assignments_desc": "为代理角色分配配置",
"nav_image_models": "图像模型",
"nav_image_models_desc": "配置图像生成模型",
"nav_vision_models": "视觉模型",
"nav_vision_models_desc": "配置具有视觉能力的LLM模型",
"nav_system_instructions": "系统指令",
"nav_system_instructions_desc": "搜索空间级别的 AI 指令",
"nav_public_links": "公开聊天链接",