Merge remote-tracking branch 'upstream/dev' into feat/azure-ocr

surfsense_backend/app/agents/autocomplete/__init__.py

@@ -0,0 +1,11 @@
+"""Agent-based vision autocomplete with scoped filesystem exploration."""
+
+from app.agents.autocomplete.autocomplete_agent import (
+    create_autocomplete_agent,
+    stream_autocomplete_agent,
+)
+
+__all__ = [
+    "create_autocomplete_agent",
+    "stream_autocomplete_agent",
+]

surfsense_backend/app/agents/autocomplete/autocomplete_agent.py

@@ -0,0 +1,497 @@
+"""Vision autocomplete agent with scoped filesystem exploration.
+
+Converts the stateless single-shot vision autocomplete into an agent that
+seeds a virtual filesystem from KB search results and lets the vision LLM
+explore documents via ``ls``, ``read_file``, ``glob``, ``grep``, etc.
+before generating the final completion.
+
+Performance: KB search and agent graph compilation run in parallel so
+the only sequential latency is KB-search (or agent compile, whichever is
+slower) + the agent's LLM turns.  There is no separate "query extraction"
+LLM call — the window title is used directly as the KB search query.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import re
+import uuid
+from collections.abc import AsyncGenerator
+from typing import Any
+
+from deepagents.graph import BASE_AGENT_PROMPT
+from deepagents.middleware.patch_tool_calls import PatchToolCallsMiddleware
+from langchain.agents import create_agent
+from langchain_anthropic.middleware import AnthropicPromptCachingMiddleware
+from langchain_core.language_models import BaseChatModel
+from langchain_core.messages import AIMessage, ToolMessage
+
+from app.agents.new_chat.middleware.filesystem import SurfSenseFilesystemMiddleware
+from app.agents.new_chat.middleware.knowledge_search import (
+    build_scoped_filesystem,
+    search_knowledge_base,
+)
+from app.services.new_streaming_service import VercelStreamingService
+
+logger = logging.getLogger(__name__)
+
+KB_TOP_K = 10
+
+# ---------------------------------------------------------------------------
+# System prompt
+# ---------------------------------------------------------------------------
+
+AUTOCOMPLETE_SYSTEM_PROMPT = """You are a smart writing assistant that analyzes the user's screen to draft or complete text.
+
+You will receive a screenshot of the user's screen. Your PRIMARY source of truth is the screenshot itself — the visual context determines what to write.
+
+Your job:
+1. Analyze the ENTIRE screenshot to understand what the user is working on (email thread, chat conversation, document, code editor, form, etc.).
+2. Identify the text area where the user will type.
+3. Generate the text the user most likely wants to write based on the visual context.
+
+You also have access to the user's knowledge base documents via filesystem tools. However:
+- ONLY consult the knowledge base if the screenshot clearly involves a topic where your KB documents are DIRECTLY relevant (e.g., the user is writing about a specific project/topic that matches a document title).
+- Do NOT explore documents just because they exist. Most autocomplete requests can be answered purely from the screenshot.
+- If you do read a document, only incorporate information that is 100% relevant to what the user is typing RIGHT NOW. Do not add extra details, background, or tangential information from the KB.
+- Keep your output SHORT — autocomplete should feel like a natural continuation, not an essay.
+
+Key behavior:
+- If the text area is EMPTY, draft a concise response or message based on what you see on screen (e.g., reply to an email, respond to a chat message, continue a document).
+- If the text area already has text, continue it naturally — typically just a sentence or two.
+
+Rules:
+- Be CONCISE. Prefer a single paragraph or a few sentences. Autocomplete is a quick assist, not a full draft.
+- Match the tone and formality of the surrounding context.
+- If the screen shows code, write code. If it shows a casual chat, be casual. If it shows a formal email, be formal.
+- Do NOT describe the screenshot or explain your reasoning.
+- Do NOT cite or reference documents explicitly — just let the knowledge inform your writing naturally.
+- If you cannot determine what to write, output an empty JSON array: []
+
+## Output Format
+
+You MUST provide exactly 3 different suggestion options. Each should be a distinct, plausible completion — vary the tone, detail level, or angle.
+
+Return your suggestions as a JSON array of exactly 3 strings. Output ONLY the JSON array, nothing else — no markdown fences, no explanation, no commentary.
+
+Example format:
+["First suggestion text here.", "Second suggestion — a different take.", "Third option with another approach."]
+
+## Filesystem Tools `ls`, `read_file`, `write_file`, `edit_file`, `glob`, `grep`
+
+All file paths must start with a `/`.
+- ls: list files and directories at a given path.
+- read_file: read a file from the filesystem.
+- write_file: create a temporary file in the session (not persisted).
+- edit_file: edit a file in the session (not persisted for /documents/ files).
+- glob: find files matching a pattern (e.g., "**/*.xml").
+- grep: search for text within files.
+
+## When to Use Filesystem Tools
+
+BEFORE reaching for any tool, ask yourself: "Can I write a good completion purely from the screenshot?" If yes, just write it — do NOT explore the KB.
+
+Only use tools when:
+- The user is clearly writing about a specific topic that likely has detailed information in their KB.
+- You need a specific fact, name, number, or reference that the screenshot doesn't provide.
+
+When you do use tools, be surgical:
+- Check the `ls` output first. If no document title looks relevant, stop — do not read files just to see what's there.
+- If a title looks relevant, read only the `<chunk_index>` (first ~20 lines) and jump to matched chunks. Do not read entire documents.
+- Extract only the specific information you need and move on to generating the completion.
+
+## Reading Documents Efficiently
+
+Documents are formatted as XML. Each document contains:
+- `<document_metadata>` — title, type, URL, etc.
+- `<chunk_index>` — a table of every chunk with its **line range** and a
+  `matched="true"` flag for chunks that matched the search query.
+- `<document_content>` — the actual chunks in original document order.
+
+**Workflow**: read the first ~20 lines to see the `<chunk_index>`, identify
+chunks marked `matched="true"`, then use `read_file(path, offset=<start_line>,
+limit=<lines>)` to jump directly to those sections."""
+
+APP_CONTEXT_BLOCK = """
+
+The user is currently working in "{app_name}" (window: "{window_title}"). Use this to understand the type of application and adapt your tone and format accordingly."""
+
+
+def _build_autocomplete_system_prompt(app_name: str, window_title: str) -> str:
+    prompt = AUTOCOMPLETE_SYSTEM_PROMPT
+    if app_name:
+        prompt += APP_CONTEXT_BLOCK.format(app_name=app_name, window_title=window_title)
+    return prompt
+
+
+# ---------------------------------------------------------------------------
+# Pre-compute KB filesystem (runs in parallel with agent compilation)
+# ---------------------------------------------------------------------------
+
+
+class _KBResult:
+    """Container for pre-computed KB filesystem results."""
+
+    __slots__ = ("files", "ls_ai_msg", "ls_tool_msg")
+
+    def __init__(
+        self,
+        files: dict[str, Any] | None = None,
+        ls_ai_msg: AIMessage | None = None,
+        ls_tool_msg: ToolMessage | None = None,
+    ) -> None:
+        self.files = files
+        self.ls_ai_msg = ls_ai_msg
+        self.ls_tool_msg = ls_tool_msg
+
+    @property
+    def has_documents(self) -> bool:
+        return bool(self.files)
+
+
+async def precompute_kb_filesystem(
+    search_space_id: int,
+    query: str,
+    top_k: int = KB_TOP_K,
+) -> _KBResult:
+    """Search the KB and build the scoped filesystem outside the agent.
+
+    This is designed to be called via ``asyncio.gather`` alongside agent
+    graph compilation so the two run concurrently.
+    """
+    if not query:
+        return _KBResult()
+
+    try:
+        search_results = await search_knowledge_base(
+            query=query,
+            search_space_id=search_space_id,
+            top_k=top_k,
+        )
+
+        if not search_results:
+            return _KBResult()
+
+        new_files, _ = await build_scoped_filesystem(
+            documents=search_results,
+            search_space_id=search_space_id,
+        )
+
+        if not new_files:
+            return _KBResult()
+
+        doc_paths = [
+            p
+            for p, v in new_files.items()
+            if p.startswith("/documents/") and v is not None
+        ]
+        tool_call_id = f"auto_ls_{uuid.uuid4().hex[:12]}"
+        ai_msg = AIMessage(
+            content="",
+            tool_calls=[
+                {"name": "ls", "args": {"path": "/documents"}, "id": tool_call_id}
+            ],
+        )
+        tool_msg = ToolMessage(
+            content=str(doc_paths) if doc_paths else "No documents found.",
+            tool_call_id=tool_call_id,
+        )
+        return _KBResult(files=new_files, ls_ai_msg=ai_msg, ls_tool_msg=tool_msg)
+
+    except Exception:
+        logger.warning(
+            "KB pre-computation failed, proceeding without KB", exc_info=True
+        )
+        return _KBResult()
+
+
+# ---------------------------------------------------------------------------
+# Filesystem middleware — no save_document, no persistence
+# ---------------------------------------------------------------------------
+
+
+class AutocompleteFilesystemMiddleware(SurfSenseFilesystemMiddleware):
+    """Filesystem middleware for autocomplete — read-only exploration only.
+
+    Strips ``save_document`` (permanent KB persistence) and passes
+    ``search_space_id=None`` so ``write_file`` / ``edit_file`` stay ephemeral.
+    """
+
+    def __init__(self) -> None:
+        super().__init__(search_space_id=None, created_by_id=None)
+        self.tools = [t for t in self.tools if t.name != "save_document"]
+
+
+# ---------------------------------------------------------------------------
+# Agent factory
+# ---------------------------------------------------------------------------
+
+
+async def _compile_agent(
+    llm: BaseChatModel,
+    app_name: str,
+    window_title: str,
+) -> Any:
+    """Compile the agent graph (CPU-bound, runs in a thread)."""
+    system_prompt = _build_autocomplete_system_prompt(app_name, window_title)
+    final_system_prompt = system_prompt + "\n\n" + BASE_AGENT_PROMPT
+
+    middleware = [
+        AutocompleteFilesystemMiddleware(),
+        PatchToolCallsMiddleware(),
+        AnthropicPromptCachingMiddleware(unsupported_model_behavior="ignore"),
+    ]
+
+    agent = await asyncio.to_thread(
+        create_agent,
+        llm,
+        system_prompt=final_system_prompt,
+        tools=[],
+        middleware=middleware,
+    )
+    return agent.with_config({"recursion_limit": 200})
+
+
+async def create_autocomplete_agent(
+    llm: BaseChatModel,
+    *,
+    search_space_id: int,
+    kb_query: str,
+    app_name: str = "",
+    window_title: str = "",
+) -> tuple[Any, _KBResult]:
+    """Create the autocomplete agent and pre-compute KB in parallel.
+
+    Returns ``(agent, kb_result)`` so the caller can inject the pre-computed
+    filesystem into the agent's initial state without any middleware delay.
+    """
+    agent, kb = await asyncio.gather(
+        _compile_agent(llm, app_name, window_title),
+        precompute_kb_filesystem(search_space_id, kb_query),
+    )
+    return agent, kb
+
+
+# ---------------------------------------------------------------------------
+# JSON suggestion parsing (with fallback)
+# ---------------------------------------------------------------------------
+
+
+def _parse_suggestions(raw: str) -> list[str]:
+    """Extract a list of suggestion strings from the agent's output.
+
+    Tries, in order:
+      1. Direct ``json.loads``
+      2. Extract content between ```json ... ``` fences
+      3. Find the first ``[`` … ``]`` span
+    Falls back to wrapping the raw text as a single suggestion.
+    """
+    text = raw.strip()
+    if not text:
+        return []
+
+    for candidate in _json_candidates(text):
+        try:
+            parsed = json.loads(candidate)
+            if isinstance(parsed, list) and all(isinstance(s, str) for s in parsed):
+                return [s for s in parsed if s.strip()]
+        except (json.JSONDecodeError, ValueError):
+            continue
+
+    return [text]
+
+
+def _json_candidates(text: str) -> list[str]:
+    """Yield candidate JSON strings from raw text."""
+    candidates = [text]
+
+    fence = re.search(r"```(?:json)?\s*\n?(.*?)```", text, re.DOTALL)
+    if fence:
+        candidates.append(fence.group(1).strip())
+
+    bracket = re.search(r"\[.*]", text, re.DOTALL)
+    if bracket:
+        candidates.append(bracket.group(0))
+
+    return candidates
+
+
+# ---------------------------------------------------------------------------
+# Streaming helper
+# ---------------------------------------------------------------------------
+
+
+async def stream_autocomplete_agent(
+    agent: Any,
+    input_data: dict[str, Any],
+    streaming_service: VercelStreamingService,
+    *,
+    emit_message_start: bool = True,
+) -> AsyncGenerator[str, None]:
+    """Stream agent events as Vercel SSE, with thinking steps for tool calls.
+
+    When ``emit_message_start`` is False the caller has already sent the
+    ``message_start`` event (e.g. to show preparation steps before the agent
+    runs).
+    """
+    thread_id = uuid.uuid4().hex
+    config = {"configurable": {"thread_id": thread_id}}
+
+    text_buffer: list[str] = []
+    active_tool_depth = 0
+    thinking_step_counter = 0
+    tool_step_ids: dict[str, str] = {}
+    step_titles: dict[str, str] = {}
+    completed_step_ids: set[str] = set()
+    last_active_step_id: str | None = None
+
+    def next_thinking_step_id() -> str:
+        nonlocal thinking_step_counter
+        thinking_step_counter += 1
+        return f"autocomplete-step-{thinking_step_counter}"
+
+    def complete_current_step() -> str | None:
+        nonlocal last_active_step_id
+        if last_active_step_id and last_active_step_id not in completed_step_ids:
+            completed_step_ids.add(last_active_step_id)
+            title = step_titles.get(last_active_step_id, "Done")
+            event = streaming_service.format_thinking_step(
+                step_id=last_active_step_id,
+                title=title,
+                status="complete",
+            )
+            last_active_step_id = None
+            return event
+        return None
+
+    if emit_message_start:
+        yield streaming_service.format_message_start()
+
+    gen_step_id = next_thinking_step_id()
+    last_active_step_id = gen_step_id
+    step_titles[gen_step_id] = "Generating suggestions"
+    yield streaming_service.format_thinking_step(
+        step_id=gen_step_id,
+        title="Generating suggestions",
+        status="in_progress",
+    )
+
+    try:
+        async for event in agent.astream_events(
+            input_data, config=config, version="v2"
+        ):
+            event_type = event.get("event", "")
+            if event_type == "on_chat_model_stream":
+                if active_tool_depth > 0:
+                    continue
+                if "surfsense:internal" in event.get("tags", []):
+                    continue
+                chunk = event.get("data", {}).get("chunk")
+                if chunk and hasattr(chunk, "content"):
+                    content = chunk.content
+                    if content and isinstance(content, str):
+                        text_buffer.append(content)
+
+            elif event_type == "on_chat_model_end":
+                if active_tool_depth > 0:
+                    continue
+                if "surfsense:internal" in event.get("tags", []):
+                    continue
+                output = event.get("data", {}).get("output")
+                if output and hasattr(output, "content"):
+                    if getattr(output, "tool_calls", None):
+                        continue
+                    content = output.content
+                    if content and isinstance(content, str) and not text_buffer:
+                        text_buffer.append(content)
+
+            elif event_type == "on_tool_start":
+                active_tool_depth += 1
+                tool_name = event.get("name", "unknown_tool")
+                run_id = event.get("run_id", "")
+                tool_input = event.get("data", {}).get("input", {})
+
+                step_event = complete_current_step()
+                if step_event:
+                    yield step_event
+
+                tool_step_id = next_thinking_step_id()
+                tool_step_ids[run_id] = tool_step_id
+                last_active_step_id = tool_step_id
+
+                title, items = _describe_tool_call(tool_name, tool_input)
+                step_titles[tool_step_id] = title
+                yield streaming_service.format_thinking_step(
+                    step_id=tool_step_id,
+                    title=title,
+                    status="in_progress",
+                    items=items,
+                )
+
+            elif event_type == "on_tool_end":
+                active_tool_depth = max(0, active_tool_depth - 1)
+                run_id = event.get("run_id", "")
+                step_id = tool_step_ids.pop(run_id, None)
+                if step_id and step_id not in completed_step_ids:
+                    completed_step_ids.add(step_id)
+                    title = step_titles.get(step_id, "Done")
+                    yield streaming_service.format_thinking_step(
+                        step_id=step_id,
+                        title=title,
+                        status="complete",
+                    )
+                    if last_active_step_id == step_id:
+                        last_active_step_id = None
+
+        step_event = complete_current_step()
+        if step_event:
+            yield step_event
+
+        raw_text = "".join(text_buffer)
+        suggestions = _parse_suggestions(raw_text)
+
+        yield streaming_service.format_data(
+            "suggestions", {"options": suggestions}
+        )
+
+        yield streaming_service.format_finish()
+        yield streaming_service.format_done()
+
+    except Exception as e:
+        logger.error(f"Autocomplete agent streaming error: {e}", exc_info=True)
+        yield streaming_service.format_error("Autocomplete failed. Please try again.")
+        yield streaming_service.format_done()
+
+
+def _describe_tool_call(tool_name: str, tool_input: Any) -> tuple[str, list[str]]:
+    """Return a human-readable (title, items) for a tool call thinking step."""
+    inp = tool_input if isinstance(tool_input, dict) else {}
+    if tool_name == "ls":
+        path = inp.get("path", "/")
+        return "Listing files", [path]
+    if tool_name == "read_file":
+        fp = inp.get("file_path", "")
+        display = fp if len(fp) <= 80 else "…" + fp[-77:]
+        return "Reading file", [display]
+    if tool_name == "write_file":
+        fp = inp.get("file_path", "")
+        display = fp if len(fp) <= 80 else "…" + fp[-77:]
+        return "Writing file", [display]
+    if tool_name == "edit_file":
+        fp = inp.get("file_path", "")
+        display = fp if len(fp) <= 80 else "…" + fp[-77:]
+        return "Editing file", [display]
+    if tool_name == "glob":
+        pat = inp.get("pattern", "")
+        base = inp.get("path", "/")
+        return "Searching files", [f"{pat} in {base}"]
+    if tool_name == "grep":
+        pat = inp.get("pattern", "")
+        path = inp.get("path", "")
+        display_pat = pat[:60] + ("…" if len(pat) > 60 else "")
+        return "Searching content", [
+            f'"{display_pat}"' + (f" in {path}" if path else "")
+        ]
+    return f"Using {tool_name}", []

surfsense_backend/app/app.py

@@ -25,7 +25,12 @@ from app.agents.new_chat.checkpointer import (
     close_checkpointer,
     setup_checkpointer_tables,
 )
-from app.config import config, initialize_image_gen_router, initialize_llm_router
+from app.config import (
+    config,
+    initialize_image_gen_router,
+    initialize_llm_router,
+    initialize_vision_llm_router,
+)
 from app.db import User, create_db_and_tables, get_async_session
 from app.routes import router as crud_router
 from app.routes.auth_routes import router as auth_router
@@ -223,6 +228,7 @@ async def lifespan(app: FastAPI):
     await setup_checkpointer_tables()
     initialize_llm_router()
     initialize_image_gen_router()
+    initialize_vision_llm_router()
     try:
         await asyncio.wait_for(seed_surfsense_docs(), timeout=120)
     except TimeoutError:

surfsense_backend/app/celery_app.py

@@ -18,10 +18,15 @@ def init_worker(**kwargs):
     This ensures the Auto mode (LiteLLM Router) is available for background tasks
     like document summarization and image generation.
     """
-    from app.config import initialize_image_gen_router, initialize_llm_router
+    from app.config import (
+        initialize_image_gen_router,
+        initialize_llm_router,
+        initialize_vision_llm_router,
+    )
 
     initialize_llm_router()
     initialize_image_gen_router()
+    initialize_vision_llm_router()
 
 
 # Get Celery configuration from environment

surfsense_backend/app/config/__init__.py

@@ -102,6 +102,44 @@ def load_global_image_gen_configs():
         return []
 
 
+def load_global_vision_llm_configs():
+    global_config_file = BASE_DIR / "app" / "config" / "global_llm_config.yaml"
+
+    if not global_config_file.exists():
+        return []
+
+    try:
+        with open(global_config_file, encoding="utf-8") as f:
+            data = yaml.safe_load(f)
+            return data.get("global_vision_llm_configs", [])
+    except Exception as e:
+        print(f"Warning: Failed to load global vision LLM configs: {e}")
+        return []
+
+
+def load_vision_llm_router_settings():
+    default_settings = {
+        "routing_strategy": "usage-based-routing",
+        "num_retries": 3,
+        "allowed_fails": 3,
+        "cooldown_time": 60,
+    }
+
+    global_config_file = BASE_DIR / "app" / "config" / "global_llm_config.yaml"
+
+    if not global_config_file.exists():
+        return default_settings
+
+    try:
+        with open(global_config_file, encoding="utf-8") as f:
+            data = yaml.safe_load(f)
+            settings = data.get("vision_llm_router_settings", {})
+            return {**default_settings, **settings}
+    except Exception as e:
+        print(f"Warning: Failed to load vision LLM router settings: {e}")
+        return default_settings
+
+
 def load_image_gen_router_settings():
     """
     Load router settings for image generation Auto mode from YAML file.
@@ -182,6 +220,29 @@ def initialize_image_gen_router():
         print(f"Warning: Failed to initialize Image Generation Router: {e}")
 
 
+def initialize_vision_llm_router():
+    vision_configs = load_global_vision_llm_configs()
+    router_settings = load_vision_llm_router_settings()
+
+    if not vision_configs:
+        print(
+            "Info: No global vision LLM configs found, "
+            "Vision LLM Auto mode will not be available"
+        )
+        return
+
+    try:
+        from app.services.vision_llm_router_service import VisionLLMRouterService
+
+        VisionLLMRouterService.initialize(vision_configs, router_settings)
+        print(
+            f"Info: Vision LLM Router initialized with {len(vision_configs)} models "
+            f"(strategy: {router_settings.get('routing_strategy', 'usage-based-routing')})"
+        )
+    except Exception as e:
+        print(f"Warning: Failed to initialize Vision LLM Router: {e}")
+
+
 class Config:
     # Check if ffmpeg is installed
     if not is_ffmpeg_installed():
@@ -335,6 +396,12 @@ class Config:
     # Router settings for Image Generation Auto mode
     IMAGE_GEN_ROUTER_SETTINGS = load_image_gen_router_settings()
 
+    # Global Vision LLM Configurations (optional)
+    GLOBAL_VISION_LLM_CONFIGS = load_global_vision_llm_configs()
+
+    # Router settings for Vision LLM Auto mode
+    VISION_LLM_ROUTER_SETTINGS = load_vision_llm_router_settings()
+
     # Chonkie Configuration | Edit this to your needs
     EMBEDDING_MODEL = os.getenv("EMBEDDING_MODEL")
     # Azure OpenAI credentials from environment variables

surfsense_backend/app/config/global_llm_config.example.yaml

@@ -263,6 +263,82 @@ global_image_generation_configs:
   #   rpm: 30
   #   litellm_params: {}
 
+# =============================================================================
+# Vision LLM Configuration
+# =============================================================================
+# These configurations power the vision autocomplete feature (screenshot analysis).
+# Only vision-capable models should be used here (e.g. GPT-4o, Gemini Pro, Claude 3).
+# Supported providers: OpenAI, Anthropic, Google, Azure OpenAI, Vertex AI, Bedrock,
+# xAI, OpenRouter, Ollama, Groq, Together AI, Fireworks AI, DeepSeek, Mistral, Custom
+#
+# Auto mode (ID 0) uses LiteLLM Router for load balancing across all vision configs.
+
+# Router Settings for Vision LLM Auto Mode
+vision_llm_router_settings:
+  routing_strategy: "usage-based-routing"
+  num_retries: 3
+  allowed_fails: 3
+  cooldown_time: 60
+
+global_vision_llm_configs:
+  # Example: OpenAI GPT-4o (recommended for vision)
+  - id: -1
+    name: "Global GPT-4o Vision"
+    description: "OpenAI's GPT-4o with strong vision capabilities"
+    provider: "OPENAI"
+    model_name: "gpt-4o"
+    api_key: "sk-your-openai-api-key-here"
+    api_base: ""
+    rpm: 500
+    tpm: 100000
+    litellm_params:
+      temperature: 0.3
+      max_tokens: 1000
+
+  # Example: Google Gemini 2.0 Flash
+  - id: -2
+    name: "Global Gemini 2.0 Flash"
+    description: "Google's fast vision model with large context"
+    provider: "GOOGLE"
+    model_name: "gemini-2.0-flash"
+    api_key: "your-google-ai-api-key-here"
+    api_base: ""
+    rpm: 1000
+    tpm: 200000
+    litellm_params:
+      temperature: 0.3
+      max_tokens: 1000
+
+  # Example: Anthropic Claude 3.5 Sonnet
+  - id: -3
+    name: "Global Claude 3.5 Sonnet Vision"
+    description: "Anthropic's Claude 3.5 Sonnet with vision support"
+    provider: "ANTHROPIC"
+    model_name: "claude-3-5-sonnet-20241022"
+    api_key: "sk-ant-your-anthropic-api-key-here"
+    api_base: ""
+    rpm: 1000
+    tpm: 100000
+    litellm_params:
+      temperature: 0.3
+      max_tokens: 1000
+
+  # Example: Azure OpenAI GPT-4o
+  # - id: -4
+  #   name: "Global Azure GPT-4o Vision"
+  #   description: "Azure-hosted GPT-4o for vision analysis"
+  #   provider: "AZURE_OPENAI"
+  #   model_name: "azure/gpt-4o-deployment"
+  #   api_key: "your-azure-api-key-here"
+  #   api_base: "https://your-resource.openai.azure.com"
+  #   api_version: "2024-02-15-preview"
+  #   rpm: 500
+  #   tpm: 100000
+  #   litellm_params:
+  #     temperature: 0.3
+  #     max_tokens: 1000
+  #     base_model: "gpt-4o"
+
 # Notes:
 # - ID 0 is reserved for "Auto" mode - uses LiteLLM Router for load balancing
 # - Use negative IDs to distinguish global configs from user configs (NewLLMConfig in DB)
@@ -283,3 +359,9 @@ global_image_generation_configs:
 # - The router uses litellm.aimage_generation() for async image generation
 # - Only RPM (requests per minute) is relevant for image generation rate limiting.
 #   TPM (tokens per minute) does not apply since image APIs are billed/rate-limited per request, not per token.
+#
+# VISION LLM NOTES:
+# - Vision configs use the same ID scheme (negative for global, positive for user DB)
+# - Only use vision-capable models (GPT-4o, Gemini, Claude 3, etc.)
+# - Lower temperature (0.3) is recommended for accurate screenshot analysis
+# - Lower max_tokens (1000) is sufficient since autocomplete produces short suggestions

surfsense_backend/app/config/vision_model_list_fallback.json

@@ -0,0 +1,23 @@
+[
+  {"value": "gpt-4o", "label": "GPT-4o", "provider": "OPENAI", "context_window": "128K"},
+  {"value": "gpt-4o-mini", "label": "GPT-4o Mini", "provider": "OPENAI", "context_window": "128K"},
+  {"value": "gpt-4-turbo", "label": "GPT-4 Turbo", "provider": "OPENAI", "context_window": "128K"},
+  {"value": "claude-sonnet-4-20250514", "label": "Claude Sonnet 4", "provider": "ANTHROPIC", "context_window": "200K"},
+  {"value": "claude-3-7-sonnet-20250219", "label": "Claude 3.7 Sonnet", "provider": "ANTHROPIC", "context_window": "200K"},
+  {"value": "claude-3-5-sonnet-20241022", "label": "Claude 3.5 Sonnet", "provider": "ANTHROPIC", "context_window": "200K"},
+  {"value": "claude-3-opus-20240229", "label": "Claude 3 Opus", "provider": "ANTHROPIC", "context_window": "200K"},
+  {"value": "claude-3-haiku-20240307", "label": "Claude 3 Haiku", "provider": "ANTHROPIC", "context_window": "200K"},
+  {"value": "gemini-2.5-flash", "label": "Gemini 2.5 Flash", "provider": "GOOGLE", "context_window": "1M"},
+  {"value": "gemini-2.5-pro", "label": "Gemini 2.5 Pro", "provider": "GOOGLE", "context_window": "1M"},
+  {"value": "gemini-2.0-flash", "label": "Gemini 2.0 Flash", "provider": "GOOGLE", "context_window": "1M"},
+  {"value": "gemini-1.5-pro", "label": "Gemini 1.5 Pro", "provider": "GOOGLE", "context_window": "1M"},
+  {"value": "gemini-1.5-flash", "label": "Gemini 1.5 Flash", "provider": "GOOGLE", "context_window": "1M"},
+  {"value": "pixtral-large-latest", "label": "Pixtral Large", "provider": "MISTRAL", "context_window": "128K"},
+  {"value": "pixtral-12b-2409", "label": "Pixtral 12B", "provider": "MISTRAL", "context_window": "128K"},
+  {"value": "grok-2-vision-1212", "label": "Grok 2 Vision", "provider": "XAI", "context_window": "32K"},
+  {"value": "llava", "label": "LLaVA", "provider": "OLLAMA"},
+  {"value": "bakllava", "label": "BakLLaVA", "provider": "OLLAMA"},
+  {"value": "llava-llama3", "label": "LLaVA Llama 3", "provider": "OLLAMA"},
+  {"value": "llama-4-scout-17b-16e-instruct", "label": "Llama 4 Scout 17B", "provider": "GROQ", "context_window": "128K"},
+  {"value": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "label": "Llama 4 Scout 17B", "provider": "TOGETHER_AI", "context_window": "128K"}
+]

surfsense_backend/app/db.py

@@ -260,6 +260,24 @@ class ImageGenProvider(StrEnum):
     NSCALE = "NSCALE"
 
 
+class VisionProvider(StrEnum):
+    OPENAI = "OPENAI"
+    ANTHROPIC = "ANTHROPIC"
+    GOOGLE = "GOOGLE"
+    AZURE_OPENAI = "AZURE_OPENAI"
+    VERTEX_AI = "VERTEX_AI"
+    BEDROCK = "BEDROCK"
+    XAI = "XAI"
+    OPENROUTER = "OPENROUTER"
+    OLLAMA = "OLLAMA"
+    GROQ = "GROQ"
+    TOGETHER_AI = "TOGETHER_AI"
+    FIREWORKS_AI = "FIREWORKS_AI"
+    DEEPSEEK = "DEEPSEEK"
+    MISTRAL = "MISTRAL"
+    CUSTOM = "CUSTOM"
+
+
 class LogLevel(StrEnum):
     DEBUG = "DEBUG"
     INFO = "INFO"
@@ -377,6 +395,11 @@ class Permission(StrEnum):
     IMAGE_GENERATIONS_READ = "image_generations:read"
     IMAGE_GENERATIONS_DELETE = "image_generations:delete"
 
+    # Vision LLM Configs
+    VISION_CONFIGS_CREATE = "vision_configs:create"
+    VISION_CONFIGS_READ = "vision_configs:read"
+    VISION_CONFIGS_DELETE = "vision_configs:delete"
+
     # Connectors
     CONNECTORS_CREATE = "connectors:create"
     CONNECTORS_READ = "connectors:read"
@@ -445,6 +468,9 @@ DEFAULT_ROLE_PERMISSIONS = {
         # Image Generations (create and read, no delete)
         Permission.IMAGE_GENERATIONS_CREATE.value,
         Permission.IMAGE_GENERATIONS_READ.value,
+        # Vision Configs (create and read, no delete)
+        Permission.VISION_CONFIGS_CREATE.value,
+        Permission.VISION_CONFIGS_READ.value,
         # Connectors (no delete)
         Permission.CONNECTORS_CREATE.value,
         Permission.CONNECTORS_READ.value,
@@ -478,6 +504,8 @@ DEFAULT_ROLE_PERMISSIONS = {
         Permission.VIDEO_PRESENTATIONS_READ.value,
         # Image Generations (read only)
         Permission.IMAGE_GENERATIONS_READ.value,
+        # Vision Configs (read only)
+        Permission.VISION_CONFIGS_READ.value,
         # Connectors (read only)
         Permission.CONNECTORS_READ.value,
         # Logs (read only)
@@ -1263,6 +1291,35 @@ class ImageGenerationConfig(BaseModel, TimestampMixin):
     user = relationship("User", back_populates="image_generation_configs")
 
 
+class VisionLLMConfig(BaseModel, TimestampMixin):
+    __tablename__ = "vision_llm_configs"
+
+    name = Column(String(100), nullable=False, index=True)
+    description = Column(String(500), nullable=True)
+
+    provider = Column(SQLAlchemyEnum(VisionProvider), nullable=False)
+    custom_provider = Column(String(100), nullable=True)
+    model_name = Column(String(100), nullable=False)
+
+    api_key = Column(String, nullable=False)
+    api_base = Column(String(500), nullable=True)
+    api_version = Column(String(50), nullable=True)
+
+    litellm_params = Column(JSON, nullable=True, default={})
+
+    search_space_id = Column(
+        Integer, ForeignKey("searchspaces.id", ondelete="CASCADE"), nullable=False
+    )
+    search_space = relationship(
+        "SearchSpace", back_populates="vision_llm_configs"
+    )
+
+    user_id = Column(
+        UUID(as_uuid=True), ForeignKey("user.id", ondelete="CASCADE"), nullable=False
+    )
+    user = relationship("User", back_populates="vision_llm_configs")
+
+
 class ImageGeneration(BaseModel, TimestampMixin):
     """
     Stores image generation requests and results using litellm.aimage_generation().
@@ -1351,7 +1408,7 @@ class SearchSpace(BaseModel, TimestampMixin):
     image_generation_config_id = Column(
         Integer, nullable=True, default=0
     )  # For image generation, defaults to Auto mode
-    vision_llm_id = Column(
+    vision_llm_config_id = Column(
         Integer, nullable=True, default=0
     )  # For vision/screenshot analysis, defaults to Auto mode
 
@@ -1432,6 +1489,12 @@ class SearchSpace(BaseModel, TimestampMixin):
         order_by="ImageGenerationConfig.id",
         cascade="all, delete-orphan",
     )
+    vision_llm_configs = relationship(
+        "VisionLLMConfig",
+        back_populates="search_space",
+        order_by="VisionLLMConfig.id",
+        cascade="all, delete-orphan",
+    )
 
     # RBAC relationships
     roles = relationship(
@@ -1961,6 +2024,12 @@ if config.AUTH_TYPE == "GOOGLE":
             passive_deletes=True,
         )
 
+        vision_llm_configs = relationship(
+            "VisionLLMConfig",
+            back_populates="user",
+            passive_deletes=True,
+        )
+
         # User memories for personalized AI responses
         memories = relationship(
             "UserMemory",
@@ -2075,6 +2144,12 @@ else:
             passive_deletes=True,
         )
 
+        vision_llm_configs = relationship(
+            "VisionLLMConfig",
+            back_populates="user",
+            passive_deletes=True,
+        )
+
         # User memories for personalized AI responses
         memories = relationship(
             "UserMemory",

surfsense_backend/app/routes/__init__.py

@@ -49,6 +49,7 @@ from .stripe_routes import router as stripe_router
 from .surfsense_docs_routes import router as surfsense_docs_router
 from .teams_add_connector_route import router as teams_add_connector_router
 from .video_presentations_routes import router as video_presentations_router
+from .vision_llm_routes import router as vision_llm_router
 from .youtube_routes import router as youtube_router
 
 router = APIRouter()
@@ -68,6 +69,7 @@ router.include_router(
 )  # Video presentation status and streaming
 router.include_router(reports_router)  # Report CRUD and multi-format export
 router.include_router(image_generation_router)  # Image generation via litellm
+router.include_router(vision_llm_router)  # Vision LLM configs for screenshot analysis
 router.include_router(search_source_connectors_router)
 router.include_router(google_calendar_add_connector_router)
 router.include_router(google_gmail_add_connector_router)

surfsense_backend/app/routes/search_spaces_routes.py

@@ -14,6 +14,7 @@ from app.db import (
     SearchSpaceMembership,
     SearchSpaceRole,
     User,
+    VisionLLMConfig,
     get_async_session,
     get_default_roles_config,
 )
@@ -483,6 +484,63 @@ async def _get_image_gen_config_by_id(
     return None
 
 
+async def _get_vision_llm_config_by_id(
+    session: AsyncSession, config_id: int | None
+) -> dict | None:
+    if config_id is None:
+        return None
+
+    if config_id == 0:
+        return {
+            "id": 0,
+            "name": "Auto (Fastest)",
+            "description": "Automatically routes requests across available vision LLM providers",
+            "provider": "AUTO",
+            "model_name": "auto",
+            "is_global": True,
+            "is_auto_mode": True,
+        }
+
+    if config_id < 0:
+        for cfg in config.GLOBAL_VISION_LLM_CONFIGS:
+            if cfg.get("id") == config_id:
+                return {
+                    "id": cfg.get("id"),
+                    "name": cfg.get("name"),
+                    "description": cfg.get("description"),
+                    "provider": cfg.get("provider"),
+                    "custom_provider": cfg.get("custom_provider"),
+                    "model_name": cfg.get("model_name"),
+                    "api_base": cfg.get("api_base") or None,
+                    "api_version": cfg.get("api_version") or None,
+                    "litellm_params": cfg.get("litellm_params", {}),
+                    "is_global": True,
+                }
+        return None
+
+    result = await session.execute(
+        select(VisionLLMConfig).filter(VisionLLMConfig.id == config_id)
+    )
+    db_config = result.scalars().first()
+    if db_config:
+        return {
+            "id": db_config.id,
+            "name": db_config.name,
+            "description": db_config.description,
+            "provider": db_config.provider.value if db_config.provider else None,
+            "custom_provider": db_config.custom_provider,
+            "model_name": db_config.model_name,
+            "api_base": db_config.api_base,
+            "api_version": db_config.api_version,
+            "litellm_params": db_config.litellm_params or {},
+            "created_at": db_config.created_at.isoformat()
+            if db_config.created_at
+            else None,
+            "search_space_id": db_config.search_space_id,
+        }
+    return None
+
+
 @router.get(
     "/search-spaces/{search_space_id}/llm-preferences",
     response_model=LLMPreferencesRead,
@@ -522,17 +580,19 @@ async def get_llm_preferences(
         image_generation_config = await _get_image_gen_config_by_id(
             session, search_space.image_generation_config_id
         )
-        vision_llm = await _get_llm_config_by_id(session, search_space.vision_llm_id)
+        vision_llm_config = await _get_vision_llm_config_by_id(
+            session, search_space.vision_llm_config_id
+        )
 
         return LLMPreferencesRead(
             agent_llm_id=search_space.agent_llm_id,
             document_summary_llm_id=search_space.document_summary_llm_id,
             image_generation_config_id=search_space.image_generation_config_id,
-            vision_llm_id=search_space.vision_llm_id,
+            vision_llm_config_id=search_space.vision_llm_config_id,
             agent_llm=agent_llm,
             document_summary_llm=document_summary_llm,
             image_generation_config=image_generation_config,
-            vision_llm=vision_llm,
+            vision_llm_config=vision_llm_config,
         )
 
     except HTTPException:
@@ -592,17 +652,19 @@ async def update_llm_preferences(
         image_generation_config = await _get_image_gen_config_by_id(
             session, search_space.image_generation_config_id
         )
-        vision_llm = await _get_llm_config_by_id(session, search_space.vision_llm_id)
+        vision_llm_config = await _get_vision_llm_config_by_id(
+            session, search_space.vision_llm_config_id
+        )
 
         return LLMPreferencesRead(
             agent_llm_id=search_space.agent_llm_id,
             document_summary_llm_id=search_space.document_summary_llm_id,
             image_generation_config_id=search_space.image_generation_config_id,
-            vision_llm_id=search_space.vision_llm_id,
+            vision_llm_config_id=search_space.vision_llm_config_id,
             agent_llm=agent_llm,
             document_summary_llm=document_summary_llm,
             image_generation_config=image_generation_config,
-            vision_llm=vision_llm,
+            vision_llm_config=vision_llm_config,
         )
 
     except HTTPException:

surfsense_backend/app/routes/vision_llm_routes.py

@@ -0,0 +1,295 @@
+import logging
+
+from fastapi import APIRouter, Depends, HTTPException
+from pydantic import BaseModel
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.config import config
+from app.db import (
+    Permission,
+    User,
+    VisionLLMConfig,
+    get_async_session,
+)
+from app.schemas import (
+    GlobalVisionLLMConfigRead,
+    VisionLLMConfigCreate,
+    VisionLLMConfigRead,
+    VisionLLMConfigUpdate,
+)
+from app.services.vision_model_list_service import get_vision_model_list
+from app.users import current_active_user
+from app.utils.rbac import check_permission
+
+router = APIRouter()
+logger = logging.getLogger(__name__)
+
+
+# =============================================================================
+# Vision Model Catalogue (from OpenRouter, filtered for image-input models)
+# =============================================================================
+
+
+class VisionModelListItem(BaseModel):
+    value: str
+    label: str
+    provider: str
+    context_window: str | None = None
+
+
+@router.get("/vision-models", response_model=list[VisionModelListItem])
+async def list_vision_models(
+    user: User = Depends(current_active_user),
+):
+    """Return vision-capable models sourced from OpenRouter (filtered by image input)."""
+    try:
+        return await get_vision_model_list()
+    except Exception as e:
+        logger.exception("Failed to fetch vision model list")
+        raise HTTPException(
+            status_code=500, detail=f"Failed to fetch vision model list: {e!s}"
+        ) from e
+
+
+# =============================================================================
+# Global Vision LLM Configs (from YAML)
+# =============================================================================
+
+
+@router.get(
+    "/global-vision-llm-configs",
+    response_model=list[GlobalVisionLLMConfigRead],
+)
+async def get_global_vision_llm_configs(
+    user: User = Depends(current_active_user),
+):
+    try:
+        global_configs = config.GLOBAL_VISION_LLM_CONFIGS
+        safe_configs = []
+
+        if global_configs and len(global_configs) > 0:
+            safe_configs.append(
+                {
+                    "id": 0,
+                    "name": "Auto (Fastest)",
+                    "description": "Automatically routes across available vision LLM providers.",
+                    "provider": "AUTO",
+                    "custom_provider": None,
+                    "model_name": "auto",
+                    "api_base": None,
+                    "api_version": None,
+                    "litellm_params": {},
+                    "is_global": True,
+                    "is_auto_mode": True,
+                }
+            )
+
+        for cfg in global_configs:
+            safe_configs.append(
+                {
+                    "id": cfg.get("id"),
+                    "name": cfg.get("name"),
+                    "description": cfg.get("description"),
+                    "provider": cfg.get("provider"),
+                    "custom_provider": cfg.get("custom_provider"),
+                    "model_name": cfg.get("model_name"),
+                    "api_base": cfg.get("api_base") or None,
+                    "api_version": cfg.get("api_version") or None,
+                    "litellm_params": cfg.get("litellm_params", {}),
+                    "is_global": True,
+                }
+            )
+
+        return safe_configs
+    except Exception as e:
+        logger.exception("Failed to fetch global vision LLM configs")
+        raise HTTPException(
+            status_code=500, detail=f"Failed to fetch configs: {e!s}"
+        ) from e
+
+
+# =============================================================================
+# VisionLLMConfig CRUD
+# =============================================================================
+
+
+@router.post("/vision-llm-configs", response_model=VisionLLMConfigRead)
+async def create_vision_llm_config(
+    config_data: VisionLLMConfigCreate,
+    session: AsyncSession = Depends(get_async_session),
+    user: User = Depends(current_active_user),
+):
+    try:
+        await check_permission(
+            session,
+            user,
+            config_data.search_space_id,
+            Permission.VISION_CONFIGS_CREATE.value,
+            "You don't have permission to create vision LLM configs in this search space",
+        )
+
+        db_config = VisionLLMConfig(**config_data.model_dump(), user_id=user.id)
+        session.add(db_config)
+        await session.commit()
+        await session.refresh(db_config)
+        return db_config
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        await session.rollback()
+        logger.exception("Failed to create VisionLLMConfig")
+        raise HTTPException(
+            status_code=500, detail=f"Failed to create config: {e!s}"
+        ) from e
+
+
+@router.get("/vision-llm-configs", response_model=list[VisionLLMConfigRead])
+async def list_vision_llm_configs(
+    search_space_id: int,
+    skip: int = 0,
+    limit: int = 100,
+    session: AsyncSession = Depends(get_async_session),
+    user: User = Depends(current_active_user),
+):
+    try:
+        await check_permission(
+            session,
+            user,
+            search_space_id,
+            Permission.VISION_CONFIGS_READ.value,
+            "You don't have permission to view vision LLM configs in this search space",
+        )
+
+        result = await session.execute(
+            select(VisionLLMConfig)
+            .filter(VisionLLMConfig.search_space_id == search_space_id)
+            .order_by(VisionLLMConfig.created_at.desc())
+            .offset(skip)
+            .limit(limit)
+        )
+        return result.scalars().all()
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.exception("Failed to list VisionLLMConfigs")
+        raise HTTPException(
+            status_code=500, detail=f"Failed to fetch configs: {e!s}"
+        ) from e
+
+
+@router.get(
+    "/vision-llm-configs/{config_id}", response_model=VisionLLMConfigRead
+)
+async def get_vision_llm_config(
+    config_id: int,
+    session: AsyncSession = Depends(get_async_session),
+    user: User = Depends(current_active_user),
+):
+    try:
+        result = await session.execute(
+            select(VisionLLMConfig).filter(VisionLLMConfig.id == config_id)
+        )
+        db_config = result.scalars().first()
+        if not db_config:
+            raise HTTPException(status_code=404, detail="Config not found")
+
+        await check_permission(
+            session,
+            user,
+            db_config.search_space_id,
+            Permission.VISION_CONFIGS_READ.value,
+            "You don't have permission to view vision LLM configs in this search space",
+        )
+        return db_config
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.exception("Failed to get VisionLLMConfig")
+        raise HTTPException(
+            status_code=500, detail=f"Failed to fetch config: {e!s}"
+        ) from e
+
+
+@router.put(
+    "/vision-llm-configs/{config_id}", response_model=VisionLLMConfigRead
+)
+async def update_vision_llm_config(
+    config_id: int,
+    update_data: VisionLLMConfigUpdate,
+    session: AsyncSession = Depends(get_async_session),
+    user: User = Depends(current_active_user),
+):
+    try:
+        result = await session.execute(
+            select(VisionLLMConfig).filter(VisionLLMConfig.id == config_id)
+        )
+        db_config = result.scalars().first()
+        if not db_config:
+            raise HTTPException(status_code=404, detail="Config not found")
+
+        await check_permission(
+            session,
+            user,
+            db_config.search_space_id,
+            Permission.VISION_CONFIGS_CREATE.value,
+            "You don't have permission to update vision LLM configs in this search space",
+        )
+
+        for key, value in update_data.model_dump(exclude_unset=True).items():
+            setattr(db_config, key, value)
+
+        await session.commit()
+        await session.refresh(db_config)
+        return db_config
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        await session.rollback()
+        logger.exception("Failed to update VisionLLMConfig")
+        raise HTTPException(
+            status_code=500, detail=f"Failed to update config: {e!s}"
+        ) from e
+
+
+@router.delete("/vision-llm-configs/{config_id}", response_model=dict)
+async def delete_vision_llm_config(
+    config_id: int,
+    session: AsyncSession = Depends(get_async_session),
+    user: User = Depends(current_active_user),
+):
+    try:
+        result = await session.execute(
+            select(VisionLLMConfig).filter(VisionLLMConfig.id == config_id)
+        )
+        db_config = result.scalars().first()
+        if not db_config:
+            raise HTTPException(status_code=404, detail="Config not found")
+
+        await check_permission(
+            session,
+            user,
+            db_config.search_space_id,
+            Permission.VISION_CONFIGS_DELETE.value,
+            "You don't have permission to delete vision LLM configs in this search space",
+        )
+
+        await session.delete(db_config)
+        await session.commit()
+        return {
+            "message": "Vision LLM config deleted successfully",
+            "id": config_id,
+        }
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        await session.rollback()
+        logger.exception("Failed to delete VisionLLMConfig")
+        raise HTTPException(
+            status_code=500, detail=f"Failed to delete config: {e!s}"
+        ) from e

surfsense_backend/app/schemas/__init__.py

@@ -125,6 +125,13 @@ from .video_presentations import (
     VideoPresentationRead,
     VideoPresentationUpdate,
 )
+from .vision_llm import (
+    GlobalVisionLLMConfigRead,
+    VisionLLMConfigCreate,
+    VisionLLMConfigPublic,
+    VisionLLMConfigRead,
+    VisionLLMConfigUpdate,
+)
 
 __all__ = [
     # Folder schemas
@@ -163,6 +170,8 @@ __all__ = [
     "FolderUpdate",
     "GlobalImageGenConfigRead",
     "GlobalNewLLMConfigRead",
+    # Vision LLM Config schemas
+    "GlobalVisionLLMConfigRead",
     "GoogleDriveIndexRequest",
     "GoogleDriveIndexingOptions",
     # Base schemas
@@ -264,4 +273,8 @@ __all__ = [
     "VideoPresentationCreate",
     "VideoPresentationRead",
     "VideoPresentationUpdate",
+    "VisionLLMConfigCreate",
+    "VisionLLMConfigPublic",
+    "VisionLLMConfigRead",
+    "VisionLLMConfigUpdate",
 ]

surfsense_backend/app/schemas/new_llm_config.py

@@ -182,8 +182,8 @@ class LLMPreferencesRead(BaseModel):
     image_generation_config_id: int | None = Field(
         None, description="ID of the image generation config to use"
     )
-    vision_llm_id: int | None = Field(
-        None, description="ID of the LLM config to use for vision/screenshot analysis"
+    vision_llm_config_id: int | None = Field(
+        None, description="ID of the vision LLM config to use for vision/screenshot analysis"
     )
     agent_llm: dict[str, Any] | None = Field(
         None, description="Full config for agent LLM"
@@ -194,7 +194,7 @@ class LLMPreferencesRead(BaseModel):
     image_generation_config: dict[str, Any] | None = Field(
         None, description="Full config for image generation"
     )
-    vision_llm: dict[str, Any] | None = Field(
+    vision_llm_config: dict[str, Any] | None = Field(
         None, description="Full config for vision LLM"
     )
 
@@ -213,6 +213,6 @@ class LLMPreferencesUpdate(BaseModel):
     image_generation_config_id: int | None = Field(
         None, description="ID of the image generation config to use"
     )
-    vision_llm_id: int | None = Field(
-        None, description="ID of the LLM config to use for vision/screenshot analysis"
+    vision_llm_config_id: int | None = Field(
+        None, description="ID of the vision LLM config to use for vision/screenshot analysis"
     )

surfsense_backend/app/schemas/vision_llm.py

@@ -0,0 +1,75 @@
+import uuid
+from datetime import datetime
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from app.db import VisionProvider
+
+
+class VisionLLMConfigBase(BaseModel):
+    name: str = Field(..., max_length=100)
+    description: str | None = Field(None, max_length=500)
+    provider: VisionProvider = Field(...)
+    custom_provider: str | None = Field(None, max_length=100)
+    model_name: str = Field(..., max_length=100)
+    api_key: str = Field(...)
+    api_base: str | None = Field(None, max_length=500)
+    api_version: str | None = Field(None, max_length=50)
+    litellm_params: dict[str, Any] | None = Field(default=None)
+
+
+class VisionLLMConfigCreate(VisionLLMConfigBase):
+    search_space_id: int = Field(...)
+
+
+class VisionLLMConfigUpdate(BaseModel):
+    name: str | None = Field(None, max_length=100)
+    description: str | None = Field(None, max_length=500)
+    provider: VisionProvider | None = None
+    custom_provider: str | None = Field(None, max_length=100)
+    model_name: str | None = Field(None, max_length=100)
+    api_key: str | None = None
+    api_base: str | None = Field(None, max_length=500)
+    api_version: str | None = Field(None, max_length=50)
+    litellm_params: dict[str, Any] | None = None
+
+
+class VisionLLMConfigRead(VisionLLMConfigBase):
+    id: int
+    created_at: datetime
+    search_space_id: int
+    user_id: uuid.UUID
+
+    model_config = ConfigDict(from_attributes=True)
+
+
+class VisionLLMConfigPublic(BaseModel):
+    id: int
+    name: str
+    description: str | None = None
+    provider: VisionProvider
+    custom_provider: str | None = None
+    model_name: str
+    api_base: str | None = None
+    api_version: str | None = None
+    litellm_params: dict[str, Any] | None = None
+    created_at: datetime
+    search_space_id: int
+    user_id: uuid.UUID
+
+    model_config = ConfigDict(from_attributes=True)
+
+
+class GlobalVisionLLMConfigRead(BaseModel):
+    id: int = Field(...)
+    name: str
+    description: str | None = None
+    provider: str
+    custom_provider: str | None = None
+    model_name: str
+    api_base: str | None = None
+    api_version: str | None = None
+    litellm_params: dict[str, Any] | None = None
+    is_global: bool = True
+    is_auto_mode: bool = False

surfsense_backend/app/services/llm_service.py

@@ -32,7 +32,6 @@ logger = logging.getLogger(__name__)
 class LLMRole:
     AGENT = "agent"  # For agent/chat operations
     DOCUMENT_SUMMARY = "document_summary"  # For document summarization
-    VISION = "vision"  # For vision/screenshot analysis
 
 
 def get_global_llm_config(llm_config_id: int) -> dict | None:
@@ -188,7 +187,7 @@ async def get_search_space_llm_instance(
     Args:
         session: Database session
         search_space_id: Search Space ID
-        role: LLM role ('agent', 'document_summary', or 'vision')
+        role: LLM role ('agent' or 'document_summary')
 
     Returns:
         ChatLiteLLM or ChatLiteLLMRouter instance, or None if not found
@@ -210,8 +209,6 @@ async def get_search_space_llm_instance(
             llm_config_id = search_space.agent_llm_id
         elif role == LLMRole.DOCUMENT_SUMMARY:
             llm_config_id = search_space.document_summary_llm_id
-        elif role == LLMRole.VISION:
-            llm_config_id = search_space.vision_llm_id
         else:
             logger.error(f"Invalid LLM role: {role}")
             return None
@@ -411,8 +408,118 @@ async def get_document_summary_llm(
 async def get_vision_llm(
     session: AsyncSession, search_space_id: int
 ) -> ChatLiteLLM | ChatLiteLLMRouter | None:
-    """Get the search space's vision LLM instance for screenshot analysis."""
-    return await get_search_space_llm_instance(session, search_space_id, LLMRole.VISION)
+    """Get the search space's vision LLM instance for screenshot analysis.
+
+    Resolves from the dedicated VisionLLMConfig system:
+    - Auto mode (ID 0): VisionLLMRouterService
+    - Global (negative ID): YAML configs
+    - DB (positive ID): VisionLLMConfig table
+    """
+    from app.db import VisionLLMConfig
+    from app.services.vision_llm_router_service import (
+        VISION_PROVIDER_MAP,
+        VisionLLMRouterService,
+        get_global_vision_llm_config,
+        is_vision_auto_mode,
+    )
+
+    try:
+        result = await session.execute(
+            select(SearchSpace).where(SearchSpace.id == search_space_id)
+        )
+        search_space = result.scalars().first()
+        if not search_space:
+            logger.error(f"Search space {search_space_id} not found")
+            return None
+
+        config_id = search_space.vision_llm_config_id
+        if config_id is None:
+            logger.error(
+                f"No vision LLM configured for search space {search_space_id}"
+            )
+            return None
+
+        if is_vision_auto_mode(config_id):
+            if not VisionLLMRouterService.is_initialized():
+                logger.error(
+                    "Vision Auto mode requested but Vision LLM Router not initialized"
+                )
+                return None
+            try:
+                return ChatLiteLLMRouter(
+                    router=VisionLLMRouterService.get_router(),
+                    streaming=True,
+                )
+            except Exception as e:
+                logger.error(f"Failed to create vision ChatLiteLLMRouter: {e}")
+                return None
+
+        if config_id < 0:
+            global_cfg = get_global_vision_llm_config(config_id)
+            if not global_cfg:
+                logger.error(f"Global vision LLM config {config_id} not found")
+                return None
+
+            if global_cfg.get("custom_provider"):
+                model_string = (
+                    f"{global_cfg['custom_provider']}/{global_cfg['model_name']}"
+                )
+            else:
+                prefix = VISION_PROVIDER_MAP.get(
+                    global_cfg["provider"].upper(),
+                    global_cfg["provider"].lower(),
+                )
+                model_string = f"{prefix}/{global_cfg['model_name']}"
+
+            litellm_kwargs = {
+                "model": model_string,
+                "api_key": global_cfg["api_key"],
+            }
+            if global_cfg.get("api_base"):
+                litellm_kwargs["api_base"] = global_cfg["api_base"]
+            if global_cfg.get("litellm_params"):
+                litellm_kwargs.update(global_cfg["litellm_params"])
+
+            return ChatLiteLLM(**litellm_kwargs)
+
+        result = await session.execute(
+            select(VisionLLMConfig).where(
+                VisionLLMConfig.id == config_id,
+                VisionLLMConfig.search_space_id == search_space_id,
+            )
+        )
+        vision_cfg = result.scalars().first()
+        if not vision_cfg:
+            logger.error(
+                f"Vision LLM config {config_id} not found in search space {search_space_id}"
+            )
+            return None
+
+        if vision_cfg.custom_provider:
+            model_string = f"{vision_cfg.custom_provider}/{vision_cfg.model_name}"
+        else:
+            prefix = VISION_PROVIDER_MAP.get(
+                vision_cfg.provider.value.upper(),
+                vision_cfg.provider.value.lower(),
+            )
+            model_string = f"{prefix}/{vision_cfg.model_name}"
+
+        litellm_kwargs = {
+            "model": model_string,
+            "api_key": vision_cfg.api_key,
+        }
+        if vision_cfg.api_base:
+            litellm_kwargs["api_base"] = vision_cfg.api_base
+        if vision_cfg.litellm_params:
+            litellm_kwargs.update(vision_cfg.litellm_params)
+
+        return ChatLiteLLM(**litellm_kwargs)
+
+    except Exception as e:
+        logger.error(
+            f"Error getting vision LLM for search space {search_space_id}: {e!s}"
+        )
+        return None
 
 
 # Backward-compatible alias (LLM preferences are now per-search-space, not per-user)

surfsense_backend/app/services/vision_autocomplete_service.py

@@ -1,149 +1,40 @@
+"""Vision autocomplete service — agent-based with scoped filesystem.
+
+Optimized pipeline:
+1. Start the SSE stream immediately so the UI shows progress.
+2. Derive a KB search query from window_title (no separate LLM call).
+3. Run KB filesystem pre-computation and agent graph compilation in PARALLEL.
+4. Inject pre-computed KB files as initial state and stream the agent.
+"""
+
 import logging
 from collections.abc import AsyncGenerator
 
-from langchain_core.messages import HumanMessage, SystemMessage
+from langchain_core.messages import HumanMessage
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from app.retriever.chunks_hybrid_search import ChucksHybridSearchRetriever
+from app.agents.autocomplete import create_autocomplete_agent, stream_autocomplete_agent
 from app.services.llm_service import get_vision_llm
 from app.services.new_streaming_service import VercelStreamingService
 
 logger = logging.getLogger(__name__)
 
-KB_TOP_K = 5
-KB_MAX_CHARS = 4000
-
-EXTRACT_QUERY_PROMPT = """Look at this screenshot and describe in 1-2 short sentences what the user is working on and what topic they need to write about. Be specific about the subject matter. Output ONLY the description, nothing else."""
-
-EXTRACT_QUERY_PROMPT_WITH_APP = """The user is currently in the application "{app_name}" with the window titled "{window_title}".
-
-Look at this screenshot and describe in 1-2 short sentences what the user is working on and what topic they need to write about. Be specific about the subject matter. Output ONLY the description, nothing else."""
-
-VISION_SYSTEM_PROMPT = """You are a smart writing assistant that analyzes the user's screen to draft or complete text.
-
-You will receive a screenshot of the user's screen. Your job:
-1. Analyze the ENTIRE screenshot to understand what the user is working on (email thread, chat conversation, document, code editor, form, etc.).
-2. Identify the text area where the user will type.
-3. Based on the full visual context, generate the text the user most likely wants to write.
-
-Key behavior:
-- If the text area is EMPTY, draft a full response or message based on what you see on screen (e.g., reply to an email, respond to a chat message, continue a document).
-- If the text area already has text, continue it naturally.
-
-Rules:
-- Output ONLY the text to be inserted. No quotes, no explanations, no meta-commentary.
-- Be concise but complete — a full thought, not a fragment.
-- Match the tone and formality of the surrounding context.
-- If the screen shows code, write code. If it shows a casual chat, be casual. If it shows a formal email, be formal.
-- Do NOT describe the screenshot or explain your reasoning.
-- If you cannot determine what to write, output nothing."""
-
-APP_CONTEXT_BLOCK = """
-
-The user is currently working in "{app_name}" (window: "{window_title}"). Use this to understand the type of application and adapt your tone and format accordingly."""
-
-KB_CONTEXT_BLOCK = """
-
-You also have access to the user's knowledge base documents below. Use them to write more accurate, informed, and contextually relevant text. Do NOT cite or reference the documents explicitly — just let the knowledge inform your writing naturally.
-
-<knowledge_base>
-{kb_context}
-</knowledge_base>"""
+PREP_STEP_ID = "autocomplete-prep"
 
 
-def _build_system_prompt(app_name: str, window_title: str, kb_context: str) -> str:
-    """Assemble the system prompt from optional context blocks."""
-    prompt = VISION_SYSTEM_PROMPT
-    if app_name:
-        prompt += APP_CONTEXT_BLOCK.format(app_name=app_name, window_title=window_title)
-    if kb_context:
-        prompt += KB_CONTEXT_BLOCK.format(kb_context=kb_context)
-    return prompt
+def _derive_kb_query(app_name: str, window_title: str) -> str:
+    parts = [p for p in (window_title, app_name) if p]
+    return " ".join(parts)
 
 
 def _is_vision_unsupported_error(e: Exception) -> bool:
-    """Check if an exception indicates the model doesn't support vision/images."""
     msg = str(e).lower()
     return "content must be a string" in msg or "does not support image" in msg
 
 
-async def _extract_query_from_screenshot(
-    llm,
-    screenshot_data_url: str,
-    app_name: str = "",
-    window_title: str = "",
-) -> str | None:
-    """Ask the Vision LLM to describe what the user is working on.
-
-    Raises vision-unsupported errors so the caller can return a
-    friendly message immediately instead of retrying with astream.
-    """
-    if app_name:
-        prompt_text = EXTRACT_QUERY_PROMPT_WITH_APP.format(
-            app_name=app_name,
-            window_title=window_title,
-        )
-    else:
-        prompt_text = EXTRACT_QUERY_PROMPT
-
-    try:
-        response = await llm.ainvoke(
-            [
-                HumanMessage(
-                    content=[
-                        {"type": "text", "text": prompt_text},
-                        {
-                            "type": "image_url",
-                            "image_url": {"url": screenshot_data_url},
-                        },
-                    ]
-                ),
-            ]
-        )
-        query = response.content.strip() if hasattr(response, "content") else ""
-        return query if query else None
-    except Exception as e:
-        if _is_vision_unsupported_error(e):
-            raise
-        logger.warning(f"Failed to extract query from screenshot: {e}")
-        return None
-
-
-async def _search_knowledge_base(
-    session: AsyncSession, search_space_id: int, query: str
-) -> str:
-    """Search the KB and return formatted context string."""
-    try:
-        retriever = ChucksHybridSearchRetriever(session)
-        results = await retriever.hybrid_search(
-            query_text=query,
-            top_k=KB_TOP_K,
-            search_space_id=search_space_id,
-        )
-
-        if not results:
-            return ""
-
-        parts: list[str] = []
-        char_count = 0
-        for doc in results:
-            title = doc.get("document", {}).get("title", "Untitled")
-            for chunk in doc.get("chunks", []):
-                content = chunk.get("content", "").strip()
-                if not content:
-                    continue
-                entry = f"[{title}]\n{content}"
-                if char_count + len(entry) > KB_MAX_CHARS:
-                    break
-                parts.append(entry)
-                char_count += len(entry)
-            if char_count >= KB_MAX_CHARS:
-                break
-
-        return "\n\n---\n\n".join(parts)
-    except Exception as e:
-        logger.warning(f"KB search failed, proceeding without context: {e}")
-        return ""
+# ---------------------------------------------------------------------------
+# Main entry point
+# ---------------------------------------------------------------------------
 
 
 async def stream_vision_autocomplete(
@@ -154,13 +45,7 @@ async def stream_vision_autocomplete(
     app_name: str = "",
     window_title: str = "",
 ) -> AsyncGenerator[str, None]:
-    """Analyze a screenshot with the vision LLM and stream a text completion.
-
-    Pipeline:
-    1. Extract a search query from the screenshot (non-streaming)
-    2. Search the knowledge base for relevant context
-    3. Stream the final completion with screenshot + KB + app context
-    """
+    """Analyze a screenshot with a vision-LLM agent and stream a text completion."""
     streaming = VercelStreamingService()
     vision_error_msg = (
         "The selected model does not support vision. "
@@ -174,71 +59,100 @@ async def stream_vision_autocomplete(
         yield streaming.format_done()
         return
 
-    kb_context = ""
+    # Start SSE stream immediately so the UI has something to show
+    yield streaming.format_message_start()
+
+    kb_query = _derive_kb_query(app_name, window_title)
+
+    # Show a preparation step while KB search + agent compile run
+    yield streaming.format_thinking_step(
+        step_id=PREP_STEP_ID,
+        title="Searching knowledge base",
+        status="in_progress",
+        items=[kb_query] if kb_query else [],
+    )
+
     try:
-        query = await _extract_query_from_screenshot(
+        agent, kb = await create_autocomplete_agent(
             llm,
-            screenshot_data_url,
+            search_space_id=search_space_id,
+            kb_query=kb_query,
             app_name=app_name,
             window_title=window_title,
         )
     except Exception as e:
-        logger.warning(
-            f"Vision autocomplete: selected model does not support vision: {e}"
-        )
-        yield streaming.format_message_start()
-        yield streaming.format_error(vision_error_msg)
+        if _is_vision_unsupported_error(e):
+            logger.warning("Vision autocomplete: model does not support vision: %s", e)
+            yield streaming.format_error(vision_error_msg)
+            yield streaming.format_done()
+            return
+        logger.error("Failed to create autocomplete agent: %s", e, exc_info=True)
+        yield streaming.format_error("Autocomplete failed. Please try again.")
         yield streaming.format_done()
         return
 
-    if query:
-        kb_context = await _search_knowledge_base(session, search_space_id, query)
+    has_kb = kb.has_documents
+    doc_count = len(kb.files) if has_kb else 0  # type: ignore[arg-type]
 
-    system_prompt = _build_system_prompt(app_name, window_title, kb_context)
+    yield streaming.format_thinking_step(
+        step_id=PREP_STEP_ID,
+        title="Searching knowledge base",
+        status="complete",
+        items=[f"Found {doc_count} document{'s' if doc_count != 1 else ''}"]
+        if kb_query
+        else ["Skipped"],
+    )
 
-    messages = [
-        SystemMessage(content=system_prompt),
-        HumanMessage(
-            content=[
-                {
-                    "type": "text",
-                    "text": "Analyze this screenshot. Understand the full context of what the user is working on, then generate the text they most likely want to write in the active text area.",
-                },
-                {
-                    "type": "image_url",
-                    "image_url": {"url": screenshot_data_url},
-                },
-            ]
-        ),
-    ]
+    # Build agent input with pre-computed KB as initial state
+    if has_kb:
+        instruction = (
+            "Analyze this screenshot, then explore the knowledge base documents "
+            "listed above — read the chunk index of any document whose title "
+            "looks relevant and check matched chunks for useful facts. "
+            "Finally, generate a concise autocomplete for the active text area, "
+            "enhanced with any relevant KB information you found."
+        )
+    else:
+        instruction = (
+            "Analyze this screenshot and generate a concise autocomplete "
+            "for the active text area based on what you see."
+        )
 
-    text_started = False
-    text_id = ""
+    user_message = HumanMessage(
+        content=[
+            {"type": "text", "text": instruction},
+            {"type": "image_url", "image_url": {"url": screenshot_data_url}},
+        ]
+    )
+
+    input_data: dict = {"messages": [user_message]}
+
+    if has_kb:
+        input_data["files"] = kb.files
+        input_data["messages"] = [kb.ls_ai_msg, kb.ls_tool_msg, user_message]
+        logger.info(
+            "Autocomplete: injected %d KB files into agent initial state", doc_count
+        )
+    else:
+        logger.info(
+            "Autocomplete: no KB documents found, proceeding with screenshot only"
+        )
+
+    # Stream the agent (message_start already sent above)
     try:
-        yield streaming.format_message_start()
-        text_id = streaming.generate_text_id()
-        yield streaming.format_text_start(text_id)
-        text_started = True
-
-        async for chunk in llm.astream(messages):
-            token = chunk.content if hasattr(chunk, "content") else str(chunk)
-            if token:
-                yield streaming.format_text_delta(text_id, token)
-
-        yield streaming.format_text_end(text_id)
-        yield streaming.format_finish()
-        yield streaming.format_done()
-
+        async for sse in stream_autocomplete_agent(
+            agent,
+            input_data,
+            streaming,
+            emit_message_start=False,
+        ):
+            yield sse
     except Exception as e:
-        if text_started:
-            yield streaming.format_text_end(text_id)
-
         if _is_vision_unsupported_error(e):
-            logger.warning(
-                f"Vision autocomplete: selected model does not support vision: {e}"
-            )
+            logger.warning("Vision autocomplete: model does not support vision: %s", e)
             yield streaming.format_error(vision_error_msg)
+            yield streaming.format_done()
         else:
-            logger.error(f"Vision autocomplete streaming error: {e}", exc_info=True)
+            logger.error("Vision autocomplete streaming error: %s", e, exc_info=True)
             yield streaming.format_error("Autocomplete failed. Please try again.")
-        yield streaming.format_done()
+            yield streaming.format_done()

surfsense_backend/app/services/vision_llm_router_service.py

@@ -0,0 +1,193 @@
+import logging
+from typing import Any
+
+from litellm import Router
+
+logger = logging.getLogger(__name__)
+
+VISION_AUTO_MODE_ID = 0
+
+VISION_PROVIDER_MAP = {
+    "OPENAI": "openai",
+    "ANTHROPIC": "anthropic",
+    "GOOGLE": "gemini",
+    "AZURE_OPENAI": "azure",
+    "VERTEX_AI": "vertex_ai",
+    "BEDROCK": "bedrock",
+    "XAI": "xai",
+    "OPENROUTER": "openrouter",
+    "OLLAMA": "ollama_chat",
+    "GROQ": "groq",
+    "TOGETHER_AI": "together_ai",
+    "FIREWORKS_AI": "fireworks_ai",
+    "DEEPSEEK": "openai",
+    "MISTRAL": "mistral",
+    "CUSTOM": "custom",
+}
+
+
+class VisionLLMRouterService:
+    _instance = None
+    _router: Router | None = None
+    _model_list: list[dict] = []
+    _router_settings: dict = {}
+    _initialized: bool = False
+
+    def __new__(cls):
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+        return cls._instance
+
+    @classmethod
+    def get_instance(cls) -> "VisionLLMRouterService":
+        if cls._instance is None:
+            cls._instance = cls()
+        return cls._instance
+
+    @classmethod
+    def initialize(
+        cls,
+        global_configs: list[dict],
+        router_settings: dict | None = None,
+    ) -> None:
+        instance = cls.get_instance()
+
+        if instance._initialized:
+            logger.debug("Vision LLM Router already initialized, skipping")
+            return
+
+        model_list = []
+        for config in global_configs:
+            deployment = cls._config_to_deployment(config)
+            if deployment:
+                model_list.append(deployment)
+
+        if not model_list:
+            logger.warning(
+                "No valid vision LLM configs found for router initialization"
+            )
+            return
+
+        instance._model_list = model_list
+        instance._router_settings = router_settings or {}
+
+        default_settings = {
+            "routing_strategy": "usage-based-routing",
+            "num_retries": 3,
+            "allowed_fails": 3,
+            "cooldown_time": 60,
+            "retry_after": 5,
+        }
+
+        final_settings = {**default_settings, **instance._router_settings}
+
+        try:
+            instance._router = Router(
+                model_list=model_list,
+                routing_strategy=final_settings.get(
+                    "routing_strategy", "usage-based-routing"
+                ),
+                num_retries=final_settings.get("num_retries", 3),
+                allowed_fails=final_settings.get("allowed_fails", 3),
+                cooldown_time=final_settings.get("cooldown_time", 60),
+                set_verbose=False,
+            )
+            instance._initialized = True
+            logger.info(
+                "Vision LLM Router initialized with %d deployments, strategy: %s",
+                len(model_list),
+                final_settings.get("routing_strategy"),
+            )
+        except Exception as e:
+            logger.error(f"Failed to initialize Vision LLM Router: {e}")
+            instance._router = None
+
+    @classmethod
+    def _config_to_deployment(cls, config: dict) -> dict | None:
+        try:
+            if not config.get("model_name") or not config.get("api_key"):
+                return None
+
+            if config.get("custom_provider"):
+                model_string = f"{config['custom_provider']}/{config['model_name']}"
+            else:
+                provider = config.get("provider", "").upper()
+                provider_prefix = VISION_PROVIDER_MAP.get(provider, provider.lower())
+                model_string = f"{provider_prefix}/{config['model_name']}"
+
+            litellm_params: dict[str, Any] = {
+                "model": model_string,
+                "api_key": config.get("api_key"),
+            }
+
+            if config.get("api_base"):
+                litellm_params["api_base"] = config["api_base"]
+
+            if config.get("api_version"):
+                litellm_params["api_version"] = config["api_version"]
+
+            if config.get("litellm_params"):
+                litellm_params.update(config["litellm_params"])
+
+            deployment: dict[str, Any] = {
+                "model_name": "auto",
+                "litellm_params": litellm_params,
+            }
+
+            if config.get("rpm"):
+                deployment["rpm"] = config["rpm"]
+            if config.get("tpm"):
+                deployment["tpm"] = config["tpm"]
+
+            return deployment
+
+        except Exception as e:
+            logger.warning(f"Failed to convert vision config to deployment: {e}")
+            return None
+
+    @classmethod
+    def get_router(cls) -> Router | None:
+        instance = cls.get_instance()
+        return instance._router
+
+    @classmethod
+    def is_initialized(cls) -> bool:
+        instance = cls.get_instance()
+        return instance._initialized and instance._router is not None
+
+    @classmethod
+    def get_model_count(cls) -> int:
+        instance = cls.get_instance()
+        return len(instance._model_list)
+
+
+def is_vision_auto_mode(config_id: int | None) -> bool:
+    return config_id == VISION_AUTO_MODE_ID
+
+
+def build_vision_model_string(
+    provider: str, model_name: str, custom_provider: str | None
+) -> str:
+    if custom_provider:
+        return f"{custom_provider}/{model_name}"
+    prefix = VISION_PROVIDER_MAP.get(provider.upper(), provider.lower())
+    return f"{prefix}/{model_name}"
+
+
+def get_global_vision_llm_config(config_id: int) -> dict | None:
+    from app.config import config
+
+    if config_id == VISION_AUTO_MODE_ID:
+        return {
+            "id": VISION_AUTO_MODE_ID,
+            "name": "Auto (Fastest)",
+            "provider": "AUTO",
+            "model_name": "auto",
+            "is_auto_mode": True,
+        }
+    if config_id > 0:
+        return None
+    for cfg in config.GLOBAL_VISION_LLM_CONFIGS:
+        if cfg.get("id") == config_id:
+            return cfg
+    return None

surfsense_backend/app/services/vision_model_list_service.py

@@ -0,0 +1,132 @@
+"""
+Service for fetching and caching the vision-capable model list.
+
+Reuses the same OpenRouter public API and local fallback as the LLM model
+list service, but filters for models that accept image input.
+"""
+
+import json
+import logging
+import time
+from pathlib import Path
+
+import httpx
+
+logger = logging.getLogger(__name__)
+
+OPENROUTER_API_URL = "https://openrouter.ai/api/v1/models"
+FALLBACK_FILE = Path(__file__).parent.parent / "config" / "vision_model_list_fallback.json"
+CACHE_TTL_SECONDS = 86400  # 24 hours
+
+_cache: list[dict] | None = None
+_cache_timestamp: float = 0
+
+OPENROUTER_SLUG_TO_VISION_PROVIDER: dict[str, str] = {
+    "openai": "OPENAI",
+    "anthropic": "ANTHROPIC",
+    "google": "GOOGLE",
+    "mistralai": "MISTRAL",
+    "x-ai": "XAI",
+}
+
+
+def _format_context_length(length: int | None) -> str | None:
+    if not length:
+        return None
+    if length >= 1_000_000:
+        return f"{length / 1_000_000:g}M"
+    if length >= 1_000:
+        return f"{length / 1_000:g}K"
+    return str(length)
+
+
+async def _fetch_from_openrouter() -> list[dict] | None:
+    try:
+        async with httpx.AsyncClient(timeout=15) as client:
+            response = await client.get(OPENROUTER_API_URL)
+            response.raise_for_status()
+            data = response.json()
+            return data.get("data", [])
+    except Exception as e:
+        logger.warning("Failed to fetch from OpenRouter API for vision models: %s", e)
+        return None
+
+
+def _load_fallback() -> list[dict]:
+    try:
+        with open(FALLBACK_FILE, encoding="utf-8") as f:
+            return json.load(f)
+    except Exception as e:
+        logger.error("Failed to load vision model fallback list: %s", e)
+        return []
+
+
+def _is_vision_model(model: dict) -> bool:
+    """Return True if the model accepts image input and outputs text."""
+    arch = model.get("architecture", {})
+    input_mods = arch.get("input_modalities", [])
+    output_mods = arch.get("output_modalities", [])
+    return "image" in input_mods and "text" in output_mods
+
+
+def _process_vision_models(raw_models: list[dict]) -> list[dict]:
+    processed: list[dict] = []
+
+    for model in raw_models:
+        model_id: str = model.get("id", "")
+        name: str = model.get("name", "")
+        context_length = model.get("context_length")
+
+        if "/" not in model_id:
+            continue
+
+        if not _is_vision_model(model):
+            continue
+
+        provider_slug, model_name = model_id.split("/", 1)
+        context_window = _format_context_length(context_length)
+
+        processed.append(
+            {
+                "value": model_id,
+                "label": name,
+                "provider": "OPENROUTER",
+                "context_window": context_window,
+            }
+        )
+
+        native_provider = OPENROUTER_SLUG_TO_VISION_PROVIDER.get(provider_slug)
+        if native_provider:
+            if native_provider == "GOOGLE" and not model_name.startswith("gemini-"):
+                continue
+
+            processed.append(
+                {
+                    "value": model_name,
+                    "label": name,
+                    "provider": native_provider,
+                    "context_window": context_window,
+                }
+            )
+
+    return processed
+
+
+async def get_vision_model_list() -> list[dict]:
+    global _cache, _cache_timestamp
+
+    if _cache is not None and (time.time() - _cache_timestamp) < CACHE_TTL_SECONDS:
+        return _cache
+
+    raw_models = await _fetch_from_openrouter()
+
+    if raw_models is None:
+        logger.info("Using fallback vision model list")
+        return _load_fallback()
+
+    processed = _process_vision_models(raw_models)
+
+    _cache = processed
+    _cache_timestamp = time.time()
+
+    return processed