diff --git a/surfsense_backend/app/routes/__init__.py b/surfsense_backend/app/routes/__init__.py index 22631bc1d..02367606b 100644 --- a/surfsense_backend/app/routes/__init__.py +++ b/surfsense_backend/app/routes/__init__.py @@ -49,6 +49,7 @@ from .stripe_routes import router as stripe_router from .surfsense_docs_routes import router as surfsense_docs_router from .teams_add_connector_route import router as teams_add_connector_router from .video_presentations_routes import router as video_presentations_router +from .vision_llm_routes import router as vision_llm_router from .youtube_routes import router as youtube_router router = APIRouter() @@ -68,6 +69,7 @@ router.include_router( ) # Video presentation status and streaming router.include_router(reports_router) # Report CRUD and multi-format export router.include_router(image_generation_router) # Image generation via litellm +router.include_router(vision_llm_router) # Vision LLM configs for screenshot analysis router.include_router(search_source_connectors_router) router.include_router(google_calendar_add_connector_router) router.include_router(google_gmail_add_connector_router) diff --git a/surfsense_backend/app/routes/search_spaces_routes.py b/surfsense_backend/app/routes/search_spaces_routes.py index c4f1ab035..78be97aa1 100644 --- a/surfsense_backend/app/routes/search_spaces_routes.py +++ b/surfsense_backend/app/routes/search_spaces_routes.py @@ -14,6 +14,7 @@ from app.db import ( SearchSpaceMembership, SearchSpaceRole, User, + VisionLLMConfig, get_async_session, get_default_roles_config, ) @@ -483,6 +484,63 @@ async def _get_image_gen_config_by_id( return None +async def _get_vision_llm_config_by_id( + session: AsyncSession, config_id: int | None +) -> dict | None: + if config_id is None: + return None + + if config_id == 0: + return { + "id": 0, + "name": "Auto (Fastest)", + "description": "Automatically routes requests across available vision LLM providers", + "provider": "AUTO", + "model_name": "auto", + "is_global": True, + "is_auto_mode": True, + } + + if config_id < 0: + for cfg in config.GLOBAL_VISION_LLM_CONFIGS: + if cfg.get("id") == config_id: + return { + "id": cfg.get("id"), + "name": cfg.get("name"), + "description": cfg.get("description"), + "provider": cfg.get("provider"), + "custom_provider": cfg.get("custom_provider"), + "model_name": cfg.get("model_name"), + "api_base": cfg.get("api_base") or None, + "api_version": cfg.get("api_version") or None, + "litellm_params": cfg.get("litellm_params", {}), + "is_global": True, + } + return None + + result = await session.execute( + select(VisionLLMConfig).filter(VisionLLMConfig.id == config_id) + ) + db_config = result.scalars().first() + if db_config: + return { + "id": db_config.id, + "name": db_config.name, + "description": db_config.description, + "provider": db_config.provider.value if db_config.provider else None, + "custom_provider": db_config.custom_provider, + "model_name": db_config.model_name, + "api_base": db_config.api_base, + "api_version": db_config.api_version, + "litellm_params": db_config.litellm_params or {}, + "created_at": db_config.created_at.isoformat() + if db_config.created_at + else None, + "search_space_id": db_config.search_space_id, + } + return None + + @router.get( "/search-spaces/{search_space_id}/llm-preferences", response_model=LLMPreferencesRead, @@ -522,17 +580,19 @@ async def get_llm_preferences( image_generation_config = await _get_image_gen_config_by_id( session, search_space.image_generation_config_id ) - vision_llm = await _get_llm_config_by_id(session, search_space.vision_llm_id) + vision_llm_config = await _get_vision_llm_config_by_id( + session, search_space.vision_llm_config_id + ) return LLMPreferencesRead( agent_llm_id=search_space.agent_llm_id, document_summary_llm_id=search_space.document_summary_llm_id, image_generation_config_id=search_space.image_generation_config_id, - vision_llm_id=search_space.vision_llm_id, + vision_llm_config_id=search_space.vision_llm_config_id, agent_llm=agent_llm, document_summary_llm=document_summary_llm, image_generation_config=image_generation_config, - vision_llm=vision_llm, + vision_llm_config=vision_llm_config, ) except HTTPException: @@ -592,17 +652,19 @@ async def update_llm_preferences( image_generation_config = await _get_image_gen_config_by_id( session, search_space.image_generation_config_id ) - vision_llm = await _get_llm_config_by_id(session, search_space.vision_llm_id) + vision_llm_config = await _get_vision_llm_config_by_id( + session, search_space.vision_llm_config_id + ) return LLMPreferencesRead( agent_llm_id=search_space.agent_llm_id, document_summary_llm_id=search_space.document_summary_llm_id, image_generation_config_id=search_space.image_generation_config_id, - vision_llm_id=search_space.vision_llm_id, + vision_llm_config_id=search_space.vision_llm_config_id, agent_llm=agent_llm, document_summary_llm=document_summary_llm, image_generation_config=image_generation_config, - vision_llm=vision_llm, + vision_llm_config=vision_llm_config, ) except HTTPException: diff --git a/surfsense_backend/app/schemas/new_llm_config.py b/surfsense_backend/app/schemas/new_llm_config.py index 6c76ca512..a466f2c99 100644 --- a/surfsense_backend/app/schemas/new_llm_config.py +++ b/surfsense_backend/app/schemas/new_llm_config.py @@ -182,8 +182,8 @@ class LLMPreferencesRead(BaseModel): image_generation_config_id: int | None = Field( None, description="ID of the image generation config to use" ) - vision_llm_id: int | None = Field( - None, description="ID of the LLM config to use for vision/screenshot analysis" + vision_llm_config_id: int | None = Field( + None, description="ID of the vision LLM config to use for vision/screenshot analysis" ) agent_llm: dict[str, Any] | None = Field( None, description="Full config for agent LLM" @@ -194,7 +194,7 @@ class LLMPreferencesRead(BaseModel): image_generation_config: dict[str, Any] | None = Field( None, description="Full config for image generation" ) - vision_llm: dict[str, Any] | None = Field( + vision_llm_config: dict[str, Any] | None = Field( None, description="Full config for vision LLM" ) @@ -213,6 +213,6 @@ class LLMPreferencesUpdate(BaseModel): image_generation_config_id: int | None = Field( None, description="ID of the image generation config to use" ) - vision_llm_id: int | None = Field( - None, description="ID of the LLM config to use for vision/screenshot analysis" + vision_llm_config_id: int | None = Field( + None, description="ID of the vision LLM config to use for vision/screenshot analysis" )