From 3369b8a83210e9afc62f9c0db91d87a13fb0c478 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Tue, 7 Apr 2026 19:24:43 +0200 Subject: [PATCH] Add frontend vision LLM config types, API, atoms, and role manager wiring --- .../vision-llm-config-mutation.atoms.ts | 84 +++++++++++++++ .../vision-llm-config-query.atoms.ts | 27 +++++ .../components/settings/llm-role-manager.tsx | 66 +++++++++--- .../contracts/enums/vision-providers.ts | 102 ++++++++++++++++++ .../contracts/types/new-llm-config.types.ts | 99 +++++++++++++++-- .../lib/apis/vision-llm-config-api.service.ts | 58 ++++++++++ surfsense_web/lib/query-client/cache-keys.ts | 5 + 7 files changed, 421 insertions(+), 20 deletions(-) create mode 100644 surfsense_web/atoms/vision-llm-config/vision-llm-config-mutation.atoms.ts create mode 100644 surfsense_web/atoms/vision-llm-config/vision-llm-config-query.atoms.ts create mode 100644 surfsense_web/contracts/enums/vision-providers.ts create mode 100644 surfsense_web/lib/apis/vision-llm-config-api.service.ts diff --git a/surfsense_web/atoms/vision-llm-config/vision-llm-config-mutation.atoms.ts b/surfsense_web/atoms/vision-llm-config/vision-llm-config-mutation.atoms.ts new file mode 100644 index 000000000..b1aa01c6b --- /dev/null +++ b/surfsense_web/atoms/vision-llm-config/vision-llm-config-mutation.atoms.ts @@ -0,0 +1,84 @@ +import { atomWithMutation } from "jotai-tanstack-query"; +import { toast } from "sonner"; +import type { + CreateVisionLLMConfigRequest, + CreateVisionLLMConfigResponse, + DeleteVisionLLMConfigResponse, + GetVisionLLMConfigsResponse, + UpdateVisionLLMConfigRequest, + UpdateVisionLLMConfigResponse, +} from "@/contracts/types/new-llm-config.types"; +import { visionLLMConfigApiService } from "@/lib/apis/vision-llm-config-api.service"; +import { cacheKeys } from "@/lib/query-client/cache-keys"; +import { queryClient } from "@/lib/query-client/client"; +import { activeSearchSpaceIdAtom } from "../search-spaces/search-space-query.atoms"; + +export const createVisionLLMConfigMutationAtom = atomWithMutation((get) => { + const searchSpaceId = get(activeSearchSpaceIdAtom); + + return { + mutationKey: ["vision-llm-configs", "create"], + enabled: !!searchSpaceId, + mutationFn: async (request: CreateVisionLLMConfigRequest) => { + return visionLLMConfigApiService.createConfig(request); + }, + onSuccess: (_: CreateVisionLLMConfigResponse, request: CreateVisionLLMConfigRequest) => { + toast.success(`${request.name} created`); + queryClient.invalidateQueries({ + queryKey: cacheKeys.visionLLMConfigs.all(Number(searchSpaceId)), + }); + }, + onError: (error: Error) => { + toast.error(error.message || "Failed to create vision model"); + }, + }; +}); + +export const updateVisionLLMConfigMutationAtom = atomWithMutation((get) => { + const searchSpaceId = get(activeSearchSpaceIdAtom); + + return { + mutationKey: ["vision-llm-configs", "update"], + enabled: !!searchSpaceId, + mutationFn: async (request: UpdateVisionLLMConfigRequest) => { + return visionLLMConfigApiService.updateConfig(request); + }, + onSuccess: (_: UpdateVisionLLMConfigResponse, request: UpdateVisionLLMConfigRequest) => { + toast.success(`${request.data.name ?? "Configuration"} updated`); + queryClient.invalidateQueries({ + queryKey: cacheKeys.visionLLMConfigs.all(Number(searchSpaceId)), + }); + queryClient.invalidateQueries({ + queryKey: cacheKeys.visionLLMConfigs.byId(request.id), + }); + }, + onError: (error: Error) => { + toast.error(error.message || "Failed to update vision model"); + }, + }; +}); + +export const deleteVisionLLMConfigMutationAtom = atomWithMutation((get) => { + const searchSpaceId = get(activeSearchSpaceIdAtom); + + return { + mutationKey: ["vision-llm-configs", "delete"], + enabled: !!searchSpaceId, + mutationFn: async (request: { id: number; name: string }) => { + return visionLLMConfigApiService.deleteConfig(request.id); + }, + onSuccess: (_: DeleteVisionLLMConfigResponse, request: { id: number; name: string }) => { + toast.success(`${request.name} deleted`); + queryClient.setQueryData( + cacheKeys.visionLLMConfigs.all(Number(searchSpaceId)), + (oldData: GetVisionLLMConfigsResponse | undefined) => { + if (!oldData) return oldData; + return oldData.filter((config) => config.id !== request.id); + } + ); + }, + onError: (error: Error) => { + toast.error(error.message || "Failed to delete vision model"); + }, + }; +}); diff --git a/surfsense_web/atoms/vision-llm-config/vision-llm-config-query.atoms.ts b/surfsense_web/atoms/vision-llm-config/vision-llm-config-query.atoms.ts new file mode 100644 index 000000000..53264fb24 --- /dev/null +++ b/surfsense_web/atoms/vision-llm-config/vision-llm-config-query.atoms.ts @@ -0,0 +1,27 @@ +import { atomWithQuery } from "jotai-tanstack-query"; +import { visionLLMConfigApiService } from "@/lib/apis/vision-llm-config-api.service"; +import { cacheKeys } from "@/lib/query-client/cache-keys"; +import { activeSearchSpaceIdAtom } from "../search-spaces/search-space-query.atoms"; + +export const visionLLMConfigsAtom = atomWithQuery((get) => { + const searchSpaceId = get(activeSearchSpaceIdAtom); + + return { + queryKey: cacheKeys.visionLLMConfigs.all(Number(searchSpaceId)), + enabled: !!searchSpaceId, + staleTime: 5 * 60 * 1000, + queryFn: async () => { + return visionLLMConfigApiService.getConfigs(Number(searchSpaceId)); + }, + }; +}); + +export const globalVisionLLMConfigsAtom = atomWithQuery(() => { + return { + queryKey: cacheKeys.visionLLMConfigs.global(), + staleTime: 10 * 60 * 1000, + queryFn: async () => { + return visionLLMConfigApiService.getGlobalConfigs(); + }, + }; +}); diff --git a/surfsense_web/components/settings/llm-role-manager.tsx b/surfsense_web/components/settings/llm-role-manager.tsx index 386845d7d..995159d58 100644 --- a/surfsense_web/components/settings/llm-role-manager.tsx +++ b/surfsense_web/components/settings/llm-role-manager.tsx @@ -24,6 +24,10 @@ import { llmPreferencesAtom, newLLMConfigsAtom, } from "@/atoms/new-llm-config/new-llm-config-query.atoms"; +import { + globalVisionLLMConfigsAtom, + visionLLMConfigsAtom, +} from "@/atoms/vision-llm-config/vision-llm-config-query.atoms"; import { Alert, AlertDescription } from "@/components/ui/alert"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; @@ -77,8 +81,8 @@ const ROLE_DESCRIPTIONS = { description: "Vision-capable model for screenshot analysis and context extraction", color: "text-amber-600 dark:text-amber-400", bgColor: "bg-amber-500/10", - prefKey: "vision_llm_id" as const, - configType: "llm" as const, + prefKey: "vision_llm_config_id" as const, + configType: "vision" as const, }, }; @@ -112,6 +116,18 @@ export function LLMRoleManager({ searchSpaceId }: LLMRoleManagerProps) { error: globalImageConfigsError, } = useAtomValue(globalImageGenConfigsAtom); + // Vision LLM configs + const { + data: userVisionConfigs = [], + isFetching: visionConfigsLoading, + error: visionConfigsError, + } = useAtomValue(visionLLMConfigsAtom); + const { + data: globalVisionConfigs = [], + isFetching: globalVisionConfigsLoading, + error: globalVisionConfigsError, + } = useAtomValue(globalVisionLLMConfigsAtom); + // Preferences const { data: preferences = {}, @@ -125,7 +141,7 @@ export function LLMRoleManager({ searchSpaceId }: LLMRoleManagerProps) { agent_llm_id: preferences.agent_llm_id ?? "", document_summary_llm_id: preferences.document_summary_llm_id ?? "", image_generation_config_id: preferences.image_generation_config_id ?? "", - vision_llm_id: preferences.vision_llm_id ?? "", + vision_llm_config_id: preferences.vision_llm_config_id ?? "", })); const [savingRole, setSavingRole] = useState(null); @@ -137,14 +153,14 @@ export function LLMRoleManager({ searchSpaceId }: LLMRoleManagerProps) { agent_llm_id: preferences.agent_llm_id ?? "", document_summary_llm_id: preferences.document_summary_llm_id ?? "", image_generation_config_id: preferences.image_generation_config_id ?? "", - vision_llm_id: preferences.vision_llm_id ?? "", + vision_llm_config_id: preferences.vision_llm_config_id ?? "", }); } }, [ preferences?.agent_llm_id, preferences?.document_summary_llm_id, preferences?.image_generation_config_id, - preferences?.vision_llm_id, + preferences?.vision_llm_config_id, ]); const handleRoleAssignment = useCallback( @@ -181,6 +197,14 @@ export function LLMRoleManager({ searchSpaceId }: LLMRoleManagerProps) { ...(userImageConfigs ?? []).filter((config) => config.id && config.id.toString().trim() !== ""), ]; + // Combine global and custom vision LLM configs + const allVisionConfigs = [ + ...globalVisionConfigs.map((config) => ({ ...config, is_global: true })), + ...(userVisionConfigs ?? []).filter( + (config) => config.id && config.id.toString().trim() !== "" + ), + ]; + const isAssignmentComplete = allLLMConfigs.some((c) => c.id === assignments.agent_llm_id) && allLLMConfigs.some((c) => c.id === assignments.document_summary_llm_id) && @@ -191,13 +215,17 @@ export function LLMRoleManager({ searchSpaceId }: LLMRoleManagerProps) { preferencesLoading || globalConfigsLoading || imageConfigsLoading || - globalImageConfigsLoading; + globalImageConfigsLoading || + visionConfigsLoading || + globalVisionConfigsLoading; const hasError = configsError || preferencesError || globalConfigsError || imageConfigsError || - globalImageConfigsError; + globalImageConfigsError || + visionConfigsError || + globalVisionConfigsError; const hasAnyConfigs = allLLMConfigs.length > 0 || allImageConfigs.length > 0; return ( @@ -291,15 +319,27 @@ export function LLMRoleManager({ searchSpaceId }: LLMRoleManagerProps) {
{Object.entries(ROLE_DESCRIPTIONS).map(([key, role]) => { const IconComponent = role.icon; - const isImageRole = role.configType === "image"; const currentAssignment = assignments[role.prefKey as keyof typeof assignments]; // Pick the right config lists based on role type - const roleGlobalConfigs = isImageRole ? globalImageConfigs : globalConfigs; - const roleUserConfigs = isImageRole - ? (userImageConfigs ?? []).filter((c) => c.id && c.id.toString().trim() !== "") - : newLLMConfigs.filter((c) => c.id && c.id.toString().trim() !== ""); - const roleAllConfigs = isImageRole ? allImageConfigs : allLLMConfigs; + const roleGlobalConfigs = + role.configType === "image" + ? globalImageConfigs + : role.configType === "vision" + ? globalVisionConfigs + : globalConfigs; + const roleUserConfigs = + role.configType === "image" + ? (userImageConfigs ?? []).filter((c) => c.id && c.id.toString().trim() !== "") + : role.configType === "vision" + ? (userVisionConfigs ?? []).filter((c) => c.id && c.id.toString().trim() !== "") + : newLLMConfigs.filter((c) => c.id && c.id.toString().trim() !== ""); + const roleAllConfigs = + role.configType === "image" + ? allImageConfigs + : role.configType === "vision" + ? allVisionConfigs + : allLLMConfigs; const assignedConfig = roleAllConfigs.find((config) => config.id === currentAssignment); const isAssigned = !!assignedConfig; diff --git a/surfsense_web/contracts/enums/vision-providers.ts b/surfsense_web/contracts/enums/vision-providers.ts new file mode 100644 index 000000000..260b03585 --- /dev/null +++ b/surfsense_web/contracts/enums/vision-providers.ts @@ -0,0 +1,102 @@ +export interface VisionProviderInfo { + value: string; + label: string; + example: string; + description: string; + apiBase?: string; +} + +export const VISION_PROVIDERS: VisionProviderInfo[] = [ + { + value: "OPENAI", + label: "OpenAI", + example: "gpt-4o, gpt-4o-mini", + description: "GPT-4o vision models", + }, + { + value: "ANTHROPIC", + label: "Anthropic", + example: "claude-sonnet-4-20250514", + description: "Claude vision models", + }, + { + value: "GOOGLE", + label: "Google AI Studio", + example: "gemini-2.5-flash, gemini-2.0-flash", + description: "Gemini vision models", + }, + { + value: "AZURE_OPENAI", + label: "Azure OpenAI", + example: "azure/gpt-4o", + description: "OpenAI vision models on Azure", + }, + { + value: "VERTEX_AI", + label: "Google Vertex AI", + example: "vertex_ai/gemini-2.5-flash", + description: "Gemini vision models on Vertex AI", + }, + { + value: "BEDROCK", + label: "AWS Bedrock", + example: "bedrock/anthropic.claude-sonnet-4-20250514-v1:0", + description: "Vision models on AWS Bedrock", + }, + { + value: "XAI", + label: "xAI", + example: "grok-2-vision", + description: "Grok vision models", + }, + { + value: "OPENROUTER", + label: "OpenRouter", + example: "openrouter/openai/gpt-4o", + description: "Vision models via OpenRouter", + }, + { + value: "OLLAMA", + label: "Ollama", + example: "llava, bakllava", + description: "Local vision models via Ollama", + apiBase: "http://localhost:11434", + }, + { + value: "GROQ", + label: "Groq", + example: "llama-4-scout-17b-16e-instruct", + description: "Vision models on Groq", + }, + { + value: "TOGETHER_AI", + label: "Together AI", + example: "meta-llama/Llama-4-Scout-17B-16E-Instruct", + description: "Vision models on Together AI", + }, + { + value: "FIREWORKS_AI", + label: "Fireworks AI", + example: "fireworks_ai/phi-3-vision-128k-instruct", + description: "Vision models on Fireworks AI", + }, + { + value: "DEEPSEEK", + label: "DeepSeek", + example: "deepseek-chat", + description: "DeepSeek vision models", + apiBase: "https://api.deepseek.com", + }, + { + value: "MISTRAL", + label: "Mistral", + example: "pixtral-large-latest", + description: "Pixtral vision models", + }, + { + value: "CUSTOM", + label: "Custom Provider", + example: "custom/my-vision-model", + description: "Custom OpenAI-compatible vision endpoint", + }, +]; diff --git a/surfsense_web/contracts/types/new-llm-config.types.ts b/surfsense_web/contracts/types/new-llm-config.types.ts index 02837cc73..6bef94bac 100644 --- a/surfsense_web/contracts/types/new-llm-config.types.ts +++ b/surfsense_web/contracts/types/new-llm-config.types.ts @@ -252,23 +252,99 @@ export const globalImageGenConfig = z.object({ export const getGlobalImageGenConfigsResponse = z.array(globalImageGenConfig); +// ============================================================================= +// Vision LLM Config (separate table for vision-capable models) +// ============================================================================= + +export const visionProviderEnum = z.enum([ + "OPENAI", + "ANTHROPIC", + "GOOGLE", + "AZURE_OPENAI", + "VERTEX_AI", + "BEDROCK", + "XAI", + "OPENROUTER", + "OLLAMA", + "GROQ", + "TOGETHER_AI", + "FIREWORKS_AI", + "DEEPSEEK", + "MISTRAL", + "CUSTOM", +]); + +export type VisionProvider = z.infer; + +export const visionLLMConfig = z.object({ + id: z.number(), + name: z.string().max(100), + description: z.string().max(500).nullable().optional(), + provider: visionProviderEnum, + custom_provider: z.string().max(100).nullable().optional(), + model_name: z.string().max(100), + api_key: z.string(), + api_base: z.string().max(500).nullable().optional(), + api_version: z.string().max(50).nullable().optional(), + litellm_params: z.record(z.string(), z.any()).nullable().optional(), + created_at: z.string(), + search_space_id: z.number(), + user_id: z.string(), +}); + +export const createVisionLLMConfigRequest = visionLLMConfig.omit({ + id: true, + created_at: true, + user_id: true, +}); + +export const createVisionLLMConfigResponse = visionLLMConfig; + +export const getVisionLLMConfigsResponse = z.array(visionLLMConfig); + +export const updateVisionLLMConfigRequest = z.object({ + id: z.number(), + data: visionLLMConfig + .omit({ id: true, created_at: true, search_space_id: true, user_id: true }) + .partial(), +}); + +export const updateVisionLLMConfigResponse = visionLLMConfig; + +export const deleteVisionLLMConfigResponse = z.object({ + message: z.string(), + id: z.number(), +}); + +export const globalVisionLLMConfig = z.object({ + id: z.number(), + name: z.string(), + description: z.string().nullable().optional(), + provider: z.string(), + custom_provider: z.string().nullable().optional(), + model_name: z.string(), + api_base: z.string().nullable().optional(), + api_version: z.string().nullable().optional(), + litellm_params: z.record(z.string(), z.any()).nullable().optional(), + is_global: z.literal(true), + is_auto_mode: z.boolean().optional().default(false), +}); + +export const getGlobalVisionLLMConfigsResponse = z.array(globalVisionLLMConfig); + // ============================================================================= // LLM Preferences (Role Assignments) // ============================================================================= -/** - * LLM Preferences schemas - for role assignments - * image_generation uses image_generation_config_id (not llm_id) - */ export const llmPreferences = z.object({ agent_llm_id: z.union([z.number(), z.null()]).optional(), document_summary_llm_id: z.union([z.number(), z.null()]).optional(), image_generation_config_id: z.union([z.number(), z.null()]).optional(), - vision_llm_id: z.union([z.number(), z.null()]).optional(), + vision_llm_config_id: z.union([z.number(), z.null()]).optional(), agent_llm: z.union([z.record(z.string(), z.unknown()), z.null()]).optional(), document_summary_llm: z.union([z.record(z.string(), z.unknown()), z.null()]).optional(), image_generation_config: z.union([z.record(z.string(), z.unknown()), z.null()]).optional(), - vision_llm: z.union([z.record(z.string(), z.unknown()), z.null()]).optional(), + vision_llm_config: z.union([z.record(z.string(), z.unknown()), z.null()]).optional(), }); /** @@ -289,7 +365,7 @@ export const updateLLMPreferencesRequest = z.object({ agent_llm_id: true, document_summary_llm_id: true, image_generation_config_id: true, - vision_llm_id: true, + vision_llm_config_id: true, }), }); @@ -341,6 +417,15 @@ export type UpdateImageGenConfigResponse = z.infer; export type GlobalImageGenConfig = z.infer; export type GetGlobalImageGenConfigsResponse = z.infer; +export type VisionLLMConfig = z.infer; +export type CreateVisionLLMConfigRequest = z.infer; +export type CreateVisionLLMConfigResponse = z.infer; +export type GetVisionLLMConfigsResponse = z.infer; +export type UpdateVisionLLMConfigRequest = z.infer; +export type UpdateVisionLLMConfigResponse = z.infer; +export type DeleteVisionLLMConfigResponse = z.infer; +export type GlobalVisionLLMConfig = z.infer; +export type GetGlobalVisionLLMConfigsResponse = z.infer; export type LLMPreferences = z.infer; export type GetLLMPreferencesRequest = z.infer; export type GetLLMPreferencesResponse = z.infer; diff --git a/surfsense_web/lib/apis/vision-llm-config-api.service.ts b/surfsense_web/lib/apis/vision-llm-config-api.service.ts new file mode 100644 index 000000000..4099c6b39 --- /dev/null +++ b/surfsense_web/lib/apis/vision-llm-config-api.service.ts @@ -0,0 +1,58 @@ +import { + type CreateVisionLLMConfigRequest, + createVisionLLMConfigRequest, + createVisionLLMConfigResponse, + deleteVisionLLMConfigResponse, + getGlobalVisionLLMConfigsResponse, + getVisionLLMConfigsResponse, + type UpdateVisionLLMConfigRequest, + updateVisionLLMConfigRequest, + updateVisionLLMConfigResponse, +} from "@/contracts/types/new-llm-config.types"; +import { ValidationError } from "../error"; +import { baseApiService } from "./base-api.service"; + +class VisionLLMConfigApiService { + getGlobalConfigs = async () => { + return baseApiService.get( + `/api/v1/global-vision-llm-configs`, + getGlobalVisionLLMConfigsResponse + ); + }; + + createConfig = async (request: CreateVisionLLMConfigRequest) => { + const parsed = createVisionLLMConfigRequest.safeParse(request); + if (!parsed.success) { + const msg = parsed.error.issues.map((i) => i.message).join(", "); + throw new ValidationError(`Invalid request: ${msg}`); + } + return baseApiService.post(`/api/v1/vision-llm-configs`, createVisionLLMConfigResponse, { + body: parsed.data, + }); + }; + + getConfigs = async (searchSpaceId: number) => { + const params = new URLSearchParams({ + search_space_id: String(searchSpaceId), + }).toString(); + return baseApiService.get(`/api/v1/vision-llm-configs?${params}`, getVisionLLMConfigsResponse); + }; + + updateConfig = async (request: UpdateVisionLLMConfigRequest) => { + const parsed = updateVisionLLMConfigRequest.safeParse(request); + if (!parsed.success) { + const msg = parsed.error.issues.map((i) => i.message).join(", "); + throw new ValidationError(`Invalid request: ${msg}`); + } + const { id, data } = parsed.data; + return baseApiService.put(`/api/v1/vision-llm-configs/${id}`, updateVisionLLMConfigResponse, { + body: data, + }); + }; + + deleteConfig = async (id: number) => { + return baseApiService.delete(`/api/v1/vision-llm-configs/${id}`, deleteVisionLLMConfigResponse); + }; +} + +export const visionLLMConfigApiService = new VisionLLMConfigApiService(); diff --git a/surfsense_web/lib/query-client/cache-keys.ts b/surfsense_web/lib/query-client/cache-keys.ts index 754886618..04f348ff8 100644 --- a/surfsense_web/lib/query-client/cache-keys.ts +++ b/surfsense_web/lib/query-client/cache-keys.ts @@ -39,6 +39,11 @@ export const cacheKeys = { byId: (configId: number) => ["image-gen-configs", "detail", configId] as const, global: () => ["image-gen-configs", "global"] as const, }, + visionLLMConfigs: { + all: (searchSpaceId: number) => ["vision-llm-configs", searchSpaceId] as const, + byId: (configId: number) => ["vision-llm-configs", "detail", configId] as const, + global: () => ["vision-llm-configs", "global"] as const, + }, auth: { user: ["auth", "user"] as const, },