apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-06-02 19:55:18 +02:00
Code Issues Projects Releases Packages Wiki Activity

Add frontend vision LLM config types, API, atoms, and role manager wiring

Browse source
This commit is contained in:
CREDO23 2026-04-07 19:24:43 +02:00
parent 4a675b64f4
commit 3369b8a832
7 changed files with 421 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

102
surfsense_web/contracts/enums/vision-providers.ts Normal file
View file

@ -0,0 +1,102 @@
export interface VisionProviderInfo {
value: string;
label: string;
example: string;
description: string;
apiBase?: string;
}
export const VISION_PROVIDERS: VisionProviderInfo[] = [
{
value: "OPENAI",
label: "OpenAI",
example: "gpt-4o, gpt-4o-mini",
description: "GPT-4o vision models",
},
{
value: "ANTHROPIC",
label: "Anthropic",
example: "claude-sonnet-4-20250514",
description: "Claude vision models",
},
{
value: "GOOGLE",
label: "Google AI Studio",
example: "gemini-2.5-flash, gemini-2.0-flash",
description: "Gemini vision models",
},
{
value: "AZURE_OPENAI",
label: "Azure OpenAI",
example: "azure/gpt-4o",
description: "OpenAI vision models on Azure",
},
{
value: "VERTEX_AI",
label: "Google Vertex AI",
example: "vertex_ai/gemini-2.5-flash",
description: "Gemini vision models on Vertex AI",
},
{
value: "BEDROCK",
label: "AWS Bedrock",
example: "bedrock/anthropic.claude-sonnet-4-20250514-v1:0",
description: "Vision models on AWS Bedrock",
},
{
value: "XAI",
label: "xAI",
example: "grok-2-vision",
description: "Grok vision models",
},
{
value: "OPENROUTER",
label: "OpenRouter",
example: "openrouter/openai/gpt-4o",
description: "Vision models via OpenRouter",
},
{
value: "OLLAMA",
label: "Ollama",
example: "llava, bakllava",
description: "Local vision models via Ollama",
apiBase: "http://localhost:11434",
},
{
value: "GROQ",
label: "Groq",
example: "llama-4-scout-17b-16e-instruct",
description: "Vision models on Groq",
},
{
value: "TOGETHER_AI",
label: "Together AI",
example: "meta-llama/Llama-4-Scout-17B-16E-Instruct",
description: "Vision models on Together AI",
},
{
value: "FIREWORKS_AI",
label: "Fireworks AI",
example: "fireworks_ai/phi-3-vision-128k-instruct",
description: "Vision models on Fireworks AI",
},
{
value: "DEEPSEEK",
label: "DeepSeek",
example: "deepseek-chat",
description: "DeepSeek vision models",
apiBase: "https://api.deepseek.com",
},
{
value: "MISTRAL",
label: "Mistral",
example: "pixtral-large-latest",
description: "Pixtral vision models",
},
{
value: "CUSTOM",
label: "Custom Provider",
example: "custom/my-vision-model",
description: "Custom OpenAI-compatible vision endpoint",
},
];

99
surfsense_web/contracts/types/new-llm-config.types.ts
View file

@ -252,23 +252,99 @@ export const globalImageGenConfig = z.object({
export const getGlobalImageGenConfigsResponse = z.array(globalImageGenConfig);
// =============================================================================
// Vision LLM Config (separate table for vision-capable models)
// =============================================================================
export const visionProviderEnum = z.enum([
"OPENAI",
"ANTHROPIC",
"GOOGLE",
"AZURE_OPENAI",
"VERTEX_AI",
"BEDROCK",
"XAI",
"OPENROUTER",
"OLLAMA",
"GROQ",
"TOGETHER_AI",
"FIREWORKS_AI",
"DEEPSEEK",
"MISTRAL",
"CUSTOM",
]);
export type VisionProvider = z.infer<typeof visionProviderEnum>;
export const visionLLMConfig = z.object({
id: z.number(),
name: z.string().max(100),
description: z.string().max(500).nullable().optional(),
provider: visionProviderEnum,
custom_provider: z.string().max(100).nullable().optional(),
model_name: z.string().max(100),
api_key: z.string(),
api_base: z.string().max(500).nullable().optional(),
api_version: z.string().max(50).nullable().optional(),
litellm_params: z.record(z.string(), z.any()).nullable().optional(),
created_at: z.string(),
search_space_id: z.number(),
user_id: z.string(),
});
export const createVisionLLMConfigRequest = visionLLMConfig.omit({
id: true,
created_at: true,
user_id: true,
});
export const createVisionLLMConfigResponse = visionLLMConfig;
export const getVisionLLMConfigsResponse = z.array(visionLLMConfig);
export const updateVisionLLMConfigRequest = z.object({
id: z.number(),
data: visionLLMConfig
.omit({ id: true, created_at: true, search_space_id: true, user_id: true })
.partial(),
});
export const updateVisionLLMConfigResponse = visionLLMConfig;
export const deleteVisionLLMConfigResponse = z.object({
message: z.string(),
id: z.number(),
});
export const globalVisionLLMConfig = z.object({
id: z.number(),
name: z.string(),
description: z.string().nullable().optional(),
provider: z.string(),
custom_provider: z.string().nullable().optional(),
model_name: z.string(),
api_base: z.string().nullable().optional(),
api_version: z.string().nullable().optional(),
litellm_params: z.record(z.string(), z.any()).nullable().optional(),
is_global: z.literal(true),
is_auto_mode: z.boolean().optional().default(false),
});
export const getGlobalVisionLLMConfigsResponse = z.array(globalVisionLLMConfig);
// =============================================================================
// LLM Preferences (Role Assignments)
// =============================================================================
/**
* LLM Preferences schemas - for role assignments
* image_generation uses image_generation_config_id (not llm_id)
*/
export const llmPreferences = z.object({
agent_llm_id: z.union([z.number(), z.null()]).optional(),
document_summary_llm_id: z.union([z.number(), z.null()]).optional(),
image_generation_config_id: z.union([z.number(), z.null()]).optional(),
vision_llm_id: z.union([z.number(), z.null()]).optional(),
vision_llm_config_id: z.union([z.number(), z.null()]).optional(),
agent_llm: z.union([z.record(z.string(), z.unknown()), z.null()]).optional(),
document_summary_llm: z.union([z.record(z.string(), z.unknown()), z.null()]).optional(),
image_generation_config: z.union([z.record(z.string(), z.unknown()), z.null()]).optional(),
vision_llm: z.union([z.record(z.string(), z.unknown()), z.null()]).optional(),
vision_llm_config: z.union([z.record(z.string(), z.unknown()), z.null()]).optional(),
});
/**
@ -289,7 +365,7 @@ export const updateLLMPreferencesRequest = z.object({
agent_llm_id: true,
document_summary_llm_id: true,
image_generation_config_id: true,
vision_llm_id: true,
vision_llm_config_id: true,
}),
});
@ -341,6 +417,15 @@ export type UpdateImageGenConfigResponse = z.infer<typeof updateImageGenConfigRe
export type DeleteImageGenConfigResponse = z.infer<typeof deleteImageGenConfigResponse>;
export type GlobalImageGenConfig = z.infer<typeof globalImageGenConfig>;
export type GetGlobalImageGenConfigsResponse = z.infer<typeof getGlobalImageGenConfigsResponse>;
export type VisionLLMConfig = z.infer<typeof visionLLMConfig>;
export type CreateVisionLLMConfigRequest = z.infer<typeof createVisionLLMConfigRequest>;
export type CreateVisionLLMConfigResponse = z.infer<typeof createVisionLLMConfigResponse>;
export type GetVisionLLMConfigsResponse = z.infer<typeof getVisionLLMConfigsResponse>;
export type UpdateVisionLLMConfigRequest = z.infer<typeof updateVisionLLMConfigRequest>;
export type UpdateVisionLLMConfigResponse = z.infer<typeof updateVisionLLMConfigResponse>;
export type DeleteVisionLLMConfigResponse = z.infer<typeof deleteVisionLLMConfigResponse>;
export type GlobalVisionLLMConfig = z.infer<typeof globalVisionLLMConfig>;
export type GetGlobalVisionLLMConfigsResponse = z.infer<typeof getGlobalVisionLLMConfigsResponse>;
export type LLMPreferences = z.infer<typeof llmPreferences>;
export type GetLLMPreferencesRequest = z.infer<typeof getLLMPreferencesRequest>;
export type GetLLMPreferencesResponse = z.infer<typeof getLLMPreferencesResponse>;