hotpatch(cloud): add llm load balancing

surfsense_web/components/assistant-ui/thread.tsx

@@ -485,7 +485,8 @@ const ComposerAction: FC<ComposerActionProps> = ({ isBlockedByOtherUser = false
 		if (agentLlmId === null || agentLlmId === undefined) return false;
 
 		// Check if the configured model actually exists
-		if (agentLlmId < 0) {
+		// Auto mode (ID 0) and global configs (negative IDs) are in globalConfigs
+		if (agentLlmId <= 0) {
 			return globalConfigs?.some((c) => c.id === agentLlmId) ?? false;
 		}
 		return userConfigs?.some((c) => c.id === agentLlmId) ?? false;

surfsense_web/components/new-chat/model-config-sidebar.tsx

@@ -1,7 +1,7 @@
 "use client";
 
 import { useAtomValue } from "jotai";
-import { AlertCircle, Bot, ChevronRight, Globe, User, X } from "lucide-react";
+import { AlertCircle, Bot, ChevronRight, Globe, Shuffle, User, X, Zap } from "lucide-react";
 import { AnimatePresence, motion } from "motion/react";
 import { useCallback, useEffect, useState } from "react";
 import { createPortal } from "react-dom";
@@ -62,9 +62,13 @@ export function ModelConfigSidebar({
 		return () => window.removeEventListener("keydown", handleEscape);
 	}, [open, onOpenChange]);
 
+	// Check if this is Auto mode
+	const isAutoMode = config && "is_auto_mode" in config && config.is_auto_mode;
+
 	// Get title based on mode
 	const getTitle = () => {
 		if (mode === "create") return "Add New Configuration";
+		if (isAutoMode) return "Auto Mode (Load Balanced)";
 		if (isGlobal) return "View Global Configuration";
 		return "Edit Configuration";
 	};
@@ -187,15 +191,37 @@ export function ModelConfigSidebar({
 						)}
 					>
 						{/* Header */}
-						<div className="flex items-center justify-between px-6 py-4 border-b border-border/50 bg-muted/20">
+						<div
+							className={cn(
+								"flex items-center justify-between px-6 py-4 border-b border-border/50",
+								isAutoMode ? "bg-gradient-to-r from-violet-500/10 to-purple-500/10" : "bg-muted/20"
+							)}
+						>
 							<div className="flex items-center gap-3">
-								<div className="flex items-center justify-center size-10 rounded-xl bg-primary/10">
-									<Bot className="size-5 text-primary" />
+								<div
+									className={cn(
+										"flex items-center justify-center size-10 rounded-xl",
+										isAutoMode ? "bg-gradient-to-br from-violet-500 to-purple-600" : "bg-primary/10"
+									)}
+								>
+									{isAutoMode ? (
+										<Shuffle className="size-5 text-white" />
+									) : (
+										<Bot className="size-5 text-primary" />
+									)}
 								</div>
 								<div>
 									<h2 className="text-base sm:text-lg font-semibold">{getTitle()}</h2>
 									<div className="flex items-center gap-2 mt-0.5">
-										{isGlobal ? (
+										{isAutoMode ? (
+											<Badge
+												variant="secondary"
+												className="gap-1 text-xs bg-violet-100 text-violet-700 dark:bg-violet-900/30 dark:text-violet-300"
+											>
+												<Zap className="size-3" />
+												Recommended
+											</Badge>
+										) : isGlobal ? (
 											<Badge variant="secondary" className="gap-1 text-xs">
 												<Globe className="size-3" />
 												Global
@@ -206,7 +232,7 @@ export function ModelConfigSidebar({
 												Custom
 											</Badge>
 										) : null}
-										{config && (
+										{config && !isAutoMode && (
 											<span className="text-xs text-muted-foreground">{config.model_name}</span>
 										)}
 									</div>
@@ -226,8 +252,19 @@ export function ModelConfigSidebar({
 						{/* Content - use overflow-y-auto instead of ScrollArea for better compatibility */}
 						<div className="flex-1 overflow-y-auto">
 							<div className="p-6">
+								{/* Auto mode info banner */}
+								{isAutoMode && (
+									<Alert className="mb-6 border-violet-500/30 bg-violet-500/5">
+										<Shuffle className="size-4 text-violet-500" />
+										<AlertDescription className="text-sm text-violet-700 dark:text-violet-400">
+											Auto mode automatically distributes requests across all available LLM
+											providers to optimize performance and avoid rate limits.
+										</AlertDescription>
+									</Alert>
+								)}
+
 								{/* Global config notice */}
-								{isGlobal && mode !== "create" && (
+								{isGlobal && !isAutoMode && mode !== "create" && (
 									<Alert className="mb-6 border-amber-500/30 bg-amber-500/5">
 										<AlertCircle className="size-4 text-amber-500" />
 										<AlertDescription className="text-sm text-amber-700 dark:text-amber-400">
@@ -247,6 +284,87 @@ export function ModelConfigSidebar({
 										mode="create"
 										submitLabel="Create & Use"
 									/>
+								) : isAutoMode && config ? (
+									// Special view for Auto mode
+									<div className="space-y-6">
+										{/* Auto Mode Features */}
+										<div className="space-y-4">
+											<div className="space-y-1.5">
+												<div className="text-xs font-medium text-muted-foreground uppercase tracking-wider">
+													How It Works
+												</div>
+												<p className="text-sm text-muted-foreground">{config.description}</p>
+											</div>
+
+											<div className="h-px bg-border/50" />
+
+											<div className="space-y-3">
+												<div className="text-xs font-medium text-muted-foreground uppercase tracking-wider">
+													Key Benefits
+												</div>
+												<div className="space-y-2">
+													<div className="flex items-start gap-3 p-3 rounded-lg bg-violet-50 dark:bg-violet-900/20 border border-violet-200 dark:border-violet-800/50">
+														<Zap className="size-4 text-violet-600 dark:text-violet-400 mt-0.5 shrink-0" />
+														<div>
+															<p className="text-sm font-medium text-violet-900 dark:text-violet-100">
+																Automatic Load Balancing
+															</p>
+															<p className="text-xs text-violet-700 dark:text-violet-300">
+																Distributes requests across all configured LLM providers
+															</p>
+														</div>
+													</div>
+													<div className="flex items-start gap-3 p-3 rounded-lg bg-violet-50 dark:bg-violet-900/20 border border-violet-200 dark:border-violet-800/50">
+														<Zap className="size-4 text-violet-600 dark:text-violet-400 mt-0.5 shrink-0" />
+														<div>
+															<p className="text-sm font-medium text-violet-900 dark:text-violet-100">
+																Rate Limit Protection
+															</p>
+															<p className="text-xs text-violet-700 dark:text-violet-300">
+																Automatically handles rate limits with cooldowns and retries
+															</p>
+														</div>
+													</div>
+													<div className="flex items-start gap-3 p-3 rounded-lg bg-violet-50 dark:bg-violet-900/20 border border-violet-200 dark:border-violet-800/50">
+														<Zap className="size-4 text-violet-600 dark:text-violet-400 mt-0.5 shrink-0" />
+														<div>
+															<p className="text-sm font-medium text-violet-900 dark:text-violet-100">
+																Automatic Failover
+															</p>
+															<p className="text-xs text-violet-700 dark:text-violet-300">
+																Falls back to other providers if one becomes unavailable
+															</p>
+														</div>
+													</div>
+												</div>
+											</div>
+										</div>
+
+										{/* Action Buttons */}
+										<div className="flex gap-3 pt-4 border-t border-border/50">
+											<Button
+												variant="outline"
+												className="flex-1"
+												onClick={() => onOpenChange(false)}
+											>
+												Close
+											</Button>
+											<Button
+												className="flex-1 gap-2 bg-gradient-to-r from-violet-500 to-purple-600 hover:from-violet-600 hover:to-purple-700"
+												onClick={handleUseGlobalConfig}
+												disabled={isSubmitting}
+											>
+												{isSubmitting ? (
+													<>Loading...</>
+												) : (
+													<>
+														<ChevronRight className="size-4" />
+														Use Auto Mode
+													</>
+												)}
+											</Button>
+										</div>
+									</div>
 								) : isGlobal && config ? (
 									// Read-only view for global configs
 									<div className="space-y-6">

surfsense_web/components/new-chat/model-selector.tsx

@@ -10,6 +10,7 @@ import {
 	Globe,
 	Plus,
 	Settings2,
+	Shuffle,
 	Sparkles,
 	User,
 	Zap,
@@ -43,8 +44,14 @@ import type {
 import { cn } from "@/lib/utils";
 
 // Provider icons mapping
-const getProviderIcon = (provider: string) => {
+const getProviderIcon = (provider: string, isAutoMode?: boolean) => {
 	const iconClass = "size-4";
+
+	// Special icon for Auto mode
+	if (isAutoMode || provider?.toUpperCase() === "AUTO") {
+		return <Shuffle className={cn(iconClass, "text-violet-500")} />;
+	}
+
 	switch (provider?.toUpperCase()) {
 		case "OPENAI":
 			return <Sparkles className={cn(iconClass, "text-emerald-500")} />;
@@ -90,14 +97,19 @@ export function ModelSelector({ onEdit, onAddNew, className }: ModelSelectorProp
 		const agentLlmId = preferences.agent_llm_id;
 		if (agentLlmId === null || agentLlmId === undefined) return null;
 
-		// Check if it's a global config (negative ID)
-		if (agentLlmId < 0) {
+		// Check if it's Auto mode (ID 0) or global config (negative ID)
+		if (agentLlmId <= 0) {
 			return globalConfigs?.find((c) => c.id === agentLlmId) ?? null;
 		}
 		// Otherwise, check user configs
 		return userConfigs?.find((c) => c.id === agentLlmId) ?? null;
 	}, [preferences, globalConfigs, userConfigs]);
 
+	// Check if current config is Auto mode
+	const isCurrentAutoMode = useMemo(() => {
+		return currentConfig && "is_auto_mode" in currentConfig && currentConfig.is_auto_mode;
+	}, [currentConfig]);
+
 	// Filter configs based on search
 	const filteredGlobalConfigs = useMemo(() => {
 		if (!globalConfigs) return [];
@@ -184,14 +196,23 @@ export function ModelSelector({ onEdit, onAddNew, className }: ModelSelectorProp
 						</>
 					) : currentConfig ? (
 						<>
-							{getProviderIcon(currentConfig.provider)}
+							{getProviderIcon(currentConfig.provider, isCurrentAutoMode ?? false)}
 							<span className="max-w-[100px] md:max-w-[150px] truncate hidden md:inline">
 								{currentConfig.name}
 							</span>
-							<Badge variant="secondary" className="ml-1 text-[10px] px-1.5 py-0 h-4 bg-muted/80">
-								{currentConfig.model_name.split("/").pop()?.slice(0, 10) ||
-									currentConfig.model_name.slice(0, 10)}
-							</Badge>
+							{isCurrentAutoMode ? (
+								<Badge
+									variant="secondary"
+									className="ml-1 text-[10px] px-1.5 py-0 h-4 bg-violet-100 text-violet-700 dark:bg-violet-900/30 dark:text-violet-300"
+								>
+									Balanced
+								</Badge>
+							) : (
+								<Badge variant="secondary" className="ml-1 text-[10px] px-1.5 py-0 h-4 bg-muted/80">
+									{currentConfig.model_name.split("/").pop()?.slice(0, 10) ||
+										currentConfig.model_name.slice(0, 10)}
+								</Badge>
+							)}
 						</>
 					) : (
 						<>
@@ -246,6 +267,7 @@ export function ModelSelector({ onEdit, onAddNew, className }: ModelSelectorProp
 								</div>
 								{filteredGlobalConfigs.map((config) => {
 									const isSelected = currentConfig?.id === config.id;
+									const isAutoMode = "is_auto_mode" in config && config.is_auto_mode;
 									return (
 										<CommandItem
 											key={`global-${config.id}`}
@@ -254,22 +276,33 @@ export function ModelSelector({ onEdit, onAddNew, className }: ModelSelectorProp
 											className={cn(
 												"mx-2 rounded-lg mb-1 cursor-pointer group transition-all",
 												"hover:bg-accent/50",
-												isSelected && "bg-accent/80"
+												isSelected && "bg-accent/80",
+												isAutoMode && "border border-violet-200 dark:border-violet-800/50"
 											)}
 										>
 											<div className="flex items-center justify-between w-full gap-2">
 												<div className="flex items-center gap-3 min-w-0 flex-1">
-													<div className="shrink-0">{getProviderIcon(config.provider)}</div>
+													<div className="shrink-0">
+														{getProviderIcon(config.provider, isAutoMode)}
+													</div>
 													<div className="min-w-0 flex-1">
 														<div className="flex items-center gap-2">
 															<span className="font-medium truncate">{config.name}</span>
+															{isAutoMode && (
+																<Badge
+																	variant="secondary"
+																	className="text-[9px] px-1 py-0 h-3.5 bg-violet-100 text-violet-700 dark:bg-violet-900/30 dark:text-violet-300 border-0"
+																>
+																	Recommended
+																</Badge>
+															)}
 															{isSelected && <Check className="size-3.5 text-primary shrink-0" />}
 														</div>
 														<div className="flex items-center gap-1.5 mt-0.5">
 															<span className="text-xs text-muted-foreground truncate">
-																{config.model_name}
+																{isAutoMode ? "Auto load balancing" : config.model_name}
 															</span>
-															{config.citations_enabled && (
+															{!isAutoMode && config.citations_enabled && (
 																<Badge
 																	variant="outline"
 																	className="text-[9px] px-1 py-0 h-3.5 bg-primary/10 text-primary border-primary/20"
@@ -280,14 +313,16 @@ export function ModelSelector({ onEdit, onAddNew, className }: ModelSelectorProp
 														</div>
 													</div>
 												</div>
-												<Button
-													variant="ghost"
-													size="icon"
-													className="size-7 shrink-0 rounded-md hover:bg-muted opacity-0 group-hover:opacity-100 transition-opacity"
-													onClick={(e) => handleEditConfig(e, config, true)}
-												>
-													<Edit3 className="size-3.5 text-muted-foreground" />
-												</Button>
+												{!isAutoMode && (
+													<Button
+														variant="ghost"
+														size="icon"
+														className="size-7 shrink-0 rounded-md hover:bg-muted opacity-0 group-hover:opacity-100 transition-opacity"
+														onClick={(e) => handleEditConfig(e, config, true)}
+													>
+														<Edit3 className="size-3.5 text-muted-foreground" />
+													</Button>
+												)}
 											</div>
 										</CommandItem>
 									);

surfsense_web/components/settings/llm-role-manager.tsx

@@ -1,7 +1,16 @@
 "use client";
 
 import { useAtomValue } from "jotai";
-import { AlertCircle, Bot, CheckCircle, FileText, RefreshCw, RotateCcw, Save } from "lucide-react";
+import {
+	AlertCircle,
+	Bot,
+	CheckCircle,
+	FileText,
+	RefreshCw,
+	RotateCcw,
+	Save,
+	Shuffle,
+} from "lucide-react";
 import { motion } from "motion/react";
 import { useEffect, useState } from "react";
 import { toast } from "sonner";
@@ -24,6 +33,7 @@ import {
 	SelectValue,
 } from "@/components/ui/select";
 import { Spinner } from "@/components/ui/spinner";
+import { cn } from "@/lib/utils";
 
 const ROLE_DESCRIPTIONS = {
 	agent: {
@@ -71,8 +81,8 @@ export function LLMRoleManager({ searchSpaceId }: LLMRoleManagerProps) {
 	const { mutateAsync: updatePreferences } = useAtomValue(updateLLMPreferencesMutationAtom);
 
 	const [assignments, setAssignments] = useState({
-		agent_llm_id: preferences.agent_llm_id || "",
-		document_summary_llm_id: preferences.document_summary_llm_id || "",
+		agent_llm_id: preferences.agent_llm_id ?? "",
+		document_summary_llm_id: preferences.document_summary_llm_id ?? "",
 	});
 
 	const [hasChanges, setHasChanges] = useState(false);
@@ -80,8 +90,8 @@ export function LLMRoleManager({ searchSpaceId }: LLMRoleManagerProps) {
 
 	useEffect(() => {
 		const newAssignments = {
-			agent_llm_id: preferences.agent_llm_id || "",
-			document_summary_llm_id: preferences.document_summary_llm_id || "",
+			agent_llm_id: preferences.agent_llm_id ?? "",
+			document_summary_llm_id: preferences.document_summary_llm_id ?? "",
 		};
 		setAssignments(newAssignments);
 		setHasChanges(false);
@@ -97,8 +107,8 @@ export function LLMRoleManager({ searchSpaceId }: LLMRoleManagerProps) {
 
 		// Check if there are changes compared to current preferences
 		const currentPrefs = {
-			agent_llm_id: preferences.agent_llm_id || "",
-			document_summary_llm_id: preferences.document_summary_llm_id || "",
+			agent_llm_id: preferences.agent_llm_id ?? "",
+			document_summary_llm_id: preferences.document_summary_llm_id ?? "",
 		};
 
 		const hasChangesNow = Object.keys(newAssignments).some(
@@ -141,13 +151,19 @@ export function LLMRoleManager({ searchSpaceId }: LLMRoleManagerProps) {
 
 	const handleReset = () => {
 		setAssignments({
-			agent_llm_id: preferences.agent_llm_id || "",
-			document_summary_llm_id: preferences.document_summary_llm_id || "",
+			agent_llm_id: preferences.agent_llm_id ?? "",
+			document_summary_llm_id: preferences.document_summary_llm_id ?? "",
 		});
 		setHasChanges(false);
 	};
 
-	const isAssignmentComplete = assignments.agent_llm_id && assignments.document_summary_llm_id;
+	const isAssignmentComplete =
+		assignments.agent_llm_id !== "" &&
+		assignments.agent_llm_id !== null &&
+		assignments.agent_llm_id !== undefined &&
+		assignments.document_summary_llm_id !== "" &&
+		assignments.document_summary_llm_id !== null &&
+		assignments.document_summary_llm_id !== undefined;
 
 	// Combine global and custom configs (new system)
 	const allConfigs = [
@@ -300,22 +316,47 @@ export function LLMRoleManager({ searchSpaceId }: LLMRoleManagerProps) {
 																	<div className="px-2 py-1.5 text-xs font-semibold text-muted-foreground">
 																		Global Configurations
 																	</div>
-																	{globalConfigs.map((config) => (
-																		<SelectItem key={config.id} value={config.id.toString()}>
-																			<div className="flex items-center gap-2">
-																				<Badge variant="outline" className="text-xs">
-																					{config.provider}
-																				</Badge>
-																				<span>{config.name}</span>
-																				<span className="text-muted-foreground">
-																					({config.model_name})
-																				</span>
-																				<Badge variant="secondary" className="text-xs">
-																					🌐 Global
-																				</Badge>
-																			</div>
-																		</SelectItem>
-																	))}
+																	{globalConfigs.map((config) => {
+																		const isAutoMode =
+																			"is_auto_mode" in config && config.is_auto_mode;
+																		return (
+																			<SelectItem key={config.id} value={config.id.toString()}>
+																				<div className="flex items-center gap-2">
+																					{isAutoMode ? (
+																						<Badge
+																							variant="outline"
+																							className="text-xs bg-violet-100 text-violet-700 dark:bg-violet-900/30 dark:text-violet-300 border-violet-200 dark:border-violet-700"
+																						>
+																							<Shuffle className="size-3 mr-1" />
+																							AUTO
+																						</Badge>
+																					) : (
+																						<Badge variant="outline" className="text-xs">
+																							{config.provider}
+																						</Badge>
+																					)}
+																					<span>{config.name}</span>
+																					{!isAutoMode && (
+																						<span className="text-muted-foreground">
+																							({config.model_name})
+																						</span>
+																					)}
+																					{isAutoMode ? (
+																						<Badge
+																							variant="secondary"
+																							className="text-xs bg-violet-100 text-violet-700 dark:bg-violet-900/30 dark:text-violet-300"
+																						>
+																							Recommended
+																						</Badge>
+																					) : (
+																						<Badge variant="secondary" className="text-xs">
+																							🌐 Global
+																						</Badge>
+																					)}
+																				</div>
+																			</SelectItem>
+																		);
+																	})}
 																</>
 															)}
 
@@ -349,27 +390,65 @@ export function LLMRoleManager({ searchSpaceId }: LLMRoleManagerProps) {
 												</div>
 
 												{assignedConfig && (
-													<div className="mt-2 md:mt-3 p-2 md:p-3 bg-muted/50 rounded-lg">
+													<div
+														className={cn(
+															"mt-2 md:mt-3 p-2 md:p-3 rounded-lg",
+															"is_auto_mode" in assignedConfig && assignedConfig.is_auto_mode
+																? "bg-violet-50 dark:bg-violet-900/20 border border-violet-200 dark:border-violet-800/50"
+																: "bg-muted/50"
+														)}
+													>
 														<div className="flex items-center gap-1.5 md:gap-2 text-xs md:text-sm flex-wrap">
-															<Bot className="w-3 h-3 md:w-4 md:h-4 shrink-0" />
+															{"is_auto_mode" in assignedConfig && assignedConfig.is_auto_mode ? (
+																<Shuffle className="w-3 h-3 md:w-4 md:h-4 shrink-0 text-violet-600 dark:text-violet-400" />
+															) : (
+																<Bot className="w-3 h-3 md:w-4 md:h-4 shrink-0" />
+															)}
 															<span className="font-medium">Assigned:</span>
-															<Badge variant="secondary" className="text-[10px] md:text-xs">
-																{assignedConfig.provider}
-															</Badge>
-															<span>{assignedConfig.name}</span>
-															{"is_global" in assignedConfig && assignedConfig.is_global && (
-																<Badge variant="outline" className="text-[9px] md:text-xs">
-																	🌐 Global
+															{"is_auto_mode" in assignedConfig && assignedConfig.is_auto_mode ? (
+																<Badge
+																	variant="secondary"
+																	className="text-[10px] md:text-xs bg-violet-100 text-violet-700 dark:bg-violet-900/30 dark:text-violet-300"
+																>
+																	AUTO
+																</Badge>
+															) : (
+																<Badge variant="secondary" className="text-[10px] md:text-xs">
+																	{assignedConfig.provider}
 																</Badge>
 															)}
+															<span>{assignedConfig.name}</span>
+															{"is_auto_mode" in assignedConfig && assignedConfig.is_auto_mode ? (
+																<Badge
+																	variant="outline"
+																	className="text-[9px] md:text-xs bg-violet-100 text-violet-700 dark:bg-violet-900/30 dark:text-violet-300 border-violet-200 dark:border-violet-700"
+																>
+																	Recommended
+																</Badge>
+															) : (
+																"is_global" in assignedConfig &&
+																assignedConfig.is_global && (
+																	<Badge variant="outline" className="text-[9px] md:text-xs">
+																		🌐 Global
+																	</Badge>
+																)
+															)}
 														</div>
-														<div className="text-[10px] md:text-xs text-muted-foreground mt-0.5 md:mt-1">
-															Model: {assignedConfig.model_name}
-														</div>
-														{assignedConfig.api_base && (
-															<div className="text-[10px] md:text-xs text-muted-foreground">
-																Base: {assignedConfig.api_base}
+														{"is_auto_mode" in assignedConfig && assignedConfig.is_auto_mode ? (
+															<div className="text-[10px] md:text-xs text-violet-600 dark:text-violet-400 mt-0.5 md:mt-1">
+																Automatically load balances across all available LLM providers
 															</div>
+														) : (
+															<>
+																<div className="text-[10px] md:text-xs text-muted-foreground mt-0.5 md:mt-1">
+																	Model: {assignedConfig.model_name}
+																</div>
+																{assignedConfig.api_base && (
+																	<div className="text-[10px] md:text-xs text-muted-foreground">
+																		Base: {assignedConfig.api_base}
+																	</div>
+																)}
+															</>
 														)}
 													</div>
 												)}