mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-22 23:31:12 +02:00
fdef50e78d
- Add three-tier LLM architecture (Mistral NeMo, TildeOpen, Gemini fallback) - Fix context window handling for mistral-nemo (128K tokens) - Add LiteLLM context override to prevent 1M token bug - Remove Google Analytics tracking from frontend - Add migration and installation documentation - Optimize for CPU-only inference on 32GB RAM servers Performance improvements: - 95% reduction in API costs - Response times: 5-25 seconds (down from 30-120s) - Better Latvian language quality with TildeOpen - Eliminated timeout errors Architecture changes: - Primary: Mistral NeMo 12B (France, local via Ollama) - Grammar: TildeOpen 30B (Latvia, local via Ollama) - Fallback: Gemini 2.0 Flash (Google API, emergency only) Technical fixes: - Fixed LiteLLM reporting incorrect 1M token context (actual: 128K) - Created mistral-nemo:128k model with proper num_ctx parameter - Added context window override in backend utils - Comprehensive security patterns in .gitignore Documentation: - MIGRATION_LOCAL_LLM.md: Complete architecture and history - INSTALLATION_LOCAL_LLM.md: Step-by-step deployment guide - PR_DESCRIPTION.md: Detailed PR description - sync-from-production.sh: Secure deployment script Tested on production at https://ai.kapteinis.lv since November 17, 2025. Co-authored-by: Ojārs Kapteiņš <ojars@kapteinis.lv> Co-authored-by: Claude AI Assistant <code@anthropic.com>
45 lines
1.9 KiB
Text
45 lines
1.9 KiB
Text
# Global LLM Configuration for SurfSense
|
|
# Three-tier architecture: Mistral NeMo (primary) -> TildeOpen (grammar) -> Gemini (fallback)
|
|
|
|
global_llm_configs:
|
|
# Mistral NeMo 12B - PRIMARY: Main response generation from documents
|
|
# French AI model with excellent multilingual capabilities
|
|
- id: -1
|
|
name: "Mistral NeMo 12B (Local)"
|
|
provider: "OLLAMA"
|
|
model_name: "mistral-nemo:128k"
|
|
api_key: ""
|
|
api_base: "http://localhost:11434"
|
|
language: "auto"
|
|
system_prompt: "You are a knowledgeable research assistant. Answer questions accurately based on the provided documents. If the documents don't contain the answer, clearly state that. Always maintain context and cite sources when possible."
|
|
litellm_params:
|
|
temperature: 0.7
|
|
max_tokens: 8000
|
|
|
|
# TildeOpen 30B - GRAMMAR CHECKER: Latvian language quality control
|
|
# Latvian AI model specifically for grammar correction
|
|
- id: -2
|
|
name: "TildeOpen 30B (Grammar Checker)"
|
|
provider: "OLLAMA"
|
|
model_name: "tildeopen:latest"
|
|
api_key: ""
|
|
api_base: "http://localhost:11434"
|
|
language: "Latvian"
|
|
system_prompt: "You are a Latvian grammar correction assistant. Your ONLY task is to correct grammatical errors in Latvian text while preserving the original meaning, style, and formatting. Do not add explanations, commentary, or change the content. Only fix grammar mistakes."
|
|
litellm_params:
|
|
temperature: 0.3
|
|
max_tokens: 8000
|
|
|
|
# Google Gemini 2.0 Flash - FALLBACK: Emergency only when local models fail
|
|
# Only used when Mistral NeMo cannot generate adequate responses
|
|
- id: -3
|
|
name: "Gemini 2.0 Flash (API Fallback)"
|
|
provider: "GOOGLE"
|
|
model_name: "gemini-2.0-flash-exp"
|
|
api_key: "${GEMINI_API_KEY}"
|
|
api_base: ""
|
|
language: "auto"
|
|
system_prompt: "You are a helpful AI assistant. Provide accurate, well-researched answers based on available information."
|
|
litellm_params:
|
|
temperature: 0.7
|
|
max_tokens: 8000
|