mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-07-24 23:41:10 +02:00
Address PR #8 review feedback
Security & Configuration fixes: - Remove sensitive files exposing production infrastructure details - Add removed files to .gitignore to prevent re-addition - Make uploads directory configurable via UPLOADS_DIR env var - Improve URL validation regex to require valid domain structure (rejects invalid URLs like 'http://a' while supporting international chars) Files removed: DEPLOYMENT.md, claude.md, sync-from-production.sh, MIGRATION_LOCAL_LLM.md, PR_DESCRIPTION.md
This commit is contained in:
Claude
parent
9248dd8b35
commit
f984d85f02
9 changed files with 14 additions and 3128 deletions
7
.gitignore
vendored
7
.gitignore
vendored
|
|
@ -58,3 +58,10 @@ tmp/
|
||||||
pyproject.toml.backup*
|
pyproject.toml.backup*
|
||||||
security_audit_baseline*.json
|
security_audit_baseline*.json
|
||||||
installed_before_update.txt
|
installed_before_update.txt
|
||||||
|
|
||||||
|
# Deployment and production documentation (store privately)
|
||||||
|
DEPLOYMENT.md
|
||||||
|
claude.md
|
||||||
|
sync-from-production.sh
|
||||||
|
MIGRATION_LOCAL_LLM.md
|
||||||
|
PR_DESCRIPTION.md
|
||||||
|
|
|
||||||
376
DEPLOYMENT.md
376
DEPLOYMENT.md
|
|
@ -1,376 +0,0 @@
|
||||||
# SurfSense Production Deployment Guide
|
|
||||||
|
|
||||||
**Server**: ai.kapteinis.lv
|
|
||||||
**Date**: November 17, 2025
|
|
||||||
**Branch**: nightly
|
|
||||||
**Environment**: Debian VPS with local Ollama LLMs
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 📋 Pre-Deployment Checklist
|
|
||||||
|
|
||||||
✅ **Code Changes Committed:**
|
|
||||||
- Minimal privacy-focused frontend UI
|
|
||||||
- Email/password authentication only
|
|
||||||
- Google Analytics removed
|
|
||||||
- Anthropic Claude removed from examples
|
|
||||||
- Social media: Mastodon, Pixelfed, Bookwyrm only
|
|
||||||
|
|
||||||
✅ **GitHub Status:**
|
|
||||||
- All changes pushed to `nightly` branch
|
|
||||||
- Repository: https://github.com/okapteinis/SurfSense
|
|
||||||
|
|
||||||
✅ **Configuration Verified:**
|
|
||||||
- Mistral NeMo 128K context window fix applied
|
|
||||||
- TildeOpen 30B grammar checker configured
|
|
||||||
- Gemini API fallback configured
|
|
||||||
- No secrets committed to repository
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🚀 Deployment Steps
|
|
||||||
|
|
||||||
### Option 1: Automated Deployment (Recommended)
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# On your VPS (ai.kapteinis.lv)
|
|
||||||
ssh your-user@ai.kapteinis.lv
|
|
||||||
|
|
||||||
# Create deployment script
|
|
||||||
cat > /opt/SurfSense/deploy.sh << 'EOF'
|
|
||||||
#!/bin/bash
|
|
||||||
set -e
|
|
||||||
|
|
||||||
echo "🚀 SurfSense Deployment to ai.kapteinis.lv"
|
|
||||||
echo "==========================================="
|
|
||||||
|
|
||||||
# Backup current state
|
|
||||||
BACKUP_DIR="/opt/SurfSense/backups/$(date +%Y%m%d_%H%M%S)"
|
|
||||||
mkdir -p "$BACKUP_DIR"
|
|
||||||
cp -r /opt/SurfSense/surfsense_web "$BACKUP_DIR/"
|
|
||||||
echo "✅ Backup created: $BACKUP_DIR"
|
|
||||||
|
|
||||||
# Pull latest code
|
|
||||||
cd /opt/SurfSense
|
|
||||||
git fetch origin
|
|
||||||
git checkout nightly
|
|
||||||
git pull origin nightly
|
|
||||||
echo "✅ Code updated from GitHub"
|
|
||||||
|
|
||||||
# Install and build frontend
|
|
||||||
cd /opt/SurfSense/surfsense_web
|
|
||||||
pnpm install
|
|
||||||
pnpm build
|
|
||||||
echo "✅ Frontend built"
|
|
||||||
|
|
||||||
# Restart services
|
|
||||||
sudo systemctl restart surfsense
|
|
||||||
sudo systemctl restart surfsense-frontend
|
|
||||||
sudo systemctl restart surfsense-celery || true
|
|
||||||
sudo systemctl restart surfsense-celery-beat || true
|
|
||||||
echo "✅ Services restarted"
|
|
||||||
|
|
||||||
# Verify
|
|
||||||
sleep 5
|
|
||||||
echo ""
|
|
||||||
echo "Service Status:"
|
|
||||||
systemctl is-active surfsense && echo " ✅ Backend: Running"
|
|
||||||
systemctl is-active surfsense-frontend && echo " ✅ Frontend: Running"
|
|
||||||
systemctl is-active ollama && echo " ✅ Ollama: Running"
|
|
||||||
|
|
||||||
echo ""
|
|
||||||
echo "🎉 Deployment complete!"
|
|
||||||
echo "🔗 Visit: https://ai.kapteinis.lv"
|
|
||||||
EOF
|
|
||||||
|
|
||||||
chmod +x /opt/SurfSense/deploy.sh
|
|
||||||
|
|
||||||
# Run deployment
|
|
||||||
/opt/SurfSense/deploy.sh
|
|
||||||
```
|
|
||||||
|
|
||||||
### Option 2: Manual Deployment
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# 1. SSH to server
|
|
||||||
ssh your-user@ai.kapteinis.lv
|
|
||||||
|
|
||||||
# 2. Navigate to SurfSense directory
|
|
||||||
cd /opt/SurfSense
|
|
||||||
|
|
||||||
# 3. Backup current installation
|
|
||||||
mkdir -p backups/$(date +%Y%m%d_%H%M%S)
|
|
||||||
cp -r surfsense_web backups/$(date +%Y%m%d_%H%M%S)/
|
|
||||||
|
|
||||||
# 4. Pull latest changes
|
|
||||||
git fetch origin
|
|
||||||
git checkout nightly
|
|
||||||
git pull origin nightly
|
|
||||||
|
|
||||||
# 5. Install frontend dependencies
|
|
||||||
cd surfsense_web
|
|
||||||
pnpm install
|
|
||||||
|
|
||||||
# 6. Build frontend
|
|
||||||
pnpm build
|
|
||||||
|
|
||||||
# 7. Restart services
|
|
||||||
sudo systemctl restart surfsense
|
|
||||||
sudo systemctl restart surfsense-frontend
|
|
||||||
sudo systemctl restart surfsense-celery
|
|
||||||
sudo systemctl restart surfsense-celery-beat
|
|
||||||
|
|
||||||
# 8. Verify services are running
|
|
||||||
systemctl status surfsense
|
|
||||||
systemctl status surfsense-frontend
|
|
||||||
systemctl status ollama
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## ✅ Post-Deployment Verification
|
|
||||||
|
|
||||||
### 1. Check Services Status
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# All services should show "active (running)"
|
|
||||||
sudo systemctl status surfsense
|
|
||||||
sudo systemctl status surfsense-frontend
|
|
||||||
sudo systemctl status ollama
|
|
||||||
sudo systemctl status surfsense-celery
|
|
||||||
```
|
|
||||||
|
|
||||||
### 2. Verify UI Changes
|
|
||||||
|
|
||||||
Visit https://ai.kapteinis.lv and verify:
|
|
||||||
|
|
||||||
**Homepage Should Show:**
|
|
||||||
- ✅ SurfSense logo and theme toggle only (no navigation links)
|
|
||||||
- ✅ "Let's Start Surfing" heading
|
|
||||||
- ✅ Tagline paragraph
|
|
||||||
- ✅ Hero screenshot/demo
|
|
||||||
- ✅ Footer with SurfSense name and social links
|
|
||||||
|
|
||||||
**Homepage Should NOT Show:**
|
|
||||||
- ❌ "Get Started" button
|
|
||||||
- ❌ "Pricing" link
|
|
||||||
- ❌ "Docs" link
|
|
||||||
- ❌ Discord/GitHub icons
|
|
||||||
- ❌ "Sign In" button in navbar
|
|
||||||
- ❌ Feature cards or integrations sections
|
|
||||||
|
|
||||||
**Login Page Should Show:**
|
|
||||||
- ✅ Email and password fields only
|
|
||||||
- ✅ "Sign In" button
|
|
||||||
- ✅ Link to register page
|
|
||||||
|
|
||||||
**Login Page Should NOT Show:**
|
|
||||||
- ❌ "Continue with Google" button
|
|
||||||
- ❌ Any OAuth options
|
|
||||||
|
|
||||||
### 3. Test Authentication
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Test login endpoint
|
|
||||||
curl -X POST https://ai.kapteinis.lv/api/v1/auth/login \
|
|
||||||
-H "Content-Type: application/json" \
|
|
||||||
-d '{"email":"test@example.com","password":"testpass"}'
|
|
||||||
```
|
|
||||||
|
|
||||||
### 4. Test LLM Backend
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Check Ollama is running
|
|
||||||
curl http://localhost:11434/api/tags
|
|
||||||
|
|
||||||
# Should show mistral-nemo:128k and tildeopen models
|
|
||||||
```
|
|
||||||
|
|
||||||
### 5. Check Logs
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Backend logs
|
|
||||||
journalctl -u surfsense -n 100 -f
|
|
||||||
|
|
||||||
# Frontend logs
|
|
||||||
journalctl -u surfsense-frontend -n 100 -f
|
|
||||||
|
|
||||||
# Ollama logs
|
|
||||||
journalctl -u ollama -n 100 -f
|
|
||||||
```
|
|
||||||
|
|
||||||
Look for:
|
|
||||||
- ✅ No errors during startup
|
|
||||||
- ✅ "Context window=131072" in logs (Mistral NeMo)
|
|
||||||
- ✅ Successful model loading messages
|
|
||||||
- ✅ No Google Analytics references
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🔧 Troubleshooting
|
|
||||||
|
|
||||||
### Frontend Build Fails
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Clear cache and rebuild
|
|
||||||
cd /opt/SurfSense/surfsense_web
|
|
||||||
rm -rf .next node_modules
|
|
||||||
pnpm install
|
|
||||||
pnpm build
|
|
||||||
```
|
|
||||||
|
|
||||||
### Services Won't Start
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Check for errors
|
|
||||||
journalctl -u surfsense -n 50
|
|
||||||
journalctl -u surfsense-frontend -n 50
|
|
||||||
|
|
||||||
# Verify ports are available
|
|
||||||
sudo lsof -i :8000 # Backend
|
|
||||||
sudo lsof -i :3000 # Frontend
|
|
||||||
sudo lsof -i :11434 # Ollama
|
|
||||||
```
|
|
||||||
|
|
||||||
### Ollama Models Missing
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# List installed models
|
|
||||||
ollama list
|
|
||||||
|
|
||||||
# Re-pull if needed
|
|
||||||
ollama pull mistral-nemo
|
|
||||||
ollama create mistral-nemo:128k -f /path/to/mistral-nemo-128k.modelfile
|
|
||||||
ollama pull tildeopen:30b-q5_k_m
|
|
||||||
```
|
|
||||||
|
|
||||||
### Google Analytics Still Showing
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Verify layout.tsx doesn't have GA
|
|
||||||
grep -r "GoogleAnalytics\|google-analytics\|gtag" /opt/SurfSense/surfsense_web/app/
|
|
||||||
|
|
||||||
# Should return nothing
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 📊 Performance Monitoring
|
|
||||||
|
|
||||||
### After Deployment, Monitor:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# RAM usage (should be 20-25GB during inference)
|
|
||||||
free -h
|
|
||||||
|
|
||||||
# CPU usage
|
|
||||||
htop
|
|
||||||
|
|
||||||
# Disk space
|
|
||||||
df -h
|
|
||||||
|
|
||||||
# Response times
|
|
||||||
curl -w "@-" -o /dev/null -s https://ai.kapteinis.lv << 'EOF'
|
|
||||||
time_total: %{time_total}s
|
|
||||||
EOF
|
|
||||||
```
|
|
||||||
|
|
||||||
### Expected Performance:
|
|
||||||
|
|
||||||
- **English queries**: ~5 seconds
|
|
||||||
- **Latvian queries**: ~23 seconds (with grammar check)
|
|
||||||
- **RAM usage**: 13-15GB idle, 20-25GB during inference
|
|
||||||
- **Disk usage**: ~28GB for Ollama models
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🔄 Rollback Procedure
|
|
||||||
|
|
||||||
If something goes wrong:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Find backup
|
|
||||||
ls -lt /opt/SurfSense/backups/
|
|
||||||
|
|
||||||
# Restore from backup
|
|
||||||
cd /opt/SurfSense
|
|
||||||
mv surfsense_web surfsense_web.failed
|
|
||||||
cp -r backups/YYYYMMDD_HHMMSS/surfsense_web .
|
|
||||||
|
|
||||||
# Restart services
|
|
||||||
sudo systemctl restart surfsense-frontend
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 📝 Production Configuration
|
|
||||||
|
|
||||||
### Environment Variables
|
|
||||||
|
|
||||||
Location: `/opt/SurfSense/surfsense_backend/.env`
|
|
||||||
|
|
||||||
**Required:**
|
|
||||||
```bash
|
|
||||||
GEMINI_API_KEY=your_key_here
|
|
||||||
OLLAMA_BASE_URL=http://localhost:11434
|
|
||||||
SECRET_KEY=your_secret_key
|
|
||||||
DATABASE_URL=postgresql://...
|
|
||||||
REDIS_URL=redis://localhost:6379
|
|
||||||
```
|
|
||||||
|
|
||||||
**Not Required (Removed):**
|
|
||||||
```bash
|
|
||||||
# GOOGLE_OAUTH_CLIENT_ID # OAuth disabled
|
|
||||||
# GOOGLE_OAUTH_CLIENT_SECRET # OAuth disabled
|
|
||||||
# ANTHROPIC_API_KEY # Not using Claude
|
|
||||||
```
|
|
||||||
|
|
||||||
### LLM Configuration
|
|
||||||
|
|
||||||
Location: `/opt/SurfSense/surfsense_backend/app/config/global_llm_config.yaml`
|
|
||||||
|
|
||||||
**Three-tier architecture:**
|
|
||||||
1. Mistral NeMo 12B (primary)
|
|
||||||
2. TildeOpen 30B (Latvian grammar)
|
|
||||||
3. Gemini 2.0 Flash (fallback)
|
|
||||||
|
|
||||||
See `global_llm_config.yaml.template` for structure.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 🎯 Success Criteria
|
|
||||||
|
|
||||||
Deployment is successful when:
|
|
||||||
|
|
||||||
- ✅ Homepage shows minimal UI (logo + tagline only)
|
|
||||||
- ✅ No navigation links visible (Pricing, Docs removed)
|
|
||||||
- ✅ Login page shows email/password form only
|
|
||||||
- ✅ No Google OAuth button
|
|
||||||
- ✅ Footer shows only Mastodon, Pixelfed, Bookwyrm links
|
|
||||||
- ✅ Services all running (backend, frontend, Ollama)
|
|
||||||
- ✅ English queries respond in ~5 seconds
|
|
||||||
- ✅ Latvian queries respond in ~23 seconds
|
|
||||||
- ✅ No errors in logs
|
|
||||||
- ✅ RAM usage normal (13-25GB)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 📞 Support
|
|
||||||
|
|
||||||
**Issues:** https://github.com/okapteinis/SurfSense/issues
|
|
||||||
**Email:** ojars@kapteinis.lv
|
|
||||||
**Deployment Date:** November 17, 2025
|
|
||||||
**Version:** nightly branch (commit 209cd24)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 📚 Related Documentation
|
|
||||||
|
|
||||||
- `INSTALLATION_LOCAL_LLM.md` - Complete Ollama setup guide
|
|
||||||
- `MIGRATION_LOCAL_LLM.md` - Architecture and migration details
|
|
||||||
- `PR_DESCRIPTION.md` - Full PR documentation
|
|
||||||
- `claude.md` - Security audit report
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
**Deployment Complete!** Your minimal, privacy-focused SurfSense instance with local European AI is ready. 🎉
|
|
||||||
|
|
@ -1,236 +0,0 @@
|
||||||
# Local LLM Migration Documentation
|
|
||||||
|
|
||||||
## Date
|
|
||||||
November 17, 2025
|
|
||||||
|
|
||||||
## Summary
|
|
||||||
Migrated SurfSense from Gemini-only API to three-tier local-first European AI architecture.
|
|
||||||
|
|
||||||
## Architecture Changes
|
|
||||||
|
|
||||||
### Previous Architecture
|
|
||||||
- **Primary**: Gemini 2.0 Flash API only
|
|
||||||
- **Issues**:
|
|
||||||
- Rate limits causing service interruptions
|
|
||||||
- High API costs
|
|
||||||
- Timeout errors on complex queries
|
|
||||||
- 30-120 second response times
|
|
||||||
|
|
||||||
### New Architecture
|
|
||||||
|
|
||||||
#### Tier 1 - Primary LLM: Mistral NeMo 12B (France)
|
|
||||||
- **Model**: `ollama/mistral-nemo:128k`
|
|
||||||
- **Size**: 7.1GB
|
|
||||||
- **Context Window**: 128K tokens (131,072)
|
|
||||||
- **Purpose**: Generate answers from user documents using RAG
|
|
||||||
- **Performance**: 5 seconds for English queries, 23 seconds for Latvian queries
|
|
||||||
- **Location**: Local via Ollama at http://localhost:11434
|
|
||||||
- **Why chosen**: Fast CPU inference, 50% smaller than Mistral Small 24B, eliminates timeout errors
|
|
||||||
|
|
||||||
#### Tier 2 - Grammar Checker: TildeOpen 30B (Latvia)
|
|
||||||
- **Model**: `ollama/tildeopen:30b-q5_k_m`
|
|
||||||
- **Size**: 21GB
|
|
||||||
- **Purpose**: Check and correct Latvian grammar in generated responses
|
|
||||||
- **Performance**: 8 second timeout for grammar checking
|
|
||||||
- **Location**: Local via Ollama at http://localhost:11434
|
|
||||||
- **Special**: Best Latvian language model, 41% better than LLaMA-3, 24% better than GPT-4o for Latvian
|
|
||||||
|
|
||||||
#### Tier 3 - API Fallback: Gemini 2.0 Flash (Google)
|
|
||||||
- **Model**: `gemini/gemini-2.0-flash-exp`
|
|
||||||
- **Purpose**: Emergency fallback when local models cannot answer
|
|
||||||
- **Location**: Google API with GEMINI_API_KEY
|
|
||||||
- **Cost**: Only used for 5% of queries, dramatically reducing API costs and rate limit issues
|
|
||||||
|
|
||||||
## Code Changes
|
|
||||||
|
|
||||||
### Backend Files Modified
|
|
||||||
|
|
||||||
#### 1. `/surfsense_backend/app/agents/researcher/utils.py`
|
|
||||||
**Changes**:
|
|
||||||
- Added context window override for mistral-nemo models
|
|
||||||
- Fixed LiteLLM incorrect reporting (was showing 1,024,000 tokens instead of 128K)
|
|
||||||
- Correctly limits document context to 128K tokens
|
|
||||||
|
|
||||||
**Key Function Modified**:
|
|
||||||
```python
|
|
||||||
def get_model_context_window(model_name: str) -> int:
|
|
||||||
"""Get the total context window size for a model."""
|
|
||||||
# Override for Ollama models with known incorrect LiteLLM values
|
|
||||||
if "mistral-nemo" in model_name.lower():
|
|
||||||
return 131072 # Mistral NeMo actual context window: 128K tokens
|
|
||||||
# ... rest of function
|
|
||||||
```
|
|
||||||
|
|
||||||
**Why**: LiteLLM was incorrectly reporting 1M token context window, causing backend to send 530K tokens which exceeded Ollama's 4K default, resulting in massive truncation and query failures.
|
|
||||||
|
|
||||||
#### 2. `/surfsense_backend/app/utils/document_converters.py`
|
|
||||||
**Changes**:
|
|
||||||
- Added same context window detection override
|
|
||||||
- Automatic document truncation to fit context
|
|
||||||
|
|
||||||
**Purpose**: Ensures documents are properly sized before sending to LLM, preventing timeout errors.
|
|
||||||
|
|
||||||
#### 3. `/surfsense_backend/app/config/global_llm_config.yaml.template`
|
|
||||||
**Changes**:
|
|
||||||
- Added three-tier model configuration
|
|
||||||
- Configured Ollama endpoints (http://localhost:11434)
|
|
||||||
- Set fallback chain: Mistral NeMo → TildeOpen (for Latvian) → Gemini (emergency)
|
|
||||||
- Uses `${GEMINI_API_KEY}` placeholder instead of actual key
|
|
||||||
|
|
||||||
**Security**: Real config file with API key is gitignored, only template is committed.
|
|
||||||
|
|
||||||
### Frontend Files Modified
|
|
||||||
|
|
||||||
#### 1. `/surfsense_frontend/app/layout.tsx`
|
|
||||||
**Changes**:
|
|
||||||
- Removed Google Analytics tracking code
|
|
||||||
- Cleaned up GTM (Google Tag Manager) references
|
|
||||||
- Removed unnecessary external analytics dependencies
|
|
||||||
|
|
||||||
**Why**: Improved privacy and reduced external dependencies.
|
|
||||||
|
|
||||||
## Configuration
|
|
||||||
|
|
||||||
### Required Environment Variables
|
|
||||||
```bash
|
|
||||||
# Required for fallback API
|
|
||||||
GEMINI_API_KEY=your_gemini_api_key_here
|
|
||||||
|
|
||||||
# Ollama endpoint
|
|
||||||
OLLAMA_BASE_URL=http://localhost:11434
|
|
||||||
```
|
|
||||||
|
|
||||||
### Ollama Models Required
|
|
||||||
```bash
|
|
||||||
# Pull base Mistral NeMo model
|
|
||||||
ollama pull mistral-nemo
|
|
||||||
|
|
||||||
# Create optimized version with 128K context
|
|
||||||
ollama create mistral-nemo:128k -f mistral-nemo-128k.modelfile
|
|
||||||
|
|
||||||
# Pull Latvian grammar checker
|
|
||||||
ollama pull tildeopen:30b-q5_k_m
|
|
||||||
```
|
|
||||||
|
|
||||||
### Mistral NeMo Model Configuration
|
|
||||||
Create `mistral-nemo-128k.modelfile`:
|
|
||||||
```
|
|
||||||
FROM mistral-nemo:latest
|
|
||||||
|
|
||||||
# Set context window to 128K tokens (Mistral NeMo maximum)
|
|
||||||
PARAMETER num_ctx 131072
|
|
||||||
|
|
||||||
# Keep other parameters optimized for RAG
|
|
||||||
PARAMETER temperature 0.7
|
|
||||||
PARAMETER top_p 0.9
|
|
||||||
PARAMETER top_k 40
|
|
||||||
```
|
|
||||||
|
|
||||||
Then create the model:
|
|
||||||
```bash
|
|
||||||
ollama create mistral-nemo:128k -f mistral-nemo-128k.modelfile
|
|
||||||
```
|
|
||||||
|
|
||||||
## Benefits
|
|
||||||
|
|
||||||
### Cost Reduction
|
|
||||||
- **95% reduction in API costs**
|
|
||||||
- Only 5% of queries hit Gemini API (fallback only)
|
|
||||||
- Unlimited local usage with no per-token charges
|
|
||||||
|
|
||||||
### Performance Improvements
|
|
||||||
- **Response times**: 5-25 seconds (down from 30-120 seconds)
|
|
||||||
- **No rate limits**: Can handle unlimited concurrent users
|
|
||||||
- **No timeouts**: 128K context properly configured
|
|
||||||
- **Better accuracy**: Documents no longer truncated to 4K
|
|
||||||
|
|
||||||
### Privacy & Security
|
|
||||||
- **Complete data privacy**: Documents never leave your server (unless fallback triggered)
|
|
||||||
- **GDPR compliant**: All processing on EU servers
|
|
||||||
- **No third-party tracking**: Removed Google Analytics
|
|
||||||
|
|
||||||
### Language Quality
|
|
||||||
- **Latvian language**: 41% better than LLaMA-3, 24% better than GPT-4o
|
|
||||||
- **Grammar correction**: Dedicated Latvian model (TildeOpen)
|
|
||||||
- **Multilingual**: Mistral NeMo supports 50+ languages
|
|
||||||
|
|
||||||
## Performance Metrics
|
|
||||||
|
|
||||||
### Response Times
|
|
||||||
- **English queries**: ~5 seconds (Mistral NeMo only)
|
|
||||||
- **Latvian queries**: ~23 seconds (Mistral NeMo + TildeOpen grammar check)
|
|
||||||
- **Complex queries**: Falls back to Gemini if needed
|
|
||||||
|
|
||||||
### Resource Usage
|
|
||||||
- **RAM usage**: 20-25GB during inference, 13-15GB idle
|
|
||||||
- **Disk usage**: 28GB for both models (7GB + 21GB)
|
|
||||||
- **CPU**: High during inference, normal otherwise
|
|
||||||
|
|
||||||
### Accuracy
|
|
||||||
- **Document retrieval**: 4,338 documents processed
|
|
||||||
- **Token optimization**: 530K tokens in context (properly handled)
|
|
||||||
- **No truncation**: Full 128K context utilized
|
|
||||||
|
|
||||||
## Known Issues & Solutions
|
|
||||||
|
|
||||||
### Issue 1: LiteLLM Context Window Bug
|
|
||||||
**Problem**: LiteLLM reports 1,024,000 tokens for mistral-nemo when actual limit is 131,072.
|
|
||||||
|
|
||||||
**Solution**: Added manual override in `get_model_context_window()` function to return correct value.
|
|
||||||
|
|
||||||
### Issue 2: Ollama Default Context
|
|
||||||
**Problem**: Ollama defaults to 4,096 token context, causing massive truncation.
|
|
||||||
|
|
||||||
**Solution**: Created custom model `mistral-nemo:128k` with `num_ctx` parameter set to 131,072.
|
|
||||||
|
|
||||||
### Issue 3: Frontend Model Name
|
|
||||||
**Problem**: Frontend showed "Mistral Small 24B (Local)" instead of "Mistral NeMo 12B".
|
|
||||||
|
|
||||||
**Solution**: Updated display name in `global_llm_config.yaml` to reflect actual model.
|
|
||||||
|
|
||||||
## Deployment History
|
|
||||||
|
|
||||||
### Production Server: ai.kapteinis.lv
|
|
||||||
- **Date**: November 17, 2025
|
|
||||||
- **Server**: Debian 6.12.57, 32GB RAM, CPU-only
|
|
||||||
- **Location**: /opt/SurfSense
|
|
||||||
- **Status**: Successfully deployed and tested
|
|
||||||
|
|
||||||
### Testing Results
|
|
||||||
- ✅ English queries: 5 second response time
|
|
||||||
- ✅ Latvian queries: 23 second response time (with grammar check)
|
|
||||||
- ✅ No timeout errors
|
|
||||||
- ✅ No rate limit errors
|
|
||||||
- ✅ 95% cost reduction confirmed
|
|
||||||
- ✅ Improved answer quality
|
|
||||||
- ✅ Full 128K context utilized
|
|
||||||
|
|
||||||
## Migration Steps Summary
|
|
||||||
|
|
||||||
1. Installed Ollama service
|
|
||||||
2. Downloaded Mistral NeMo 12B model
|
|
||||||
3. Downloaded TildeOpen 30B model
|
|
||||||
4. Created mistral-nemo:128k with proper context window
|
|
||||||
5. Updated backend configuration YAML
|
|
||||||
6. Patched Python code for context window handling
|
|
||||||
7. Removed Google Analytics from frontend
|
|
||||||
8. Rebuilt frontend with pnpm build
|
|
||||||
9. Restarted all services
|
|
||||||
10. Verified functionality with test queries
|
|
||||||
|
|
||||||
## Authors
|
|
||||||
- **Ojārs Kapteiņš** <ojars@kapteinis.lv> - Implementation
|
|
||||||
- **Claude AI Assistant** (Anthropic) - Architecture design and debugging
|
|
||||||
|
|
||||||
## References
|
|
||||||
- Mistral NeMo: https://mistral.ai/news/mistral-nemo/
|
|
||||||
- TildeOpen: https://tilde.ai/tildeopen
|
|
||||||
- Ollama: https://ollama.com/
|
|
||||||
- LiteLLM: https://github.com/BerriAI/litellm
|
|
||||||
|
|
||||||
## License
|
|
||||||
This implementation is part of SurfSense and follows the same license terms.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
**100% European AI Solution**: Mistral AI (France) for primary intelligence, Tilde AI (Latvia) for grammar expertise, with Google (USA) only for emergencies. This architecture represents cutting-edge deployment of European AI technology for production use at enterprise scale.
|
|
||||||
|
|
@ -1,275 +0,0 @@
|
||||||
# Local LLM Implementation - Nightly to Main
|
|
||||||
|
|
||||||
## Overview
|
|
||||||
This PR implements a local-first European AI architecture for SurfSense, replacing the Gemini-only API approach with a three-tier system that dramatically reduces costs, improves performance, and enhances privacy.
|
|
||||||
|
|
||||||
## 🎯 Problem Statement
|
|
||||||
|
|
||||||
### Previous Issues
|
|
||||||
- **High API Costs**: Every query hit Gemini API, resulting in substantial monthly costs
|
|
||||||
- **Rate Limits**: Frequent 429 errors during peak usage
|
|
||||||
- **Slow Response Times**: 30-120 seconds per query due to API latency
|
|
||||||
- **Timeout Errors**: Complex queries with large document sets failed
|
|
||||||
- **Privacy Concerns**: All user data sent to external APIs
|
|
||||||
- **Vendor Lock-in**: Complete dependency on Google's API availability
|
|
||||||
|
|
||||||
## 🚀 Solution: Three-Tier European AI Architecture
|
|
||||||
|
|
||||||
### Tier 1 - Primary LLM: Mistral NeMo 12B (🇫🇷 France)
|
|
||||||
- **Model**: `ollama/mistral-nemo:128k`
|
|
||||||
- **Purpose**: Generate answers from user documents using RAG
|
|
||||||
- **Performance**: 5-10 second response time
|
|
||||||
- **Context**: Full 128K tokens (131,072) for large document sets
|
|
||||||
- **Location**: Local inference via Ollama
|
|
||||||
|
|
||||||
### Tier 2 - Grammar Checker: TildeOpen 30B (🇱🇻 Latvia)
|
|
||||||
- **Model**: `ollama/tildeopen:30b-q5_k_m`
|
|
||||||
- **Purpose**: Latvian grammar correction and quality assurance
|
|
||||||
- **Performance**: Additional 5-10 seconds for Latvian queries
|
|
||||||
- **Quality**: 41% better than LLaMA-3, 24% better than GPT-4o for Latvian
|
|
||||||
- **Location**: Local inference via Ollama
|
|
||||||
|
|
||||||
### Tier 3 - API Fallback: Gemini 2.0 Flash (🇺🇸 Google)
|
|
||||||
- **Model**: `gemini-2.0-flash-exp`
|
|
||||||
- **Purpose**: Emergency fallback only when local models cannot answer
|
|
||||||
- **Usage**: Only ~5% of queries
|
|
||||||
- **Cost**: 95% reduction in API costs
|
|
||||||
|
|
||||||
## 📊 Performance Improvements
|
|
||||||
|
|
||||||
### Response Times
|
|
||||||
| Query Type | Before | After | Improvement |
|
|
||||||
|------------|--------|-------|-------------|
|
|
||||||
| English queries | 30-120s | ~5s | **6-24x faster** |
|
|
||||||
| Latvian queries | 30-120s | ~23s | **1.3-5x faster** |
|
|
||||||
| Complex queries | Often timeout | Reliable | **100% success** |
|
|
||||||
|
|
||||||
### Cost Reduction
|
|
||||||
- **95% reduction** in API costs
|
|
||||||
- From: $XXX/month (all queries via API)
|
|
||||||
- To: $X/month (5% via API)
|
|
||||||
- **Unlimited local usage** with no per-token charges
|
|
||||||
|
|
||||||
### Quality Improvements
|
|
||||||
- **No truncation**: Full 128K context utilized (was limited to 4K)
|
|
||||||
- **Better Latvian**: Dedicated Latvian model instead of generic multilingual
|
|
||||||
- **More accurate**: Documents no longer truncated, full context preserved
|
|
||||||
|
|
||||||
## 🔒 Privacy & Security Enhancements
|
|
||||||
|
|
||||||
- ✅ **Complete data privacy**: 95% of queries never leave your server
|
|
||||||
- ✅ **GDPR compliant**: All primary processing on EU servers
|
|
||||||
- ✅ **No third-party tracking**: Removed Google Analytics from frontend
|
|
||||||
- ✅ **EU sovereignty**: Primary AI from France (Mistral) and Latvia (Tilde)
|
|
||||||
|
|
||||||
## 📝 Technical Details
|
|
||||||
|
|
||||||
### Backend Changes
|
|
||||||
|
|
||||||
#### 1. Context Window Bug Fix (`app/agents/researcher/utils.py`)
|
|
||||||
**Problem**: LiteLLM incorrectly reported 1,024,000 token context for mistral-nemo (actual: 128K)
|
|
||||||
|
|
||||||
**Solution**: Added manual override to return correct 131,072 token context
|
|
||||||
```python
|
|
||||||
def get_model_context_window(model_name: str) -> int:
|
|
||||||
if "mistral-nemo" in model_name.lower():
|
|
||||||
return 131072 # Correct context window
|
|
||||||
# ... rest of implementation
|
|
||||||
```
|
|
||||||
|
|
||||||
**Impact**: Prevents backend from sending 530K tokens to model with 128K limit
|
|
||||||
|
|
||||||
#### 2. Document Converter Update (`app/utils/document_converters.py`)
|
|
||||||
- Same context window fix applied
|
|
||||||
- Ensures proper document truncation before LLM processing
|
|
||||||
|
|
||||||
#### 3. LLM Configuration (`app/config/global_llm_config.yaml.template`)
|
|
||||||
**New three-tier configuration**:
|
|
||||||
```yaml
|
|
||||||
global_llm_configs:
|
|
||||||
# Primary: Mistral NeMo 12B (Local)
|
|
||||||
- model_name: "mistral-nemo:128k"
|
|
||||||
provider: "OLLAMA"
|
|
||||||
api_base: "http://localhost:11434"
|
|
||||||
|
|
||||||
# Grammar: TildeOpen 30B (Local)
|
|
||||||
- model_name: "tildeopen:latest"
|
|
||||||
provider: "OLLAMA"
|
|
||||||
api_base: "http://localhost:11434"
|
|
||||||
|
|
||||||
# Fallback: Gemini 2.0 Flash (API)
|
|
||||||
- model_name: "gemini-2.0-flash-exp"
|
|
||||||
provider: "GOOGLE"
|
|
||||||
api_key: "${GEMINI_API_KEY}"
|
|
||||||
```
|
|
||||||
|
|
||||||
**Security**: Template uses `${GEMINI_API_KEY}` placeholder, real config gitignored
|
|
||||||
|
|
||||||
### Frontend Changes
|
|
||||||
|
|
||||||
#### 1. Analytics Removal (`app/layout.tsx`)
|
|
||||||
- Removed Google Analytics tracking code
|
|
||||||
- Removed GTM (Google Tag Manager) references
|
|
||||||
- Improved privacy and reduced external dependencies
|
|
||||||
|
|
||||||
### Configuration Changes
|
|
||||||
|
|
||||||
#### 1. Updated `.gitignore`
|
|
||||||
Added comprehensive security patterns:
|
|
||||||
- Environment files (`.env`, `.env.local`, etc.)
|
|
||||||
- Config files with API keys
|
|
||||||
- Python cache and virtual environments
|
|
||||||
- Logs, databases, and uploads
|
|
||||||
- SSL certificates and private keys
|
|
||||||
|
|
||||||
## 📦 Files Changed
|
|
||||||
|
|
||||||
### Modified
|
|
||||||
- `surfsense_backend/app/agents/researcher/utils.py` - Context window fix
|
|
||||||
- `surfsense_backend/app/utils/document_converters.py` - Context window fix
|
|
||||||
- `surfsense_frontend/app/layout.tsx` - Removed analytics
|
|
||||||
- `.gitignore` - Enhanced security patterns
|
|
||||||
|
|
||||||
### Added
|
|
||||||
- `surfsense_backend/app/config/global_llm_config.yaml.template` - Secure config template
|
|
||||||
- `MIGRATION_LOCAL_LLM.md` - Complete migration documentation
|
|
||||||
- `INSTALLATION_LOCAL_LLM.md` - Installation guide
|
|
||||||
- `PR_DESCRIPTION.md` - This PR description
|
|
||||||
- `sync-from-production.sh` - Secure rsync script for deployments
|
|
||||||
|
|
||||||
## 🧪 Testing
|
|
||||||
|
|
||||||
Tested on production at **https://ai.kapteinis.lv** with:
|
|
||||||
|
|
||||||
### Test Results
|
|
||||||
- ✅ English queries: 5 second response time
|
|
||||||
- ✅ Latvian queries: 23 second response time (with grammar check)
|
|
||||||
- ✅ Large document sets: 4,338 documents, 530K tokens processed successfully
|
|
||||||
- ✅ No timeout errors
|
|
||||||
- ✅ No rate limit errors
|
|
||||||
- ✅ 95% cost reduction confirmed
|
|
||||||
- ✅ Improved answer quality and accuracy
|
|
||||||
- ✅ Full 128K context properly utilized
|
|
||||||
|
|
||||||
### Load Testing
|
|
||||||
- ✅ Concurrent users: Handled without rate limits
|
|
||||||
- ✅ Peak RAM usage: 20-25GB during inference
|
|
||||||
- ✅ Idle RAM usage: 13-15GB
|
|
||||||
- ✅ Response consistency: Stable performance across queries
|
|
||||||
|
|
||||||
## 📋 Installation Requirements
|
|
||||||
|
|
||||||
### New Dependencies
|
|
||||||
- **Ollama**: Local LLM inference server
|
|
||||||
- **Disk Space**: Additional 28GB for models (7GB + 21GB)
|
|
||||||
- **RAM**: 32GB recommended (24GB minimum)
|
|
||||||
- **Environment Variable**: `OLLAMA_BASE_URL=http://localhost:11434`
|
|
||||||
|
|
||||||
### Installation Steps
|
|
||||||
See `INSTALLATION_LOCAL_LLM.md` for complete guide:
|
|
||||||
1. Install Ollama
|
|
||||||
2. Download models (mistral-nemo, tildeopen)
|
|
||||||
3. Create mistral-nemo:128k with proper context window
|
|
||||||
4. Update backend configuration
|
|
||||||
5. Apply code patches
|
|
||||||
6. Restart services
|
|
||||||
|
|
||||||
## ⚠️ Breaking Changes
|
|
||||||
|
|
||||||
### Required Changes
|
|
||||||
- **Ollama must be installed** and running on port 11434
|
|
||||||
- **Models must be downloaded**: 28GB total (mistral-nemo:128k, tildeopen:30b-q5_k_m)
|
|
||||||
- **Backend code patches** must be applied (context window functions)
|
|
||||||
- **Environment variable** `OLLAMA_BASE_URL` must be set
|
|
||||||
|
|
||||||
### Migration Path
|
|
||||||
Existing deployments can upgrade incrementally:
|
|
||||||
1. Install Ollama alongside existing setup
|
|
||||||
2. Download models in background
|
|
||||||
3. Update configuration to add local models as primary
|
|
||||||
4. Gemini automatically becomes fallback
|
|
||||||
5. Monitor and adjust as needed
|
|
||||||
|
|
||||||
**No downtime required** - fallback ensures continuous operation during migration.
|
|
||||||
|
|
||||||
## 🐛 Known Issues & Solutions
|
|
||||||
|
|
||||||
### Issue 1: Ollama Default Context Window
|
|
||||||
**Problem**: Ollama defaults to 4K context, causing truncation
|
|
||||||
|
|
||||||
**Solution**: Create custom model with `num_ctx 131072` parameter
|
|
||||||
|
|
||||||
### Issue 2: LiteLLM Context Detection
|
|
||||||
**Problem**: LiteLLM reports incorrect context window sizes
|
|
||||||
|
|
||||||
**Solution**: Manual override in backend code (implemented in this PR)
|
|
||||||
|
|
||||||
### Issue 3: Frontend Model Name
|
|
||||||
**Problem**: UI showed "Mistral Small 24B" instead of "Mistral NeMo 12B"
|
|
||||||
|
|
||||||
**Solution**: Updated display name in configuration (included in this PR)
|
|
||||||
|
|
||||||
## 📖 Documentation
|
|
||||||
|
|
||||||
### Added Documentation
|
|
||||||
- **MIGRATION_LOCAL_LLM.md**: Complete architecture explanation and migration history
|
|
||||||
- **INSTALLATION_LOCAL_LLM.md**: Step-by-step installation guide with troubleshooting
|
|
||||||
- **PR_DESCRIPTION.md**: This detailed PR description
|
|
||||||
|
|
||||||
### Updated Documentation
|
|
||||||
- **.gitignore**: Comprehensive security patterns documented
|
|
||||||
- **README updates**: (Recommend adding link to new docs in main README)
|
|
||||||
|
|
||||||
## 🔄 Deployment History
|
|
||||||
|
|
||||||
### Production Server: ai.kapteinis.lv
|
|
||||||
- **Date**: November 17, 2025
|
|
||||||
- **Server**: Debian 6.12.57, 32GB RAM, CPU-only
|
|
||||||
- **Status**: ✅ Successfully deployed and operational
|
|
||||||
- **Uptime**: Stable with zero downtime during migration
|
|
||||||
|
|
||||||
## 👥 Authors & Contributors
|
|
||||||
|
|
||||||
- **Ojārs Kapteiņš** (@okapteinis) - Implementation and deployment
|
|
||||||
- **Claude AI Assistant** (Anthropic) - Architecture design and debugging assistance
|
|
||||||
|
|
||||||
## 🔗 References
|
|
||||||
|
|
||||||
- **Mistral NeMo**: https://mistral.ai/news/mistral-nemo/
|
|
||||||
- **TildeOpen**: https://tilde.ai/tildeopen
|
|
||||||
- **Ollama**: https://ollama.com/
|
|
||||||
- **LiteLLM**: https://github.com/BerriAI/litellm
|
|
||||||
|
|
||||||
## ✅ Checklist
|
|
||||||
|
|
||||||
- [x] Code changes implemented and tested
|
|
||||||
- [x] Documentation created (migration + installation guides)
|
|
||||||
- [x] Security review completed (no secrets in repository)
|
|
||||||
- [x] Production deployment successful
|
|
||||||
- [x] Performance metrics validated
|
|
||||||
- [x] Cost reduction confirmed (95%)
|
|
||||||
- [x] .gitignore updated with security patterns
|
|
||||||
- [x] Config templates created with placeholders
|
|
||||||
- [x] Frontend cleaned of external tracking
|
|
||||||
- [x] Backward compatibility maintained (Gemini fallback)
|
|
||||||
|
|
||||||
## 🎉 Summary
|
|
||||||
|
|
||||||
This PR represents a **fundamental architectural improvement** for SurfSense:
|
|
||||||
|
|
||||||
- 💰 **95% cost reduction** through local-first approach
|
|
||||||
- ⚡ **6-24x faster** response times
|
|
||||||
- 🔒 **Enhanced privacy** with EU-based processing
|
|
||||||
- 🇪🇺 **European AI sovereignty** (Mistral + Tilde)
|
|
||||||
- 🎯 **Better quality** with proper context handling
|
|
||||||
- 📈 **Unlimited scaling** without rate limits
|
|
||||||
|
|
||||||
**This is production-ready** and has been successfully deployed at https://ai.kapteinis.lv since November 17, 2025.
|
|
||||||
|
|
||||||
## 🚀 Ready to Merge
|
|
||||||
|
|
||||||
This PR is ready for review and merge to main. All testing completed successfully, production deployment verified, and documentation comprehensive.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
**Questions or concerns?** Please review the documentation or contact @okapteinis.
|
|
||||||
|
|
@ -1,5 +1,8 @@
|
||||||
DATABASE_URL=postgresql+asyncpg://postgres:postgres@localhost:5432/surfsense
|
DATABASE_URL=postgresql+asyncpg://postgres:postgres@localhost:5432/surfsense
|
||||||
|
|
||||||
|
# Uploads directory for file storage
|
||||||
|
UPLOADS_DIR=./uploads
|
||||||
|
|
||||||
#Celery Config
|
#Celery Config
|
||||||
CELERY_BROKER_URL=redis://localhost:6379/0
|
CELERY_BROKER_URL=redis://localhost:6379/0
|
||||||
CELERY_RESULT_BACKEND=redis://localhost:6379/0
|
CELERY_RESULT_BACKEND=redis://localhost:6379/0
|
||||||
|
|
|
||||||
|
|
@ -114,7 +114,7 @@ async def create_documents_file_upload(
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
# Create uploads directory if it doesn't exist
|
# Create uploads directory if it doesn't exist
|
||||||
uploads_dir = Path("/opt/SurfSense/surfsense_backend/uploads")
|
uploads_dir = Path(os.getenv("UPLOADS_DIR", "./uploads"))
|
||||||
uploads_dir.mkdir(parents=True, exist_ok=True)
|
uploads_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
# Create unique filename
|
# Create unique filename
|
||||||
|
|
|
||||||
|
|
@ -18,8 +18,9 @@ import {
|
||||||
import { Label } from "@/components/ui/label";
|
import { Label } from "@/components/ui/label";
|
||||||
import { AUTH_TOKEN_KEY } from "@/lib/constants";
|
import { AUTH_TOKEN_KEY } from "@/lib/constants";
|
||||||
|
|
||||||
// URL validation regex - updated to support percent-encoded URLs (e.g., Latvian characters)
|
// URL validation regex - requires valid domain structure while supporting international characters
|
||||||
const urlRegex = /^https?:\/\/[^\s]+$/;
|
// Accepts: localhost, domain.tld format, and percent-encoded URLs
|
||||||
|
const urlRegex = /^https?:\/\/(?:localhost|[\w.-]+\.[a-zA-Z]{2,})(?::\d+)?(?:\/[^\s]*)?$/;
|
||||||
|
|
||||||
export default function WebpageCrawler() {
|
export default function WebpageCrawler() {
|
||||||
const t = useTranslations("add_webpage");
|
const t = useTranslations("add_webpage");
|
||||||
|
|
|
||||||
|
|
@ -1,27 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
SERVER="root@46.62.230.195"
|
|
||||||
REMOTE_DIR="/opt/SurfSense/surfsense_backend"
|
|
||||||
LOCAL_DIR="$HOME/Documents/Kods/SurfSense/surfsense_backend"
|
|
||||||
|
|
||||||
echo "Syncing backend files from production..."
|
|
||||||
rsync -avz --progress \
|
|
||||||
--exclude='.env' \
|
|
||||||
--exclude='*.env.local' \
|
|
||||||
--exclude='*.env.production' \
|
|
||||||
--exclude='node_modules/' \
|
|
||||||
--exclude='__pycache__/' \
|
|
||||||
--exclude='*.pyc' \
|
|
||||||
--exclude='.pytest_cache/' \
|
|
||||||
--exclude='venv/' \
|
|
||||||
--exclude='.venv/' \
|
|
||||||
--exclude='*.log' \
|
|
||||||
--exclude='*.db' \
|
|
||||||
--exclude='*.sqlite' \
|
|
||||||
--exclude='*.pem' \
|
|
||||||
--exclude='*.key' \
|
|
||||||
--exclude='*.crt' \
|
|
||||||
--exclude='.DS_Store' \
|
|
||||||
-e "ssh -i ~/.ssh/id_ed25519_surfsense" \
|
|
||||||
"$SERVER:$REMOTE_DIR/" "$LOCAL_DIR/"
|
|
||||||
|
|
||||||
echo "Sync complete!"
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue