diff --git a/.gitignore b/.gitignore index ab8f0a4dc..042124ab3 100644 --- a/.gitignore +++ b/.gitignore @@ -58,3 +58,10 @@ tmp/ pyproject.toml.backup* security_audit_baseline*.json installed_before_update.txt + +# Deployment and production documentation (store privately) +DEPLOYMENT.md +claude.md +sync-from-production.sh +MIGRATION_LOCAL_LLM.md +PR_DESCRIPTION.md diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md deleted file mode 100644 index a439acaaa..000000000 --- a/DEPLOYMENT.md +++ /dev/null @@ -1,376 +0,0 @@ -# SurfSense Production Deployment Guide - -**Server**: ai.kapteinis.lv -**Date**: November 17, 2025 -**Branch**: nightly -**Environment**: Debian VPS with local Ollama LLMs - ---- - -## ๐Ÿ“‹ Pre-Deployment Checklist - -โœ… **Code Changes Committed:** -- Minimal privacy-focused frontend UI -- Email/password authentication only -- Google Analytics removed -- Anthropic Claude removed from examples -- Social media: Mastodon, Pixelfed, Bookwyrm only - -โœ… **GitHub Status:** -- All changes pushed to `nightly` branch -- Repository: https://github.com/okapteinis/SurfSense - -โœ… **Configuration Verified:** -- Mistral NeMo 128K context window fix applied -- TildeOpen 30B grammar checker configured -- Gemini API fallback configured -- No secrets committed to repository - ---- - -## ๐Ÿš€ Deployment Steps - -### Option 1: Automated Deployment (Recommended) - -```bash -# On your VPS (ai.kapteinis.lv) -ssh your-user@ai.kapteinis.lv - -# Create deployment script -cat > /opt/SurfSense/deploy.sh << 'EOF' -#!/bin/bash -set -e - -echo "๐Ÿš€ SurfSense Deployment to ai.kapteinis.lv" -echo "===========================================" - -# Backup current state -BACKUP_DIR="/opt/SurfSense/backups/$(date +%Y%m%d_%H%M%S)" -mkdir -p "$BACKUP_DIR" -cp -r /opt/SurfSense/surfsense_web "$BACKUP_DIR/" -echo "โœ… Backup created: $BACKUP_DIR" - -# Pull latest code -cd /opt/SurfSense -git fetch origin -git checkout nightly -git pull origin nightly -echo "โœ… Code updated from GitHub" - -# Install and build frontend -cd /opt/SurfSense/surfsense_web -pnpm install -pnpm build -echo "โœ… Frontend built" - -# Restart services -sudo systemctl restart surfsense -sudo systemctl restart surfsense-frontend -sudo systemctl restart surfsense-celery || true -sudo systemctl restart surfsense-celery-beat || true -echo "โœ… Services restarted" - -# Verify -sleep 5 -echo "" -echo "Service Status:" -systemctl is-active surfsense && echo " โœ… Backend: Running" -systemctl is-active surfsense-frontend && echo " โœ… Frontend: Running" -systemctl is-active ollama && echo " โœ… Ollama: Running" - -echo "" -echo "๐ŸŽ‰ Deployment complete!" -echo "๐Ÿ”— Visit: https://ai.kapteinis.lv" -EOF - -chmod +x /opt/SurfSense/deploy.sh - -# Run deployment -/opt/SurfSense/deploy.sh -``` - -### Option 2: Manual Deployment - -```bash -# 1. SSH to server -ssh your-user@ai.kapteinis.lv - -# 2. Navigate to SurfSense directory -cd /opt/SurfSense - -# 3. Backup current installation -mkdir -p backups/$(date +%Y%m%d_%H%M%S) -cp -r surfsense_web backups/$(date +%Y%m%d_%H%M%S)/ - -# 4. Pull latest changes -git fetch origin -git checkout nightly -git pull origin nightly - -# 5. Install frontend dependencies -cd surfsense_web -pnpm install - -# 6. Build frontend -pnpm build - -# 7. Restart services -sudo systemctl restart surfsense -sudo systemctl restart surfsense-frontend -sudo systemctl restart surfsense-celery -sudo systemctl restart surfsense-celery-beat - -# 8. Verify services are running -systemctl status surfsense -systemctl status surfsense-frontend -systemctl status ollama -``` - ---- - -## โœ… Post-Deployment Verification - -### 1. Check Services Status - -```bash -# All services should show "active (running)" -sudo systemctl status surfsense -sudo systemctl status surfsense-frontend -sudo systemctl status ollama -sudo systemctl status surfsense-celery -``` - -### 2. Verify UI Changes - -Visit https://ai.kapteinis.lv and verify: - -**Homepage Should Show:** -- โœ… SurfSense logo and theme toggle only (no navigation links) -- โœ… "Let's Start Surfing" heading -- โœ… Tagline paragraph -- โœ… Hero screenshot/demo -- โœ… Footer with SurfSense name and social links - -**Homepage Should NOT Show:** -- โŒ "Get Started" button -- โŒ "Pricing" link -- โŒ "Docs" link -- โŒ Discord/GitHub icons -- โŒ "Sign In" button in navbar -- โŒ Feature cards or integrations sections - -**Login Page Should Show:** -- โœ… Email and password fields only -- โœ… "Sign In" button -- โœ… Link to register page - -**Login Page Should NOT Show:** -- โŒ "Continue with Google" button -- โŒ Any OAuth options - -### 3. Test Authentication - -```bash -# Test login endpoint -curl -X POST https://ai.kapteinis.lv/api/v1/auth/login \ - -H "Content-Type: application/json" \ - -d '{"email":"test@example.com","password":"testpass"}' -``` - -### 4. Test LLM Backend - -```bash -# Check Ollama is running -curl http://localhost:11434/api/tags - -# Should show mistral-nemo:128k and tildeopen models -``` - -### 5. Check Logs - -```bash -# Backend logs -journalctl -u surfsense -n 100 -f - -# Frontend logs -journalctl -u surfsense-frontend -n 100 -f - -# Ollama logs -journalctl -u ollama -n 100 -f -``` - -Look for: -- โœ… No errors during startup -- โœ… "Context window=131072" in logs (Mistral NeMo) -- โœ… Successful model loading messages -- โœ… No Google Analytics references - ---- - -## ๐Ÿ”ง Troubleshooting - -### Frontend Build Fails - -```bash -# Clear cache and rebuild -cd /opt/SurfSense/surfsense_web -rm -rf .next node_modules -pnpm install -pnpm build -``` - -### Services Won't Start - -```bash -# Check for errors -journalctl -u surfsense -n 50 -journalctl -u surfsense-frontend -n 50 - -# Verify ports are available -sudo lsof -i :8000 # Backend -sudo lsof -i :3000 # Frontend -sudo lsof -i :11434 # Ollama -``` - -### Ollama Models Missing - -```bash -# List installed models -ollama list - -# Re-pull if needed -ollama pull mistral-nemo -ollama create mistral-nemo:128k -f /path/to/mistral-nemo-128k.modelfile -ollama pull tildeopen:30b-q5_k_m -``` - -### Google Analytics Still Showing - -```bash -# Verify layout.tsx doesn't have GA -grep -r "GoogleAnalytics\|google-analytics\|gtag" /opt/SurfSense/surfsense_web/app/ - -# Should return nothing -``` - ---- - -## ๐Ÿ“Š Performance Monitoring - -### After Deployment, Monitor: - -```bash -# RAM usage (should be 20-25GB during inference) -free -h - -# CPU usage -htop - -# Disk space -df -h - -# Response times -curl -w "@-" -o /dev/null -s https://ai.kapteinis.lv << 'EOF' -time_total: %{time_total}s -EOF -``` - -### Expected Performance: - -- **English queries**: ~5 seconds -- **Latvian queries**: ~23 seconds (with grammar check) -- **RAM usage**: 13-15GB idle, 20-25GB during inference -- **Disk usage**: ~28GB for Ollama models - ---- - -## ๐Ÿ”„ Rollback Procedure - -If something goes wrong: - -```bash -# Find backup -ls -lt /opt/SurfSense/backups/ - -# Restore from backup -cd /opt/SurfSense -mv surfsense_web surfsense_web.failed -cp -r backups/YYYYMMDD_HHMMSS/surfsense_web . - -# Restart services -sudo systemctl restart surfsense-frontend -``` - ---- - -## ๐Ÿ“ Production Configuration - -### Environment Variables - -Location: `/opt/SurfSense/surfsense_backend/.env` - -**Required:** -```bash -GEMINI_API_KEY=your_key_here -OLLAMA_BASE_URL=http://localhost:11434 -SECRET_KEY=your_secret_key -DATABASE_URL=postgresql://... -REDIS_URL=redis://localhost:6379 -``` - -**Not Required (Removed):** -```bash -# GOOGLE_OAUTH_CLIENT_ID # OAuth disabled -# GOOGLE_OAUTH_CLIENT_SECRET # OAuth disabled -# ANTHROPIC_API_KEY # Not using Claude -``` - -### LLM Configuration - -Location: `/opt/SurfSense/surfsense_backend/app/config/global_llm_config.yaml` - -**Three-tier architecture:** -1. Mistral NeMo 12B (primary) -2. TildeOpen 30B (Latvian grammar) -3. Gemini 2.0 Flash (fallback) - -See `global_llm_config.yaml.template` for structure. - ---- - -## ๐ŸŽฏ Success Criteria - -Deployment is successful when: - -- โœ… Homepage shows minimal UI (logo + tagline only) -- โœ… No navigation links visible (Pricing, Docs removed) -- โœ… Login page shows email/password form only -- โœ… No Google OAuth button -- โœ… Footer shows only Mastodon, Pixelfed, Bookwyrm links -- โœ… Services all running (backend, frontend, Ollama) -- โœ… English queries respond in ~5 seconds -- โœ… Latvian queries respond in ~23 seconds -- โœ… No errors in logs -- โœ… RAM usage normal (13-25GB) - ---- - -## ๐Ÿ“ž Support - -**Issues:** https://github.com/okapteinis/SurfSense/issues -**Email:** ojars@kapteinis.lv -**Deployment Date:** November 17, 2025 -**Version:** nightly branch (commit 209cd24) - ---- - -## ๐Ÿ“š Related Documentation - -- `INSTALLATION_LOCAL_LLM.md` - Complete Ollama setup guide -- `MIGRATION_LOCAL_LLM.md` - Architecture and migration details -- `PR_DESCRIPTION.md` - Full PR documentation -- `claude.md` - Security audit report - ---- - -**Deployment Complete!** Your minimal, privacy-focused SurfSense instance with local European AI is ready. ๐ŸŽ‰ diff --git a/MIGRATION_LOCAL_LLM.md b/MIGRATION_LOCAL_LLM.md deleted file mode 100644 index 51569f3b3..000000000 --- a/MIGRATION_LOCAL_LLM.md +++ /dev/null @@ -1,236 +0,0 @@ -# Local LLM Migration Documentation - -## Date -November 17, 2025 - -## Summary -Migrated SurfSense from Gemini-only API to three-tier local-first European AI architecture. - -## Architecture Changes - -### Previous Architecture -- **Primary**: Gemini 2.0 Flash API only -- **Issues**: - - Rate limits causing service interruptions - - High API costs - - Timeout errors on complex queries - - 30-120 second response times - -### New Architecture - -#### Tier 1 - Primary LLM: Mistral NeMo 12B (France) -- **Model**: `ollama/mistral-nemo:128k` -- **Size**: 7.1GB -- **Context Window**: 128K tokens (131,072) -- **Purpose**: Generate answers from user documents using RAG -- **Performance**: 5 seconds for English queries, 23 seconds for Latvian queries -- **Location**: Local via Ollama at http://localhost:11434 -- **Why chosen**: Fast CPU inference, 50% smaller than Mistral Small 24B, eliminates timeout errors - -#### Tier 2 - Grammar Checker: TildeOpen 30B (Latvia) -- **Model**: `ollama/tildeopen:30b-q5_k_m` -- **Size**: 21GB -- **Purpose**: Check and correct Latvian grammar in generated responses -- **Performance**: 8 second timeout for grammar checking -- **Location**: Local via Ollama at http://localhost:11434 -- **Special**: Best Latvian language model, 41% better than LLaMA-3, 24% better than GPT-4o for Latvian - -#### Tier 3 - API Fallback: Gemini 2.0 Flash (Google) -- **Model**: `gemini/gemini-2.0-flash-exp` -- **Purpose**: Emergency fallback when local models cannot answer -- **Location**: Google API with GEMINI_API_KEY -- **Cost**: Only used for 5% of queries, dramatically reducing API costs and rate limit issues - -## Code Changes - -### Backend Files Modified - -#### 1. `/surfsense_backend/app/agents/researcher/utils.py` -**Changes**: -- Added context window override for mistral-nemo models -- Fixed LiteLLM incorrect reporting (was showing 1,024,000 tokens instead of 128K) -- Correctly limits document context to 128K tokens - -**Key Function Modified**: -```python -def get_model_context_window(model_name: str) -> int: - """Get the total context window size for a model.""" - # Override for Ollama models with known incorrect LiteLLM values - if "mistral-nemo" in model_name.lower(): - return 131072 # Mistral NeMo actual context window: 128K tokens - # ... rest of function -``` - -**Why**: LiteLLM was incorrectly reporting 1M token context window, causing backend to send 530K tokens which exceeded Ollama's 4K default, resulting in massive truncation and query failures. - -#### 2. `/surfsense_backend/app/utils/document_converters.py` -**Changes**: -- Added same context window detection override -- Automatic document truncation to fit context - -**Purpose**: Ensures documents are properly sized before sending to LLM, preventing timeout errors. - -#### 3. `/surfsense_backend/app/config/global_llm_config.yaml.template` -**Changes**: -- Added three-tier model configuration -- Configured Ollama endpoints (http://localhost:11434) -- Set fallback chain: Mistral NeMo โ†’ TildeOpen (for Latvian) โ†’ Gemini (emergency) -- Uses `${GEMINI_API_KEY}` placeholder instead of actual key - -**Security**: Real config file with API key is gitignored, only template is committed. - -### Frontend Files Modified - -#### 1. `/surfsense_frontend/app/layout.tsx` -**Changes**: -- Removed Google Analytics tracking code -- Cleaned up GTM (Google Tag Manager) references -- Removed unnecessary external analytics dependencies - -**Why**: Improved privacy and reduced external dependencies. - -## Configuration - -### Required Environment Variables -```bash -# Required for fallback API -GEMINI_API_KEY=your_gemini_api_key_here - -# Ollama endpoint -OLLAMA_BASE_URL=http://localhost:11434 -``` - -### Ollama Models Required -```bash -# Pull base Mistral NeMo model -ollama pull mistral-nemo - -# Create optimized version with 128K context -ollama create mistral-nemo:128k -f mistral-nemo-128k.modelfile - -# Pull Latvian grammar checker -ollama pull tildeopen:30b-q5_k_m -``` - -### Mistral NeMo Model Configuration -Create `mistral-nemo-128k.modelfile`: -``` -FROM mistral-nemo:latest - -# Set context window to 128K tokens (Mistral NeMo maximum) -PARAMETER num_ctx 131072 - -# Keep other parameters optimized for RAG -PARAMETER temperature 0.7 -PARAMETER top_p 0.9 -PARAMETER top_k 40 -``` - -Then create the model: -```bash -ollama create mistral-nemo:128k -f mistral-nemo-128k.modelfile -``` - -## Benefits - -### Cost Reduction -- **95% reduction in API costs** -- Only 5% of queries hit Gemini API (fallback only) -- Unlimited local usage with no per-token charges - -### Performance Improvements -- **Response times**: 5-25 seconds (down from 30-120 seconds) -- **No rate limits**: Can handle unlimited concurrent users -- **No timeouts**: 128K context properly configured -- **Better accuracy**: Documents no longer truncated to 4K - -### Privacy & Security -- **Complete data privacy**: Documents never leave your server (unless fallback triggered) -- **GDPR compliant**: All processing on EU servers -- **No third-party tracking**: Removed Google Analytics - -### Language Quality -- **Latvian language**: 41% better than LLaMA-3, 24% better than GPT-4o -- **Grammar correction**: Dedicated Latvian model (TildeOpen) -- **Multilingual**: Mistral NeMo supports 50+ languages - -## Performance Metrics - -### Response Times -- **English queries**: ~5 seconds (Mistral NeMo only) -- **Latvian queries**: ~23 seconds (Mistral NeMo + TildeOpen grammar check) -- **Complex queries**: Falls back to Gemini if needed - -### Resource Usage -- **RAM usage**: 20-25GB during inference, 13-15GB idle -- **Disk usage**: 28GB for both models (7GB + 21GB) -- **CPU**: High during inference, normal otherwise - -### Accuracy -- **Document retrieval**: 4,338 documents processed -- **Token optimization**: 530K tokens in context (properly handled) -- **No truncation**: Full 128K context utilized - -## Known Issues & Solutions - -### Issue 1: LiteLLM Context Window Bug -**Problem**: LiteLLM reports 1,024,000 tokens for mistral-nemo when actual limit is 131,072. - -**Solution**: Added manual override in `get_model_context_window()` function to return correct value. - -### Issue 2: Ollama Default Context -**Problem**: Ollama defaults to 4,096 token context, causing massive truncation. - -**Solution**: Created custom model `mistral-nemo:128k` with `num_ctx` parameter set to 131,072. - -### Issue 3: Frontend Model Name -**Problem**: Frontend showed "Mistral Small 24B (Local)" instead of "Mistral NeMo 12B". - -**Solution**: Updated display name in `global_llm_config.yaml` to reflect actual model. - -## Deployment History - -### Production Server: ai.kapteinis.lv -- **Date**: November 17, 2025 -- **Server**: Debian 6.12.57, 32GB RAM, CPU-only -- **Location**: /opt/SurfSense -- **Status**: Successfully deployed and tested - -### Testing Results -- โœ… English queries: 5 second response time -- โœ… Latvian queries: 23 second response time (with grammar check) -- โœ… No timeout errors -- โœ… No rate limit errors -- โœ… 95% cost reduction confirmed -- โœ… Improved answer quality -- โœ… Full 128K context utilized - -## Migration Steps Summary - -1. Installed Ollama service -2. Downloaded Mistral NeMo 12B model -3. Downloaded TildeOpen 30B model -4. Created mistral-nemo:128k with proper context window -5. Updated backend configuration YAML -6. Patched Python code for context window handling -7. Removed Google Analytics from frontend -8. Rebuilt frontend with pnpm build -9. Restarted all services -10. Verified functionality with test queries - -## Authors -- **Ojฤrs Kapteiล†ลก** - Implementation -- **Claude AI Assistant** (Anthropic) - Architecture design and debugging - -## References -- Mistral NeMo: https://mistral.ai/news/mistral-nemo/ -- TildeOpen: https://tilde.ai/tildeopen -- Ollama: https://ollama.com/ -- LiteLLM: https://github.com/BerriAI/litellm - -## License -This implementation is part of SurfSense and follows the same license terms. - ---- - -**100% European AI Solution**: Mistral AI (France) for primary intelligence, Tilde AI (Latvia) for grammar expertise, with Google (USA) only for emergencies. This architecture represents cutting-edge deployment of European AI technology for production use at enterprise scale. diff --git a/PR_DESCRIPTION.md b/PR_DESCRIPTION.md deleted file mode 100644 index 0f1e529f1..000000000 --- a/PR_DESCRIPTION.md +++ /dev/null @@ -1,275 +0,0 @@ -# Local LLM Implementation - Nightly to Main - -## Overview -This PR implements a local-first European AI architecture for SurfSense, replacing the Gemini-only API approach with a three-tier system that dramatically reduces costs, improves performance, and enhances privacy. - -## ๐ŸŽฏ Problem Statement - -### Previous Issues -- **High API Costs**: Every query hit Gemini API, resulting in substantial monthly costs -- **Rate Limits**: Frequent 429 errors during peak usage -- **Slow Response Times**: 30-120 seconds per query due to API latency -- **Timeout Errors**: Complex queries with large document sets failed -- **Privacy Concerns**: All user data sent to external APIs -- **Vendor Lock-in**: Complete dependency on Google's API availability - -## ๐Ÿš€ Solution: Three-Tier European AI Architecture - -### Tier 1 - Primary LLM: Mistral NeMo 12B (๐Ÿ‡ซ๐Ÿ‡ท France) -- **Model**: `ollama/mistral-nemo:128k` -- **Purpose**: Generate answers from user documents using RAG -- **Performance**: 5-10 second response time -- **Context**: Full 128K tokens (131,072) for large document sets -- **Location**: Local inference via Ollama - -### Tier 2 - Grammar Checker: TildeOpen 30B (๐Ÿ‡ฑ๐Ÿ‡ป Latvia) -- **Model**: `ollama/tildeopen:30b-q5_k_m` -- **Purpose**: Latvian grammar correction and quality assurance -- **Performance**: Additional 5-10 seconds for Latvian queries -- **Quality**: 41% better than LLaMA-3, 24% better than GPT-4o for Latvian -- **Location**: Local inference via Ollama - -### Tier 3 - API Fallback: Gemini 2.0 Flash (๐Ÿ‡บ๐Ÿ‡ธ Google) -- **Model**: `gemini-2.0-flash-exp` -- **Purpose**: Emergency fallback only when local models cannot answer -- **Usage**: Only ~5% of queries -- **Cost**: 95% reduction in API costs - -## ๐Ÿ“Š Performance Improvements - -### Response Times -| Query Type | Before | After | Improvement | -|------------|--------|-------|-------------| -| English queries | 30-120s | ~5s | **6-24x faster** | -| Latvian queries | 30-120s | ~23s | **1.3-5x faster** | -| Complex queries | Often timeout | Reliable | **100% success** | - -### Cost Reduction -- **95% reduction** in API costs -- From: $XXX/month (all queries via API) -- To: $X/month (5% via API) -- **Unlimited local usage** with no per-token charges - -### Quality Improvements -- **No truncation**: Full 128K context utilized (was limited to 4K) -- **Better Latvian**: Dedicated Latvian model instead of generic multilingual -- **More accurate**: Documents no longer truncated, full context preserved - -## ๐Ÿ”’ Privacy & Security Enhancements - -- โœ… **Complete data privacy**: 95% of queries never leave your server -- โœ… **GDPR compliant**: All primary processing on EU servers -- โœ… **No third-party tracking**: Removed Google Analytics from frontend -- โœ… **EU sovereignty**: Primary AI from France (Mistral) and Latvia (Tilde) - -## ๐Ÿ“ Technical Details - -### Backend Changes - -#### 1. Context Window Bug Fix (`app/agents/researcher/utils.py`) -**Problem**: LiteLLM incorrectly reported 1,024,000 token context for mistral-nemo (actual: 128K) - -**Solution**: Added manual override to return correct 131,072 token context -```python -def get_model_context_window(model_name: str) -> int: - if "mistral-nemo" in model_name.lower(): - return 131072 # Correct context window - # ... rest of implementation -``` - -**Impact**: Prevents backend from sending 530K tokens to model with 128K limit - -#### 2. Document Converter Update (`app/utils/document_converters.py`) -- Same context window fix applied -- Ensures proper document truncation before LLM processing - -#### 3. LLM Configuration (`app/config/global_llm_config.yaml.template`) -**New three-tier configuration**: -```yaml -global_llm_configs: - # Primary: Mistral NeMo 12B (Local) - - model_name: "mistral-nemo:128k" - provider: "OLLAMA" - api_base: "http://localhost:11434" - - # Grammar: TildeOpen 30B (Local) - - model_name: "tildeopen:latest" - provider: "OLLAMA" - api_base: "http://localhost:11434" - - # Fallback: Gemini 2.0 Flash (API) - - model_name: "gemini-2.0-flash-exp" - provider: "GOOGLE" - api_key: "${GEMINI_API_KEY}" -``` - -**Security**: Template uses `${GEMINI_API_KEY}` placeholder, real config gitignored - -### Frontend Changes - -#### 1. Analytics Removal (`app/layout.tsx`) -- Removed Google Analytics tracking code -- Removed GTM (Google Tag Manager) references -- Improved privacy and reduced external dependencies - -### Configuration Changes - -#### 1. Updated `.gitignore` -Added comprehensive security patterns: -- Environment files (`.env`, `.env.local`, etc.) -- Config files with API keys -- Python cache and virtual environments -- Logs, databases, and uploads -- SSL certificates and private keys - -## ๐Ÿ“ฆ Files Changed - -### Modified -- `surfsense_backend/app/agents/researcher/utils.py` - Context window fix -- `surfsense_backend/app/utils/document_converters.py` - Context window fix -- `surfsense_frontend/app/layout.tsx` - Removed analytics -- `.gitignore` - Enhanced security patterns - -### Added -- `surfsense_backend/app/config/global_llm_config.yaml.template` - Secure config template -- `MIGRATION_LOCAL_LLM.md` - Complete migration documentation -- `INSTALLATION_LOCAL_LLM.md` - Installation guide -- `PR_DESCRIPTION.md` - This PR description -- `sync-from-production.sh` - Secure rsync script for deployments - -## ๐Ÿงช Testing - -Tested on production at **https://ai.kapteinis.lv** with: - -### Test Results -- โœ… English queries: 5 second response time -- โœ… Latvian queries: 23 second response time (with grammar check) -- โœ… Large document sets: 4,338 documents, 530K tokens processed successfully -- โœ… No timeout errors -- โœ… No rate limit errors -- โœ… 95% cost reduction confirmed -- โœ… Improved answer quality and accuracy -- โœ… Full 128K context properly utilized - -### Load Testing -- โœ… Concurrent users: Handled without rate limits -- โœ… Peak RAM usage: 20-25GB during inference -- โœ… Idle RAM usage: 13-15GB -- โœ… Response consistency: Stable performance across queries - -## ๐Ÿ“‹ Installation Requirements - -### New Dependencies -- **Ollama**: Local LLM inference server -- **Disk Space**: Additional 28GB for models (7GB + 21GB) -- **RAM**: 32GB recommended (24GB minimum) -- **Environment Variable**: `OLLAMA_BASE_URL=http://localhost:11434` - -### Installation Steps -See `INSTALLATION_LOCAL_LLM.md` for complete guide: -1. Install Ollama -2. Download models (mistral-nemo, tildeopen) -3. Create mistral-nemo:128k with proper context window -4. Update backend configuration -5. Apply code patches -6. Restart services - -## โš ๏ธ Breaking Changes - -### Required Changes -- **Ollama must be installed** and running on port 11434 -- **Models must be downloaded**: 28GB total (mistral-nemo:128k, tildeopen:30b-q5_k_m) -- **Backend code patches** must be applied (context window functions) -- **Environment variable** `OLLAMA_BASE_URL` must be set - -### Migration Path -Existing deployments can upgrade incrementally: -1. Install Ollama alongside existing setup -2. Download models in background -3. Update configuration to add local models as primary -4. Gemini automatically becomes fallback -5. Monitor and adjust as needed - -**No downtime required** - fallback ensures continuous operation during migration. - -## ๐Ÿ› Known Issues & Solutions - -### Issue 1: Ollama Default Context Window -**Problem**: Ollama defaults to 4K context, causing truncation - -**Solution**: Create custom model with `num_ctx 131072` parameter - -### Issue 2: LiteLLM Context Detection -**Problem**: LiteLLM reports incorrect context window sizes - -**Solution**: Manual override in backend code (implemented in this PR) - -### Issue 3: Frontend Model Name -**Problem**: UI showed "Mistral Small 24B" instead of "Mistral NeMo 12B" - -**Solution**: Updated display name in configuration (included in this PR) - -## ๐Ÿ“– Documentation - -### Added Documentation -- **MIGRATION_LOCAL_LLM.md**: Complete architecture explanation and migration history -- **INSTALLATION_LOCAL_LLM.md**: Step-by-step installation guide with troubleshooting -- **PR_DESCRIPTION.md**: This detailed PR description - -### Updated Documentation -- **.gitignore**: Comprehensive security patterns documented -- **README updates**: (Recommend adding link to new docs in main README) - -## ๐Ÿ”„ Deployment History - -### Production Server: ai.kapteinis.lv -- **Date**: November 17, 2025 -- **Server**: Debian 6.12.57, 32GB RAM, CPU-only -- **Status**: โœ… Successfully deployed and operational -- **Uptime**: Stable with zero downtime during migration - -## ๐Ÿ‘ฅ Authors & Contributors - -- **Ojฤrs Kapteiล†ลก** (@okapteinis) - Implementation and deployment -- **Claude AI Assistant** (Anthropic) - Architecture design and debugging assistance - -## ๐Ÿ”— References - -- **Mistral NeMo**: https://mistral.ai/news/mistral-nemo/ -- **TildeOpen**: https://tilde.ai/tildeopen -- **Ollama**: https://ollama.com/ -- **LiteLLM**: https://github.com/BerriAI/litellm - -## โœ… Checklist - -- [x] Code changes implemented and tested -- [x] Documentation created (migration + installation guides) -- [x] Security review completed (no secrets in repository) -- [x] Production deployment successful -- [x] Performance metrics validated -- [x] Cost reduction confirmed (95%) -- [x] .gitignore updated with security patterns -- [x] Config templates created with placeholders -- [x] Frontend cleaned of external tracking -- [x] Backward compatibility maintained (Gemini fallback) - -## ๐ŸŽ‰ Summary - -This PR represents a **fundamental architectural improvement** for SurfSense: - -- ๐Ÿ’ฐ **95% cost reduction** through local-first approach -- โšก **6-24x faster** response times -- ๐Ÿ”’ **Enhanced privacy** with EU-based processing -- ๐Ÿ‡ช๐Ÿ‡บ **European AI sovereignty** (Mistral + Tilde) -- ๐ŸŽฏ **Better quality** with proper context handling -- ๐Ÿ“ˆ **Unlimited scaling** without rate limits - -**This is production-ready** and has been successfully deployed at https://ai.kapteinis.lv since November 17, 2025. - -## ๐Ÿš€ Ready to Merge - -This PR is ready for review and merge to main. All testing completed successfully, production deployment verified, and documentation comprehensive. - ---- - -**Questions or concerns?** Please review the documentation or contact @okapteinis. diff --git a/claude.md b/claude.md deleted file mode 100644 index 86291baf6..000000000 --- a/claude.md +++ /dev/null @@ -1,2211 +0,0 @@ -# SurfSense Security & Architecture Audit Report - -**Audit Date:** 2025-11-17 -**Audited By:** Claude AI Assistant (Anthropic) -**Repository:** https://github.com/okapteinis/SurfSense -**Branch:** claude/surfsense-audit-automation-01MeVuvAybnL5NXHG5W3XuXe -**Upstream:** https://github.com/MODSetter/SurfSense -**Session Topic:** Comprehensive Security, Architecture, and Compliance Audit - ---- - -## Executive Summary - -This report documents a comprehensive security and architecture audit of the SurfSense repository. SurfSense is an AI-powered research agent and personal knowledge base system that integrates with various external sources (Slack, Linear, GitHub, Notion, etc.) and supports local/cloud LLM deployments. - -**Overall Assessment:** โœ… **GOOD** with recommendations for enhancement - -The codebase demonstrates strong security practices with automated tooling (detect-secrets, pre-commit hooks, bandit), modern architecture (FastAPI + Next.js), and comprehensive documentation. Several recommendations are provided to further strengthen security posture and automation capabilities. - ---- - -## 1. Scope of Audit - -### 1.1 What Was Checked - -โœ… **Security & Secrets Scanning** -- Scanned all source files for hardcoded credentials, API keys, tokens -- Reviewed `.env.example` files for placeholder safety -- Validated `.gitignore` coverage for sensitive files -- Analyzed pre-commit hooks and detect-secrets baseline -- Checked for exposed private keys, certificates, database credentials - -โœ… **Architecture & LLM/RAG Components** -- Reviewed FastAPI backend architecture -- Analyzed hybrid search implementation (vector + full-text) -- Examined LangGraph agent configurations -- Validated embedding models and reranker setup -- Assessed PostgreSQL/pgvector integration -- Reviewed Celery/Redis async task architecture - -โœ… **Platform Compatibility** -- Python version requirements (3.12+) -- Node.js/npm package versions -- Docker containerization setup -- Multi-platform support (x86_64, ARM) - -โœ… **Licensing & Copyright** -- Scanned all LICENSE files -- Checked for license headers in source files -- Verified third-party dependency licenses - -โœ… **Documentation Quality** -- Reviewed README.md, CONTRIBUTING.md -- Examined setup guides (Chinese LLM guide, etc.) -- Validated configuration examples - ---- - -## 2. Security Audit Results - -### 2.1 Secrets & Credentials Analysis - -#### โœ… **Status: SECURE - No Real Secrets Found** - -**Findings:** -1. **All API keys in codebase are placeholders:** - - `.env.example` files use obvious placeholders: - - `SECRET_KEY=SECRET` - - `GOOGLE_OAUTH_CLIENT_ID=924507538m` (partial/example) - - `FIRECRAWL_API_KEY=fcr-01J0000000000000000000000` (zeros) - - `UNSTRUCTURED_API_KEY=Tpu3P0U8iy` (short placeholder) - - `LLAMA_CLOUD_API_KEY=llx-nnn` (obvious placeholder) - -2. **Configuration templates are safe:** - - `surfsense_backend/app/config/global_llm_config.example.yaml` uses: - - `sk-your-openai-api-key-here` - - `sk-ant-your-anthropic-api-key-here` - - `your-deepseek-api-key-here` - -3. **Documentation examples use masked placeholders:** - - `docs/chinese-llm-setup.md` uses `sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx` - -4. **No .env files committed:** - - Only `.env.example` files present (verified with `ls -la`) - - Real `.env` files properly gitignored - -#### โœ… **Security Tooling in Place** - -**Excellent automated security measures:** - -1. **detect-secrets** (v1.5.0) with comprehensive detectors: - - โœ… AWS Key Detector - - โœ… Azure Storage Key Detector - - โœ… GitHub Token Detector - - โœ… JWT Token Detector - - โœ… Private Key Detector - - โœ… Slack Detector - - โœ… Discord Bot Token Detector - - โœ… High Entropy String Detectors (Base64, Hex) - - โœ… And 15+ more detectors - - Baseline file: `.secrets.baseline` (currently clean: `"results": {}`) - -2. **Pre-commit hooks** (`.pre-commit-config.yaml`): - - โœ… Secret detection on every commit - - โœ… Bandit (Python security linter) - high severity checks - - โœ… Ruff (Python linting + formatting) - - โœ… Biome (TypeScript/JavaScript linting) - - โœ… YAML/JSON/TOML validation - - โœ… Large file prevention (10MB limit) - - โœ… Commit message linting (commitizen) - -### 2.2 .gitignore Analysis - -#### โš ๏ธ **Status: MINIMAL - Needs Enhancement** - -**Current .gitignore (5 entries only):** -``` -.flashrank_cache* -podcasts/ -.env -node_modules/ -.ruff_cache/ -``` - -**Issues:** -- โŒ Missing common Python cache patterns (`__pycache__/`, `*.pyc`, `*.pyo`) -- โŒ Missing test/coverage artifacts (`.pytest_cache/`, `.coverage`, `htmlcov/`) -- โŒ Missing build artifacts (`dist/`, `build/`, `*.egg-info/`) -- โŒ Missing environment variations (`.env.local`, `.env.production`, `.env.*.local`) -- โŒ Missing credentials/secrets patterns (`*.key`, `*.pem`, `*.cert`, `credentials.json`, `secrets.*`) -- โŒ Missing log files (`*.log`, `logs/`, `.log/`) -- โŒ Missing OS-specific files (`.DS_Store`, `Thumbs.db`) -- โŒ Missing IDE files (`.idea/`, `*.swp`, `*.swo`) -- โŒ Missing database files (`*.db`, `*.sqlite`, `*.sqlite3`) - -**Recommendation:** Enhance `.gitignore` with comprehensive patterns (see Recommendations section). - -### 2.3 Docker Security - -#### โœ… **Status: GOOD** with minor notes - -**Findings:** -1. **Multi-stage builds:** Not used, but acceptable for this project size -2. **Base image:** `python:3.12-slim` (good - minimal attack surface) -3. **Security updates:** โœ… Includes `apt-get update`, certificate management -4. **Non-root user:** โŒ Runs as root (line 75: no USER directive) -5. **Secrets handling:** โœ… Uses `.env` files, not baked into image -6. **SSL certificates:** โœ… Properly configured with certifi - -**docker-compose.yml:** -- โœ… Uses environment variable substitution -- โœ… Default credentials are weak (`postgres:postgres`) but documented for local dev -- โœ… Services properly isolated with Docker networks -- โœ… pgAdmin included for database management - ---- - -## 3. LLM/RAG Architecture Summary - -### 3.1 Architecture Overview - -**SurfSense implements a sophisticated 2-tier Hierarchical RAG system:** - -``` -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ USER QUERY โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - โ†“ -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ TIER 1: Document-Level Search โ”‚ -โ”‚ โ€ข Hybrid Search (Vector + Full-Text) โ”‚ -โ”‚ โ€ข PostgreSQL pgvector (<=> operator) โ”‚ -โ”‚ โ€ข Reciprocal Rank Fusion (RRF) โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - โ†“ -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ TIER 2: Chunk-Level Search โ”‚ -โ”‚ โ€ข Fine-grained retrieval within selected documents โ”‚ -โ”‚ โ€ข Vector + Full-Text Hybrid Search โ”‚ -โ”‚ โ€ข RRF fusion for optimal ranking โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - โ†“ -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ RERANKING (Optional) โ”‚ -โ”‚ โ€ข Cohere, FlashRank, Pinecone rerankers โ”‚ -โ”‚ โ€ข Model: ms-marco-MiniLM-L-12-v2 (default) โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - โ†“ -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ LLM RESPONSE GENERATION โ”‚ -โ”‚ โ€ข LiteLLM (100+ LLM support) โ”‚ -โ”‚ โ€ข LangGraph agent orchestration โ”‚ -โ”‚ โ€ข Cited answers (Perplexity-style) โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ -``` - -### 3.2 Core Components - -#### **A. Embedding Models** -- **Library:** Chonkie `AutoEmbeddings` (v1.4.0+) -- **Default:** `sentence-transformers/all-MiniLM-L6-v2` (local) -- **Supported Providers:** - - Sentence Transformers (local, 6000+ models) - - OpenAI (`openai://text-embedding-ada-002`) - - Anthropic (`anthropic://claude-v1`) - - Cohere (`cohere://embed-english-light-v3.0`) - - Configurable via `EMBEDDING_MODEL` environment variable - -#### **B. Vector Database** -- **Technology:** PostgreSQL 14+ with pgvector extension -- **Storage:** `ankane/pgvector:latest` Docker image -- **Operations:** - - Cosine similarity: `<=>` operator - - Efficient indexing for large-scale vector search - -#### **C. Hybrid Search Implementation** - -**File:** `surfsense_backend/app/retriver/documents_hybrid_search.py` -**File:** `surfsense_backend/app/retriver/chunks_hybrid_search.py` - -**Features:** -1. **Vector Search:** - - Embeddings stored in PostgreSQL vector columns - - Cosine similarity ranking via `<=>` operator - -2. **Full-Text Search:** - - PostgreSQL `to_tsvector()` / `plainto_tsquery()` - - English language support (configurable) - -3. **Reciprocal Rank Fusion (RRF):** - - Combines vector + full-text results - - Optimal ranking for diverse query types - -4. **Security:** - - User-based access control (`user_id` filtering) - - Search space isolation (`search_space_id`) - -#### **D. Chunking Strategy** - -- **Library:** Chonkie `LateChunker` (v1.4.0+) -- **Approach:** Adaptive chunking based on embedding model's max sequence length -- **Benefits:** - - Prevents token overflow - - Optimizes chunk size for embedding quality - - Automatic adaptation to different models - -#### **E. Rerankers (Optional)** - -**Enabled via:** `RERANKERS_ENABLED=TRUE` - -**Supported Models:** -- FlashRank (`ms-marco-MiniLM-L-12-v2`) - default -- Cohere -- Pinecone -- Others via `rerankers` library (v0.7.1+) - -#### **F. LLM Integration** - -**Primary Framework:** LiteLLM (v1.77.5+) - -**Supported Providers (100+):** -- OpenAI (GPT-4, GPT-3.5) -- Anthropic (Claude Sonnet, Opus, Haiku) -- Google (Gemini, Vertex AI) -- Chinese LLMs: - - DeepSeek - - Alibaba Qwen (้€šไน‰ๅƒ้—ฎ) - - Moonshot Kimi (ๆœˆไน‹ๆš—้ข) - - Zhipu GLM (ๆ™บ่ฐฑ AI) -- OpenRouter -- Comet API -- Groq -- And 90+ more... - -**Configuration:** `surfsense_backend/app/config/global_llm_config.example.yaml` - -#### **G. Agent Framework** - -**Technology:** LangGraph (v0.3.29+) - -**Agents:** -1. **Researcher Agent** (`app/agents/researcher/`) - - Multi-step research workflows - - External source integration (Slack, GitHub, Notion, etc.) - - Search engine connectors (Tavily, LinkUp, SearxNG) - -2. **QNA Agent** (`app/agents/researcher/qna_agent/`) - - Question-answering on saved content - - Citation generation (Perplexity-style) - -3. **Podcaster Agent** (`app/agents/podcaster/`) - - Blazingly fast podcast generation (<20s for 3-min podcast) - - TTS support: - - Local: Kokoro TTS - - Cloud: OpenAI, Azure, Google Vertex AI - - STT support: Faster-Whisper (local), LiteLLM providers - -#### **H. Document Processing (ETL)** - -**Configurable via:** `ETL_SERVICE` environment variable - -**Options:** -1. **Docling** (default, privacy-focused, no API key) - - Local processing - - Supports: PDF, DOCX, HTML, images, CSV - -2. **Unstructured.io** (API-based, 34+ formats) - - Cloud processing - - Requires `UNSTRUCTURED_API_KEY` - -3. **LlamaCloud** (API-based, 50+ formats) - - Enhanced parsing - - Requires `LLAMA_CLOUD_API_KEY` - -#### **I. Async Task Queue** - -- **Technology:** Celery (v5.5.3+) + Redis (v5.2.1+) -- **Workers:** Handle document processing, podcast generation -- **Scheduler:** Celery Beat for periodic connector syncing -- **Monitoring:** Flower (v2.0.1+) on port 5555 - -### 3.3 External Connectors - -**Supported Integrations (15+):** -- Search Engines: Tavily, LinkUp, SearxNG, Baidu, Serper -- Collaboration: Slack, Discord -- Project Management: Linear, Jira, ClickUp, Airtable, Luma -- Documentation: Notion, Confluence -- Cloud Storage: Google Calendar, Gmail -- Development: GitHub -- Custom: Elasticsearch - ---- - -## 4. Platform Compatibility - -### 4.1 Backend (Python) - -**Version:** Python 3.12+ -**Package Manager:** uv + pip -**Status:** โœ… **MODERN & COMPATIBLE** - -**Key Dependencies:** -- FastAPI 0.115.8+ (latest stable) -- SQLAlchemy 2.x (async support) -- Alembic 1.13.0+ (database migrations) -- LangChain/LangGraph (latest stable) -- PyTorch (CUDA 12.1 support on x86_64, CPU on ARM) - -**Compatibility Notes:** -- โœ… Python 3.12 is current stable (released Oct 2023) -- โœ… All major dependencies are actively maintained -- โœ… Multi-platform Docker support (x86_64, ARM) -- โš ๏ธ Requires significant system resources (ML models, embeddings) - -### 4.2 Frontend (Node.js/React) - -**Versions:** -- Node.js: 20+ (inferred from package.json `@types/node: ^20.19.9`) -- Next.js: 15.5.6 (latest stable) -- React: 19.1.0 (latest stable, released Dec 2024) -- TypeScript: 5.8.3 (latest stable) - -**Status:** โœ… **CUTTING-EDGE & MODERN** - -**Build System:** -- Next.js 15 with Turbopack (ultra-fast bundler) -- Tailwind CSS 4.x (latest major version) -- Biome 2.1.2 (Rust-based linter/formatter) - -**Compatibility Notes:** -- โœ… React 19 is production-ready (Dec 2024 release) -- โœ… Next.js 15 App Router (modern architecture) -- โœ… All dependencies actively maintained -- โš ๏ธ React 19 is very new - ensure thorough testing for SSR edge cases - -### 4.3 Docker & DevOps - -**Docker Compose:** v3.8 -**Images:** -- Backend: Python 3.12-slim -- Database: `ankane/pgvector:latest` -- Redis: `redis:7-alpine` -- pgAdmin: `dpage/pgadmin4` - -**Status:** โœ… **PRODUCTION-READY** - -**Features:** -- โœ… Multi-container orchestration -- โœ… Volume persistence (postgres_data, redis_data, pgadmin_data) -- โœ… Environment variable configuration -- โœ… Health checks and dependencies -- โš ๏ธ No explicit resource limits (CPU/memory) - consider for production - ---- - -## 5. Licensing & Copyright - -### 5.1 Main License - -**License:** Apache License 2.0 -**File:** `/LICENSE` (202 lines) - -**Key Terms:** -- โœ… Commercial use allowed -- โœ… Modification allowed -- โœ… Distribution allowed -- โœ… Patent grant included -- โœ… Requires attribution notice -- โœ… Requires license inclusion in distributions -- โœ… Changes must be documented - -**Note:** User mentioned "CC BY-NC-ND 4.0" in instructions, but repository uses **Apache 2.0**. This is a **permissive open-source license**, not Creative Commons. - -### 5.2 License Headers - -**Status:** โŒ **NOT PRESENT** in source files - -**Findings:** -- No license headers found in Python files -- No license headers found in TypeScript/JavaScript files -- Only top-level LICENSE files present - -**Recommendation:** -While not required by Apache 2.0, adding SPDX license identifiers to source files is a best practice: - -```python -# SPDX-License-Identifier: Apache-2.0 -# Copyright 2025 SurfSense Contributors -``` - -### 5.3 Third-Party Dependencies - -**Backend (Python):** -- All dependencies use permissive licenses (MIT, Apache, BSD) -- No GPL/AGPL "viral" licenses detected - -**Frontend (Node.js):** -- All dependencies use permissive licenses (MIT, Apache, BSD) -- React, Next.js, Tailwind: MIT License - -**Status:** โœ… **LICENSE-COMPATIBLE** - No conflicts found - -### 5.4 Co-Authorship Policy - -**Current Status:** -- No explicit co-authorship markers in commit history -- Standard Git commit metadata used - -**Recommendation (per user instructions):** -For audit sessions and significant contributions, use Git co-author trailers: - -```bash -git commit -m "Add feature X - -Co-authored-by: Ojฤrs Kapteiล†ลก -Co-authored-by: Claude AI Assistant " -``` - -**For file-level attribution:** -```python -# Contributors: -# - Original Author: SurfSense Team -# - Security Review: Ojฤrs Kapteiล†ลก , 2025-11-17 -# - Architecture Audit: Claude AI Assistant, 2025-11-17 -``` - ---- - -## 6. Documentation Review - -### 6.1 Files Reviewed - -โœ… **README.md** (316 lines) -- Comprehensive feature list -- Installation options (Cloud, Docker, Manual) -- Tech stack documentation -- Screenshots and demos -- Star history and contribution guidelines - -โœ… **CONTRIBUTING.md** (exists) -- Contribution workflow documented - -โœ… **CODE_OF_CONDUCT.md** (exists) -- Community standards documented - -โœ… **docs/chinese-llm-setup.md** -- Excellent multi-language support -- Detailed setup for DeepSeek, Qwen, Moonshot, Zhipu -- Includes API key examples (all properly masked: `sk-xxx`) - -### 6.2 Missing Documentation (per user instructions) - -โŒ **INSTALLATION_LOCAL_LLM.md** - Not found -โŒ **MIGRATION_LOCAL_LLM.md** - Not found -โŒ **PR_DESCRIPTION.md** - Not found - -**Note:** These specific files were mentioned in audit requirements but don't exist. Equivalent documentation may exist under different names or in online docs (surfsense.net/docs). - -### 6.3 Documentation Quality - -**Status:** โœ… **EXCELLENT** - -**Strengths:** -- Clear, well-structured README -- Multi-language support (English, Chinese) -- Comprehensive setup guides -- Code examples properly sanitized -- Active roadmap (GitHub Projects) - -**Recommendations:** -- Add local LLM migration guide (if needed) -- Document disaster recovery procedures -- Add API reference documentation - ---- - -## 7. Key Findings & Recommendations - -### 7.1 Critical (Fix Immediately) - -**None found.** โœ… No critical security vulnerabilities detected. - -### 7.2 High Priority - -1. **Enhance .gitignore** - - **Risk:** Accidental commit of secrets/caches - - **Action:** Add comprehensive patterns (see Appendix A) - - **Effort:** 10 minutes - -2. **Add Docker non-root user** - - **Risk:** Container runs as root (privilege escalation if compromised) - - **Action:** Add `USER` directive to Dockerfile - - **Effort:** 5 minutes - -### 7.3 Medium Priority - -3. **Add SPDX license identifiers to source files** - - **Risk:** License ambiguity in file extracts - - **Action:** Add `# SPDX-License-Identifier: Apache-2.0` to files - - **Effort:** 2 hours (automated via script) - -4. **Document upstream sync automation** - - **Risk:** Manual sync errors, stale fork - - **Action:** Create GitHub Actions workflow (see Appendix B) - - **Effort:** 1 hour - -5. **Add resource limits to docker-compose.yml** - - **Risk:** Resource exhaustion in production - - **Action:** Add `deploy.resources.limits` to services - - **Effort:** 30 minutes - -### 7.4 Low Priority - -6. **Strengthen default database credentials** - - **Risk:** Production deployments with weak defaults - - **Action:** Document strong password generation in setup guide - - **Effort:** 15 minutes - -7. **Add security.txt** - - **Risk:** Unclear vulnerability disclosure process - - **Action:** Create `.well-known/security.txt` (RFC 9116) - - **Effort:** 30 minutes - -8. **Enable Dependabot** - - **Risk:** Outdated dependencies with known CVEs - - **Action:** Add `.github/dependabot.yml` - - **Effort:** 15 minutes - ---- - -## 8. Compliance Checklist - -### 8.1 Security Checklist - -- [x] No hardcoded secrets in codebase -- [x] `.env.example` files use placeholders only -- [x] Real `.env` files gitignored -- [x] Automated secret scanning (detect-secrets) -- [x] Pre-commit hooks for security -- [x] SQL injection prevention (SQLAlchemy ORM) -- [x] XSS prevention (React auto-escaping, rehype-sanitize) -- [x] CSRF protection (FastAPI CORS, SameSite cookies) -- [x] User authentication (FastAPI Users, OAuth) -- [x] Database access control (user_id filtering) -- [ ] Docker non-root user (RECOMMENDED) -- [x] SSL/TLS certificate management -- [ ] Comprehensive .gitignore (NEEDS IMPROVEMENT) - -### 8.2 Code Quality Checklist - -- [x] Python linting (Ruff) -- [x] Python security scanning (Bandit) -- [x] TypeScript linting (Biome) -- [x] Pre-commit hooks enabled -- [x] Commit message linting (commitizen) -- [x] Type safety (TypeScript, Python type hints) -- [x] Database migrations (Alembic) -- [x] API versioning (/api/v1/) - -### 8.3 Documentation Checklist - -- [x] README with installation instructions -- [x] Contributing guidelines -- [x] Code of conduct -- [x] License file (Apache 2.0) -- [x] Setup guides for major features -- [ ] API reference documentation (RECOMMENDED) -- [ ] Disaster recovery guide (RECOMMENDED) -- [ ] Local LLM migration guide (per user requirements) - -### 8.4 DevOps Checklist - -- [x] Docker containerization -- [x] Docker Compose orchestration -- [x] Environment variable configuration -- [x] Database persistence (volumes) -- [x] Health checks (implicit in dependencies) -- [ ] Resource limits (RECOMMENDED) -- [ ] GitHub Actions CI/CD (RECOMMENDED) -- [ ] Automated dependency updates (RECOMMENDED) -- [ ] Automated upstream sync (per user requirements) - ---- - -## 9. Upstream Sync Automation - -### 9.1 Current Sync Status - -**Fork:** okapteinis/SurfSense -**Upstream:** MODSetter/SurfSense -**Branch Strategy:** -- `main` - stable releases, synced with upstream -- `nightly` - development branch for testing -- `claude/*` - audit/feature branches - -### 9.2 Recommended Sync Workflow - -**Per user instructions:** - -```bash -# Step 1: Add upstream remote (one-time) -git remote add upstream https://github.com/MODSetter/SurfSense -git remote -v # Verify - -# Step 2: Fetch upstream changes -git fetch upstream - -# Step 3: Sync main branch -git checkout main -git merge upstream/main --no-edit -git push origin main - -# Step 4: Sync nightly from main (after audit) -git checkout nightly -git merge main --no-edit -git push origin nightly -``` - -### 9.3 Automated Sync (GitHub Actions) - -**See Appendix B** for complete GitHub Actions workflow. - -**Schedule:** Daily at 00:00 UTC -**Trigger:** Manual dispatch available -**Safety:** Creates PR instead of direct push (requires approval) - ---- - -## 10. TODOs for Future Maintainers - -### 10.1 Immediate Actions - -- [ ] Review and apply .gitignore enhancements (Appendix A) -- [ ] Add non-root Docker user to Dockerfile -- [ ] Document upstream sync workflow in CONTRIBUTING.md -- [ ] Create GitHub Actions workflow for automated sync (Appendix B) - -### 10.2 Short-Term (1-2 weeks) - -- [ ] Add SPDX license identifiers to source files (script in Appendix C) -- [ ] Enable Dependabot for automated dependency updates -- [ ] Add resource limits to docker-compose.yml for production -- [ ] Create API reference documentation (OpenAPI/Swagger) -- [ ] Document disaster recovery procedures - -### 10.3 Long-Term (1-3 months) - -- [ ] Set up GitHub Actions CI/CD pipeline -- [ ] Add end-to-end testing with Playwright -- [ ] Create security.txt for vulnerability disclosure -- [ ] Implement monitoring/observability (OpenTelemetry) -- [ ] Add load testing for production readiness -- [ ] Create migration guides for major version upgrades - ---- - -## Appendices - -### Appendix A: Enhanced .gitignore - -```gitignore -# SurfSense - Comprehensive .gitignore -# Generated by Claude Code Audit (2025-11-17) - -# ============================================================================ -# SECRETS & CREDENTIALS (CRITICAL) -# ============================================================================ -.env -.env.local -.env.*.local -.env.production -.env.development -.env.staging - -# API Keys & Credentials -*.key -*.pem -*.cert -*.crt -*.p12 -*.pfx -credentials.json -secrets.* -secret.* -auth.json -service-account*.json - -# ============================================================================ -# PYTHON -# ============================================================================ -__pycache__/ -*.py[cod] -*$py.class -*.so -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# Virtual Environments -venv/ -ENV/ -env/ -.venv - -# Testing & Coverage -.pytest_cache/ -.coverage -.coverage.* -htmlcov/ -.tox/ -.nox/ -.hypothesis/ -nosetests.xml -coverage.xml -*.cover -.cache - -# Jupyter Notebook -.ipynb_checkpoints - -# Ruff -.ruff_cache/ - -# MyPy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pytype -.pytype/ - -# ============================================================================ -# NODE.JS / JAVASCRIPT / TYPESCRIPT -# ============================================================================ -node_modules/ -npm-debug.log* -yarn-debug.log* -yarn-error.log* -.npm -.yarn-integrity -.pnpm-debug.log* - -# Next.js -.next/ -out/ -.turbo/ - -# Build outputs -dist/ -build/ -*.tsbuildinfo - -# Testing -coverage/ - -# ============================================================================ -# DATABASES -# ============================================================================ -*.db -*.sqlite -*.sqlite3 -*.db-shm -*.db-wal - -# PostgreSQL -*.dump -*.backup - -# ============================================================================ -# LOGS -# ============================================================================ -logs/ -*.log -npm-debug.log* -yarn-debug.log* -yarn-error.log* -lerna-debug.log* -.pnpm-debug.log* - -# ============================================================================ -# OS SPECIFIC -# ============================================================================ -.DS_Store -.DS_Store? -._* -.Spotlight-V100 -.Trashes -ehthumbs.db -Thumbs.db -Desktop.ini - -# ============================================================================ -# IDE / EDITOR -# ============================================================================ -.vscode/ -.idea/ -*.swp -*.swo -*~ -*.bak -*.tmp - -# ============================================================================ -# PROJECT SPECIFIC -# ============================================================================ -# Cache directories -.flashrank_cache* -.ruff_cache/ - -# Generated content -podcasts/ - -# Temporary files -temp/ -tmp/ -*.tmp - -# Docker volumes (if using local bind mounts) -postgres_data/ -redis_data/ -pgadmin_data/ - -# ============================================================================ -# MISC -# ============================================================================ -.pytest_cache/ -.mypy_cache/ -.dmypy.json -.pyre/ -celerybeat-schedule -celerybeat.pid -``` - -### Appendix B: GitHub Actions Upstream Sync Workflow - -Create `.github/workflows/sync-upstream.yml`: - -```yaml -name: Sync Fork with Upstream - -on: - schedule: - # Run daily at 00:00 UTC - - cron: '0 0 * * *' - workflow_dispatch: # Allow manual trigger - -jobs: - sync: - runs-on: ubuntu-latest - permissions: - contents: write - pull-requests: write - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Fetch all history - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Configure Git - run: | - git config user.name "GitHub Actions Bot" - git config user.email "actions@github.com" - - - name: Add upstream remote - run: | - git remote add upstream https://github.com/MODSetter/SurfSense.git || true - git remote -v - - - name: Fetch upstream changes - run: | - git fetch upstream - git fetch origin - - - name: Check for upstream changes - id: check - run: | - UPSTREAM_SHA=$(git rev-parse upstream/main) - ORIGIN_SHA=$(git rev-parse origin/main) - if [ "$UPSTREAM_SHA" != "$ORIGIN_SHA" ]; then - echo "changes=true" >> $GITHUB_OUTPUT - echo "Upstream has new commits" - else - echo "changes=false" >> $GITHUB_OUTPUT - echo "No upstream changes" - fi - - - name: Create sync branch - if: steps.check.outputs.changes == 'true' - run: | - BRANCH_NAME="sync/upstream-$(date +%Y%m%d-%H%M%S)" - git checkout -b $BRANCH_NAME origin/main - echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV - - - name: Merge upstream changes - if: steps.check.outputs.changes == 'true' - run: | - git merge upstream/main --no-edit || { - echo "Merge conflict detected. Manual intervention required." - exit 1 - } - - - name: Push sync branch - if: steps.check.outputs.changes == 'true' - run: | - git push origin $BRANCH_NAME - - - name: Create Pull Request - if: steps.check.outputs.changes == 'true' - uses: peter-evans/create-pull-request@v6 - with: - token: ${{ secrets.GITHUB_TOKEN }} - branch: ${{ env.BRANCH_NAME }} - base: main - title: "๐Ÿ”„ Sync with upstream MODSetter/SurfSense" - body: | - ## Automated Upstream Sync - - This PR syncs the fork with the latest changes from [MODSetter/SurfSense](https://github.com/MODSetter/SurfSense). - - **Changes:** - - Merged commits from upstream/main - - Review carefully for potential conflicts or breaking changes - - **Checklist before merge:** - - [ ] Review upstream commits for compatibility - - [ ] Run security audit if needed (see `claude.md`) - - [ ] Test locally before merging - - [ ] Ensure licensing and co-authorship are preserved - - **After merge:** - - Sync `nightly` branch: `git checkout nightly && git merge main && git push` - - Rerun security audit if significant changes detected - - --- - *Generated by [GitHub Actions Sync Workflow](/.github/workflows/sync-upstream.yml)* - *Date: $(date -u +"%Y-%m-%d %H:%M:%S UTC")* - labels: | - automated - upstream-sync - - - name: Summary - if: steps.check.outputs.changes == 'false' - run: | - echo "โœ… No upstream changes detected. Fork is up to date." -``` - -### Appendix C: Add SPDX License Headers (Script) - -Create `scripts/add_license_headers.py`: - -```python -#!/usr/bin/env python3 -""" -Add SPDX license identifiers to source files. -Usage: python scripts/add_license_headers.py -""" - -import os -from pathlib import Path - -PYTHON_HEADER = """# SPDX-License-Identifier: Apache-2.0 -# Copyright 2025 SurfSense Contributors -""" - -TYPESCRIPT_HEADER = """// SPDX-License-Identifier: Apache-2.0 -// Copyright 2025 SurfSense Contributors -""" - -def add_header_to_file(filepath: Path, header: str): - """Add license header to file if not already present.""" - content = filepath.read_text(encoding='utf-8') - - if 'SPDX-License-Identifier' in content: - print(f"โญ๏ธ Skipping {filepath} (already has header)") - return False - - new_content = header + '\n' + content - filepath.write_text(new_content, encoding='utf-8') - print(f"โœ… Added header to {filepath}") - return True - -def main(): - root = Path(__file__).parent.parent - count = 0 - - # Python files - for py_file in root.rglob('*.py'): - if 'venv' in str(py_file) or 'node_modules' in str(py_file): - continue - if add_header_to_file(py_file, PYTHON_HEADER): - count += 1 - - # TypeScript/JavaScript files - for ts_file in root.rglob('*.ts'): - if 'node_modules' in str(ts_file): - continue - if add_header_to_file(ts_file, TYPESCRIPT_HEADER): - count += 1 - - for tsx_file in root.rglob('*.tsx'): - if 'node_modules' in str(tsx_file): - continue - if add_header_to_file(tsx_file, TYPESCRIPT_HEADER): - count += 1 - - print(f"\nโœ… Added license headers to {count} files") - -if __name__ == '__main__': - main() -``` - -Run with: `python scripts/add_license_headers.py` - ---- - -## Site Configuration System - -**Implementation Date:** 2025-11-18 -**Implemented By:** Claude AI Assistant (Anthropic Sonnet 4.5) -**Feature:** Dynamic Site Appearance Configuration - -### Overview - -The site configuration system provides administrators with a centralized, database-driven approach to control the visibility and behavior of homepage elements, navigation links, footer sections, and route availability. This eliminates hardcoded UI elements and enables runtime customization without code changes. - -### Architecture - -**Backend Stack:** -- **Database:** PostgreSQL singleton pattern (single row with id=1) -- **ORM:** SQLAlchemy with AsyncSession -- **Migration:** Alembic migration #38 -- **API Framework:** FastAPI with Pydantic validation -- **Access Control:** Superuser-only admin endpoints, public read endpoint - -**Frontend Stack:** -- **State Management:** React Context API (SiteConfigContext) -- **UI Framework:** Next.js 15.5.6 App Router -- **Conditional Rendering:** Client-side based on configuration -- **Route Guards:** RouteGuard component for disabled routes - -### Database Schema - -**Table:** `site_configuration` (singleton - only 1 row) - -```sql -CREATE TABLE site_configuration ( - id INTEGER PRIMARY KEY, -- Always 1 (singleton) - - -- Header/Navbar toggles - show_pricing_link BOOLEAN DEFAULT FALSE, - show_docs_link BOOLEAN DEFAULT FALSE, - show_github_link BOOLEAN DEFAULT FALSE, - show_sign_in BOOLEAN DEFAULT TRUE, - - -- Homepage toggles - show_get_started_button BOOLEAN DEFAULT FALSE, - show_talk_to_us_button BOOLEAN DEFAULT FALSE, - - -- Footer toggles - show_pages_section BOOLEAN DEFAULT FALSE, - show_legal_section BOOLEAN DEFAULT FALSE, - show_register_section BOOLEAN DEFAULT FALSE, - - -- Route disabling - disable_pricing_route BOOLEAN DEFAULT TRUE, - disable_docs_route BOOLEAN DEFAULT TRUE, - disable_contact_route BOOLEAN DEFAULT TRUE, - disable_terms_route BOOLEAN DEFAULT TRUE, - disable_privacy_route BOOLEAN DEFAULT TRUE, - - -- Custom text - custom_copyright VARCHAR(200) DEFAULT 'SurfSense 2025', - - CONSTRAINT check_singleton CHECK (id = 1) -); -``` - -**Migration:** `surfsense_backend/alembic/versions/38_add_site_configuration_table.py` - -### API Endpoints - -#### 1. Public Configuration (Unauthenticated) - -```http -GET /api/v1/site-config/public -``` - -**Response:** -```json -{ - "show_pricing_link": false, - "show_docs_link": false, - "show_github_link": false, - "show_sign_in": true, - "show_get_started_button": false, - "show_talk_to_us_button": false, - "show_pages_section": false, - "show_legal_section": false, - "show_register_section": false, - "disable_pricing_route": true, - "disable_docs_route": true, - "disable_contact_route": true, - "disable_terms_route": true, - "disable_privacy_route": true, - "custom_copyright": "SurfSense 2025" -} -``` - -#### 2. Admin Configuration (Superuser Only) - -```http -GET /api/v1/site-config -Authorization: Bearer -``` - -**Response:** Same as public endpoint - -#### 3. Update Configuration (Superuser Only) - -```http -PUT /api/v1/site-config -Authorization: Bearer -Content-Type: application/json - -{ - "show_pricing_link": true, - "custom_copyright": "ยฉ MyCompany 2025" -} -``` - -**Response:** Updated configuration object - -### Frontend Integration - -#### 1. Global Context - -**File:** `surfsense_web/contexts/SiteConfigContext.tsx` - -```typescript -const { config, isLoading, error, refetch } = useSiteConfig(); - -// Access any configuration value -const showPricing = config.show_pricing_link; -const copyright = config.custom_copyright; -``` - -**Initialization:** Wrapped in `app/layout.tsx` for global availability - -#### 2. Conditional Rendering Examples - -**Navbar** (`components/homepage/navbar.tsx`): -```tsx -{config.show_pricing_link && !config.disable_pricing_route && ( - Pricing -)} - -{config.show_github_link && ( - - - -)} -``` - -**Homepage Hero** (`components/homepage/hero-section.tsx`): -```tsx -{config.show_get_started_button && ( - - Get Started - -)} -``` - -**Footer** (`components/homepage/footer.tsx`): -```tsx -

© {config.custom_copyright || "SurfSense 2025"}

- -{config.show_legal_section && ( -
- {!config.disable_terms_route && Terms} - {!config.disable_privacy_route && Privacy} -
-)} -``` - -#### 3. Route Guarding - -**File:** `components/RouteGuard.tsx` - -```tsx - - - -``` - -**Protected Routes:** -- `/pricing` โ†’ checks `disable_pricing_route` -- `/contact` โ†’ checks `disable_contact_route` -- `/terms` โ†’ checks `disable_terms_route` -- `/privacy` โ†’ checks `disable_privacy_route` - -**Behavior:** Redirects to `/404` if route is disabled - -### Admin Panel - -**Location:** `/dashboard/site-settings` - -**Features:** -- โœ… Visual toggle switches for all boolean flags -- โœ… Text input for custom copyright -- โœ… Real-time updates via API -- โœ… Organized by section (Header, Homepage, Footer, Routes, Custom Text) -- โœ… Descriptions for each toggle -- โœ… Save button with loading state -- โœ… Toast notifications for success/error - -**Access Control:** Requires superuser authentication (JWT token) - -**UI Components:** -- Custom ToggleSwitch component with animated transitions -- Gradient save button matching brand colors -- Dark mode support -- Responsive design (mobile-friendly) - -### Configuration Options - -#### Header & Navigation -| Toggle | Description | Default | -|--------|-------------|---------| -| `show_pricing_link` | Show pricing link in navbar | `false` | -| `show_docs_link` | Show docs link in navbar | `false` | -| `show_github_link` | Show GitHub icon in navbar | `false` | -| `show_sign_in` | Show sign in button | `true` | - -#### Homepage Buttons -| Toggle | Description | Default | -|--------|-------------|---------| -| `show_get_started_button` | Show "Get Started" CTA button | `false` | -| `show_talk_to_us_button` | Show "Talk to Us" CTA button | `false` | - -#### Footer Sections -| Toggle | Description | Default | -|--------|-------------|---------| -| `show_pages_section` | Show pages section (Pricing, Docs, Contact) | `false` | -| `show_legal_section` | Show legal section (Terms, Privacy) | `false` | -| `show_register_section` | Show register section (Sign Up, Sign In) | `false` | - -#### Route Disabling -| Toggle | Description | Default | -|--------|-------------|---------| -| `disable_pricing_route` | Disable /pricing route (404) | `true` | -| `disable_docs_route` | Disable /docs route (404) | `true` | -| `disable_contact_route` | Disable /contact route (404) | `true` | -| `disable_terms_route` | Disable /terms route (404) | `true` | -| `disable_privacy_route` | Disable /privacy route (404) | `true` | - -#### Custom Text -| Field | Description | Default | -|-------|-------------|---------| -| `custom_copyright` | Footer copyright text (max 200 chars) | `"SurfSense 2025"` | - -### Security Considerations - -1. **Access Control:** - - Admin endpoints require `is_superuser = true` - - Public endpoint is read-only and unauthenticated - - JWT token validation on all admin operations - -2. **Input Validation:** - - Pydantic schemas enforce data types - - String length limits (copyright max 200 chars) - - Boolean validation for all toggles - -3. **Singleton Pattern:** - - Database constraint ensures only 1 configuration row - - `get_or_create_config()` helper prevents missing config - - Atomic updates via SQLAlchemy transactions - -4. **Client-Side Security:** - - Configuration fetched at app startup - - No sensitive data exposed (public-facing settings only) - - RouteGuard prevents access to disabled routes - -### Migration Guide - -#### Step 1: Apply Database Migration - -```bash -cd surfsense_backend -alembic upgrade head -``` - -This creates the `site_configuration` table and inserts the default row. - -#### Step 2: Verify Configuration - -```bash -# Check that singleton row exists -psql -d surfsense -c "SELECT * FROM site_configuration WHERE id = 1;" -``` - -#### Step 3: Access Admin Panel - -1. Log in as superuser -2. Navigate to `/dashboard/site-settings` -3. Configure toggles as desired -4. Click "Save Configuration" - -#### Step 4: Verify Frontend Changes - -1. Visit homepage at `/` -2. Check navbar for conditional links -3. Check footer for conditional sections -4. Test disabled routes (should show 404) - -### Testing Checklist - -- [x] Database migration runs successfully -- [x] Singleton row created with default values -- [x] Public API endpoint returns configuration (unauthenticated) -- [x] Admin GET endpoint requires authentication -- [x] Admin PUT endpoint requires superuser role -- [x] Frontend context loads configuration on app startup -- [x] Navbar shows/hides links based on config -- [x] Homepage shows/hides buttons based on config -- [x] Footer shows/hides sections based on config -- [x] Custom copyright displays correctly -- [x] RouteGuard redirects disabled routes to 404 -- [x] Admin panel UI loads and displays current config -- [x] Admin panel saves changes successfully -- [x] Changes reflect immediately after save (via refetch) - -### Files Changed - -**Backend:** -``` -surfsense_backend/app/db.py # Added SiteConfiguration model -surfsense_backend/alembic/versions/38_add_site_configuration_table.py # Migration -surfsense_backend/app/schemas/site_configuration.py # Pydantic schemas -surfsense_backend/app/routes/site_configuration_routes.py # API routes -surfsense_backend/app/routes/__init__.py # Route registration -``` - -**Frontend:** -``` -surfsense_web/contexts/SiteConfigContext.tsx # React context -surfsense_web/app/layout.tsx # Context provider -surfsense_web/components/RouteGuard.tsx # Route guard component -surfsense_web/components/homepage/navbar.tsx # Conditional navbar -surfsense_web/components/homepage/hero-section.tsx # Conditional buttons -surfsense_web/components/homepage/footer.tsx # Conditional footer -surfsense_web/app/(home)/pricing/page.tsx # Route guard -surfsense_web/app/(home)/contact/page.tsx # Route guard -surfsense_web/app/(home)/terms/page.tsx # Route guard -surfsense_web/app/(home)/privacy/page.tsx # Route guard -surfsense_web/app/dashboard/site-settings/page.tsx # Admin UI -``` - -### Benefits - -1. **No Code Deployments:** Site appearance changes without code changes -2. **Self-Service:** Administrators manage UI without developer intervention -3. **Branding Flexibility:** Custom copyright text for white-label deployments -4. **Privacy-Focused:** Disable routes/features you don't need -5. **Minimal UI:** Default configuration hides all non-essential elements -6. **Type Safety:** Full TypeScript support with validated schemas -7. **Performance:** Configuration cached in React context (no repeated API calls) -8. **Scalability:** Singleton pattern ensures consistent state across all users - -### Future Enhancements - -**Potential additions (not yet implemented):** -- Multi-tenant configurations (per-user or per-organization) -- Custom navigation links (dynamic menu items) -- Theme color customization -- Logo upload and management -- Custom homepage headline/tagline -- Feature flag management for experimental features -- Analytics opt-out configuration -- GDPR compliance toggles - -### Comparison to Previous Approach - -**Before (Hardcoded):** -- UI elements always visible -- Changes required code modifications -- No runtime customization -- Developer-only control - -**After (Database-Driven):** -- UI elements conditionally rendered -- Changes via admin panel -- Runtime customization -- Self-service for admins - -### Support - -For questions or issues related to site configuration: -- Review API documentation: `surfsense_backend/app/routes/site_configuration_routes.py` -- Check schema definitions: `surfsense_backend/app/schemas/site_configuration.py` -- Examine frontend context: `surfsense_web/contexts/SiteConfigContext.tsx` -- Access admin panel: `/dashboard/site-settings` - ---- - -## Google OAuth Removal for User Authentication - -**Implementation Date:** 2025-11-18 -**Implemented By:** Claude AI Assistant (Anthropic Sonnet 4.5) -**Feature:** Email/Password-Only Authentication - -### Overview - -Removed Google OAuth as an authentication method for user accounts, enforcing email/password authentication only. Google OAuth credentials are now exclusively used for Gmail and Google Calendar connector integrations, not for user sign-in/registration. - -### Changes Implemented - -#### Backend Configuration -**File:** `surfsense_backend/.env.example` - -- Changed `AUTH_TYPE` from `GOOGLE or LOCAL` to `LOCAL` (email/password only) -- Updated Google OAuth credentials documentation to clarify they're only for Gmail/Calendar connectors -- Removed misleading "For Google Auth Only" comment - -**Before:** -```env -AUTH_TYPE=GOOGLE or LOCAL -# For Google Auth Only -GOOGLE_OAUTH_CLIENT_ID=924507538m -GOOGLE_OAUTH_CLIENT_SECRET=GOCSV -``` - -**After:** -```env -AUTH_TYPE=LOCAL -# Google OAuth Credentials (OPTIONAL - Required only for Gmail and Google Calendar connectors) -GOOGLE_OAUTH_CLIENT_ID=your_google_client_id -GOOGLE_OAUTH_CLIENT_SECRET=your_google_client_secret -``` - -#### Frontend Configuration -**File:** `surfsense_web/.env.example` - -- Updated `NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE` to `LOCAL` only -- Removed `or GOOGLE` option - -#### Frontend Components -**File:** `surfsense_web/app/(home)/login/LocalLoginForm.tsx` - -**Changes:** -- Removed `authType` state variable (line 27) -- Removed `useEffect` that checked `NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE` environment variable (lines 31-34) -- Removed conditional check for showing registration link (line 236) -- Now always displays "Don't have an account? Sign up" link - -**Rationale:** Since authentication is email/password only, the registration link should always be visible. The authType check was unnecessary. - -#### Documentation Updates -**File:** `surfsense_web/content/docs/index.mdx` - -**Changes:** -- Updated Auth Setup section to state "SurfSense uses email/password authentication for user accounts" -- Removed references to deleted Google OAuth setup images (google_oauth_people_api.png, google_oauth_screen.png, google_oauth_client.png, google_oauth_config.png) -- Clarified Google OAuth is only for Gmail/Calendar connectors -- Added redirect URI examples for connector setup - -**Before:** -```markdown -## Auth Setup -SurfSense supports both Google OAuth and local email/password authentication... -![Google Developer Console People API](/docs/google_oauth_people_api.png) -``` - -**After:** -```markdown -## Auth Setup -SurfSense uses email/password authentication for user accounts. - -### Google OAuth for Connectors (Optional) -**Note**: Google OAuth setup is **required** only if you want to use the Gmail and Google Calendar connectors... -``` - -### What Was NOT Changed - -The following components were intentionally preserved: - -1. **OAuth Error Handling** (`surfsense_web/lib/auth-errors.ts`): - - Generic OAuth error codes (lines 65-93) remain because they're needed for connector OAuth flows - - Error handling applies to any OAuth integration, not just user authentication - -2. **Backend OAuth Configuration** (`surfsense_backend/app/config/__init__.py`): - - `GOOGLE_OAUTH_CLIENT_ID` and `GOOGLE_OAUTH_CLIENT_SECRET` environment variables still loaded - - Still used by `google_gmail_add_connector_route.py` and `google_calendar_add_connector_route.py` - -3. **Backend User Authentication Logic** (`surfsense_backend/app/users.py`, `app/app.py`, `app/db.py`): - - Conditional logic checking `AUTH_TYPE == "GOOGLE"` remains - - Allows future re-enabling of Google OAuth if needed - - No breaking changes to existing authentication flow when `AUTH_TYPE=LOCAL` - -4. **Connector Integration**: - - Google Calendar connector (`app/dashboard/[search_space_id]/connectors/add/google-calendar-connector/page.tsx`) - - Google Gmail connector (`app/dashboard/[search_space_id]/connectors/add/google-gmail-connector/page.tsx`) - - Both still use Google OAuth for accessing user's Google data - -### Deployment Steps Completed - -1. โœ… Updated local repository to latest nightly branch -2. โœ… Identified and analyzed all OAuth references in codebase -3. โœ… Removed authType state check from LocalLoginForm -4. โœ… Updated `.env.example` files (frontend and backend) -5. โœ… Fixed documentation to remove deleted OAuth images -6. โœ… Committed changes with descriptive messages: - - Commit 1 (586ab48): "Remove Google OAuth from login/register, enforce email-only authentication" - - Commit 2 (a148614): "Fix documentation: Remove deleted Google OAuth images, clarify auth is email-only" -7. โœ… Pushed changes to `nightly` branch -8. โœ… Deployed to VPS at 46.62.230.195 -9. โœ… Rebuilt frontend on VPS (`pnpm build`) -10. โœ… Restarted all services: - - surfsense.service (Backend) - - surfsense-frontend.service (Frontend) - - surfsense-celery.service (Worker) - - surfsense-celery-beat.service (Scheduler) -11. โœ… Verified deployment at https://ai.kapteinis.lv/login and https://ai.kapteinis.lv/register - -### Verification Results - -**Login Page** (`https://ai.kapteinis.lv/login`): -- โœ… Email and password fields present -- โœ… "Don't have an account? Sign up" link visible -- โœ… No Google OAuth or "Continue with Google" button -- โœ… Page loads correctly without errors -- โœ… Only email/password authentication available - -**Register Page** (`https://ai.kapteinis.lv/register`): -- โœ… Email, password, and confirm password fields present -- โœ… No Google OAuth references -- โœ… "Already have an account? Sign In" link visible -- โœ… Page loads successfully - -### Files Modified - -**Backend:** -``` -surfsense_backend/.env.example # AUTH_TYPE=LOCAL, clarified OAuth usage -``` - -**Frontend:** -``` -surfsense_web/.env.example # AUTH_TYPE=LOCAL -surfsense_web/app/(home)/login/LocalLoginForm.tsx # Removed authType check -surfsense_web/content/docs/index.mdx # Updated auth documentation -``` - -### Benefits - -1. **Simplified Authentication**: Users no longer confused by multiple auth methods -2. **Reduced Dependencies**: No dependency on Google OAuth service for user authentication -3. **Privacy-Focused**: User accounts managed entirely within SurfSense -4. **Clear Documentation**: Distinction between user auth and connector OAuth -5. **Maintained Functionality**: Gmail/Calendar connectors still work with OAuth - -### Future Considerations - -If Google OAuth needs to be re-enabled for user authentication: -1. Set `AUTH_TYPE=GOOGLE` in backend `.env` -2. Set `NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE=GOOGLE` in frontend `.env` -3. Provide valid `GOOGLE_OAUTH_CLIENT_ID` and `GOOGLE_OAUTH_CLIENT_SECRET` -4. Backend conditional logic will automatically enable Google OAuth routes - -The backend code architecture supports both authentication methods through configuration, making this a non-breaking change. - -### Testing Checklist - -- [x] Login page shows only email/password fields -- [x] Register page shows only email/password fields -- [x] "Don't have an account?" link always visible on login -- [x] "Already have an account?" link visible on register -- [x] No Google OAuth button or references in UI -- [x] Documentation updated to reflect email-only auth -- [x] .env.example files updated -- [x] Frontend builds successfully -- [x] Services restart without errors -- [x] Login page loads at https://ai.kapteinis.lv/login -- [x] Register page loads at https://ai.kapteinis.lv/register -- [x] Gmail connector OAuth still functional (preserved) -- [x] Calendar connector OAuth still functional (preserved) - -### Support - -For questions or issues related to authentication: -- Review backend users configuration: `surfsense_backend/app/users.py` -- Check authentication routes: `surfsense_backend/app/app.py` -- Examine frontend login form: `surfsense_web/app/(home)/login/LocalLoginForm.tsx` -- Verify environment variables in `.env` files - ---- - -## Registration Toggle System - -**Implementation Date:** 2025-11-18 -**Implemented By:** Claude AI Assistant (Anthropic Sonnet 4.5) -**Feature:** Disable User Registration via Admin Panel - -### Overview - -Implemented a comprehensive registration control system that allows administrators to disable new user registrations through the admin panel. This provides better control over user access and supports closed registration scenarios for private deployments. - -### Implementation Summary - -The registration toggle system operates at multiple layers: - -1. **Database Layer**: Added `disable_registration` boolean field to `site_configuration` table (migration #39) -2. **Backend API Layer**: Updated `registration_allowed()` dependency to check database toggle and return 403 when disabled -3. **Frontend UI Layer**: Conditionally hide "Sign Up" link on login page based on configuration -4. **Frontend Route Layer**: RouteGuard component protects `/register` route, showing 404 when disabled -5. **Admin Panel Layer**: Added "Registration Control" section to site-settings page for easy toggling - -### Backend Changes - -#### Database Migration (#39) - -**File:** `surfsense_backend/alembic/versions/39_add_disable_registration_to_site_configuration.py` - -```python -def upgrade() -> None: - op.add_column( - 'site_configuration', - sa.Column('disable_registration', sa.Boolean(), nullable=False, server_default='false') - ) - -def downgrade() -> None: - op.drop_column('site_configuration', 'disable_registration') -``` - -**Revision ID:** `'39'` (shortened to fit varchar(32) constraint) -**Down Revision:** `'38_add_site_configuration_table'` -**Default Value:** `false` (registration enabled by default) - -#### Database Model - -**File:** `surfsense_backend/app/db.py` - -```python -class SiteConfiguration(SQLModel, table=True): - # ... existing fields ... - - # Registration control - disable_registration = Column(Boolean, nullable=False, default=False) -``` - -#### API Schemas - -**File:** `surfsense_backend/app/schemas/site_configuration.py` - -```python -class SiteConfigurationBase(BaseModel): - # ... existing fields ... - - # Registration control - disable_registration: bool = False - -class SiteConfigurationUpdate(SiteConfigurationBase): - # ... existing fields ... - disable_registration: bool | None = None -``` - -#### Registration Endpoint Protection - -**File:** `surfsense_backend/app/app.py` - -```python -async def registration_allowed(session: AsyncSession = Depends(get_async_session)): - # Check environment variable first - if not config.REGISTRATION_ENABLED: - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail="Registration is disabled by system configuration" - ) - - # Check site configuration database toggle - result = await session.execute(select(SiteConfiguration).where(SiteConfiguration.id == 1)) - site_config = result.scalar_one_or_none() - - if site_config and site_config.disable_registration: - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail="Registration is currently disabled. Please contact the administrator if you need access." - ) - - return True - -# Applied to registration routes -app.include_router( - fastapi_users.get_register_router(UserRead, UserCreate), - prefix="/auth", - tags=["auth"], - dependencies=[Depends(registration_allowed)], # Enforces registration check -) -``` - -**Error Response** (when disabled): -```json -{ - "detail": "Registration is currently disabled. Please contact the administrator if you need access." -} -``` - -### Frontend Changes - -#### Context/State Management - -**File:** `surfsense_web/contexts/SiteConfigContext.tsx` - -```typescript -export interface SiteConfig { - // ... existing fields ... - - // Registration control - disable_registration: boolean; -} - -const defaultConfig: SiteConfig = { - // ... existing fields ... - disable_registration: false, - // ... -}; -``` - -#### Login Form - -**File:** `surfsense_web/app/(home)/login/LocalLoginForm.tsx` - -**Changes:** -- Added `useSiteConfig()` hook to access site configuration -- Conditionally render "Sign up" link only when `!config.disable_registration` - -```typescript -import { useSiteConfig } from "@/contexts/SiteConfigContext"; - -export function LocalLoginForm() { - const { config } = useSiteConfig(); - - return ( - {/* ... login form ... */} - - {!config.disable_registration && ( -
-

- {t("dont_have_account")}{" "} - - {t("sign_up")} - -

-
- )} - ); -} -``` - -#### Route Guard - -**File:** `surfsense_web/components/RouteGuard.tsx` - -**Changes:** -- Extended `routeKey` type to include `"registration"` -- Added special case handling for registration toggle - -```typescript -interface RouteGuardProps { - children: React.ReactNode; - routeKey: "pricing" | "docs" | "contact" | "terms" | "privacy" | "registration"; -} - -export function RouteGuard({ children, routeKey }: RouteGuardProps) { - const { config, isLoading } = useSiteConfig(); - const router = useRouter(); - - // Special case: registration uses disable_registration instead of disable_registration_route - const disableKey = routeKey === "registration" - ? "disable_registration" - : (`disable_${routeKey}_route` as keyof typeof config); - const isDisabled = config[disableKey as keyof typeof config]; - - if (isDisabled) { - router.replace("/404"); - return null; - } - - return <>{children}; -} -``` - -#### Register Page - -**File:** `surfsense_web/app/(home)/register/page.tsx` - -**Changes:** -- Imported `RouteGuard` component -- Wrapped entire page content with `` - -```typescript -import { RouteGuard } from "@/components/RouteGuard"; - -export default function RegisterPage() { - return ( - - {/* ... registration form content ... */} - - ); -} -``` - -**Behavior:** When `disable_registration` is true, accessing `/register` redirects to `/404` - -#### Admin Panel - -**File:** `surfsense_web/app/dashboard/site-settings/page.tsx` - -**Changes:** -- Added `disable_registration: boolean` to `SiteConfigForm` interface -- Added default value `disable_registration: false` to form state -- Added field to `useEffect` that loads config from API -- Added new "Registration Control" section in UI - -```typescript -interface SiteConfigForm { - // ... existing fields ... - disable_registration: boolean; -} - -{/* Registration Control Section */} -
-

- Registration Control -

-

- Control user registration availability. When disabled, the Sign Up link will be hidden - and the registration page will show a 404 error. The backend will also block registration - requests with a 403 error. -

-
- handleToggle("disable_registration")} - /> -
-
-``` - -### Behavior Summary - -#### When Registration is ENABLED (default: `disable_registration = false`): - -1. **Login Page (`/login`):** - - Shows "Don't have an account? Sign up" link - - Link navigates to `/register` - -2. **Register Page (`/register`):** - - Page loads normally - - Registration form accepts submissions - - Backend POST `/auth/register` succeeds (200) - -3. **Admin Panel (`/dashboard/site-settings`):** - - "Disable Registration" toggle is OFF - - Description explains what happens when enabled - -#### When Registration is DISABLED (`disable_registration = true`): - -1. **Login Page (`/login`):** - - "Don't have an account? Sign up" link is HIDDEN - - No visual indication of registration functionality - -2. **Register Page (`/register`):** - - RouteGuard redirects to `/404` (route disabled) - - Page shows standard 404 error - -3. **Backend API (`POST /auth/register`):** - - Returns `403 Forbidden` - - Error message: "Registration is currently disabled. Please contact the administrator if you need access." - -4. **Admin Panel (`/dashboard/site-settings`):** - - "Disable Registration" toggle is ON (orange) - - Admins can toggle back to enable registration - -### Security Considerations - -1. **Layered Protection:** - - Frontend hides UI elements (Sign Up link) - - Frontend route guard blocks page access - - Backend API validates and blocks requests - - Defense in depth approach prevents bypass attempts - -2. **Database-Driven:** - - Configuration stored in PostgreSQL (singleton row with id=1) - - Atomic updates via SQLAlchemy transactions - - Consistent state across all application instances - -3. **Admin-Only Access:** - - Only superusers can access `/dashboard/site-settings` - - JWT authentication required for PUT `/api/v1/site-config` - - Public GET `/api/v1/site-config/public` is read-only - -4. **Graceful Error Messages:** - - User-friendly 403 message explains registration is disabled - - Directs users to contact administrator - - No technical details exposed - -### Deployment Steps - -1. โœ… **Pull latest code:** `git pull origin nightly` -2. โœ… **Apply migration:** `alembic upgrade head` (migration #39) -3. โœ… **Rebuild frontend:** `pnpm build` -4. โœ… **Restart services:** `systemctl restart surfsense.service surfsense-frontend.service surfsense-celery.service surfsense-celery-beat.service` -5. โœ… **Verify deployment:** All services active - -**Deployment Date:** 2025-11-18 -**Deployment Location:** https://ai.kapteinis.lv -**VPS:** 46.62.230.195 - -### Migration Notes - -#### Issue Encountered: Revision ID Too Long - -**Problem:** Alembic `alembic_version` table has `varchar(32)` constraint on `version_num` column. Original revision ID `'39_add_disable_registration_to_site_configuration'` (50 characters) exceeded limit. - -**Error:** -``` -sqlalchemy.exc.DBAPIError: : -value too long for type character varying(32) -[SQL: UPDATE alembic_version SET version_num='39_add_disable_registration_to_site_configuration' -WHERE alembic_version.version_num = '38_add_site_configuration_table'] -``` - -**Solution:** Shortened revision ID using sed on VPS: -```bash -sed -i "s/revision: str = '39_add_disable_registration_to_site_configuration'/revision: str = '39'/g" \ - alembic/versions/39_add_disable_registration_to_site_configuration.py -``` - -**Final Revision ID:** `'39'` (2 characters, fits varchar(32)) - -**Note:** Migration #38 uses long ID `'38_add_site_configuration_table'` but succeeded before (possibly database was modified to allow longer IDs, or varchar constraint was later added). - -### Testing Checklist - -**Default State (Registration Enabled):** -- [x] Login page shows "Sign up" link -- [x] `/register` page loads successfully -- [x] Registration form accepts submissions -- [x] Backend POST `/auth/register` returns 200/201 -- [x] Admin panel toggle is OFF (not orange) - -**Toggled State (Registration Disabled):** -- [ ] Admin can toggle "Disable Registration" ON in `/dashboard/site-settings` -- [ ] Changes saved successfully (toast notification) -- [ ] Login page "Sign up" link disappears after refetch -- [ ] `/register` page shows 404 error -- [ ] Backend POST `/auth/register` returns 403 with message -- [ ] Admin panel toggle is ON (orange highlight) - -**Toggle Back (Re-enable Registration):** -- [ ] Admin can toggle "Disable Registration" OFF -- [ ] Login page "Sign up" link reappears -- [ ] `/register` page loads normally -- [ ] Backend POST `/auth/register` accepts submissions again - -### Files Modified - -**Backend (4 files + 1 migration):** -``` -surfsense_backend/alembic/versions/39_add_disable_registration_to_site_configuration.py # NEW -surfsense_backend/app/db.py # +3 lines -surfsense_backend/app/app.py # +20 lines -surfsense_backend/app/schemas/site_configuration.py # +4 lines -``` - -**Frontend (5 files):** -``` -surfsense_web/contexts/SiteConfigContext.tsx # +4 lines -surfsense_web/app/(home)/login/LocalLoginForm.tsx # +11 lines, -15 lines (conditional) -surfsense_web/app/(home)/register/page.tsx # +2 lines (import, wrapper) -surfsense_web/components/RouteGuard.tsx # +10 lines (registration support) -surfsense_web/app/dashboard/site-settings/page.tsx # +25 lines (new section) -``` - -**Total Changes:** 9 files modified, 1 file created, ~80 lines added - -### Commit Information - -**Commit Hash:** `b8bb74e` -**Branch:** `nightly` -**Message:** -``` -Add disable_registration toggle to site configuration - -Implemented comprehensive registration control system that allows administrators -to disable new user registrations through the admin panel. This provides better -control over user access and supports closed registration scenarios. - -Backend changes: -- Added disable_registration field to SiteConfiguration model (db.py) -- Created migration #39 to add disable_registration column with default false -- Updated SiteConfigurationBase, Update, and Read schemas -- Enhanced registration_allowed() dependency to check database toggle -- Returns 403 with clear message when registration is disabled - -Frontend changes: -- Added disable_registration to SiteConfig interface and default config -- Updated LocalLoginForm to conditionally hide "Sign up" link when disabled -- Added RouteGuard support for "registration" route key -- Protected /register page with RouteGuard (shows 404 when disabled) -- Added "Registration Control" section to admin site-settings page - -๐Ÿค– Generated with [Claude Code](https://claude.com/claude-code) - -Co-Authored-By: Claude -``` - -### Use Cases - -1. **Private Deployments:** - - Organization wants to control who can create accounts - - Only invite approved users, disable public registration - - Admin manually creates accounts for new employees - -2. **Capacity Management:** - - Server approaching resource limits - - Temporarily disable registration until scaling completed - - Re-enable when infrastructure ready - -3. **Closed Beta:** - - Limited initial user base for testing - - Disable registration after target number reached - - Re-enable for public launch - -4. **Security Incident Response:** - - Spam/abuse accounts being created - - Quickly disable registration to stop attack - - Investigate and re-enable when resolved - -### Troubleshooting - -#### Issue: Toggle doesn't save in admin panel - -**Symptoms:** Clicking toggle works, but changes don't persist after refresh - -**Diagnosis:** -1. Check browser console for API errors -2. Verify JWT token is valid (`localStorage.getItem('access_token')`) -3. Check backend logs for 401/403 errors -4. Confirm user is superuser (`is_superuser = true`) - -**Solution:** Ensure logged-in user has superuser privileges - -#### Issue: Sign Up link still visible after disabling registration - -**Symptoms:** Login page shows Sign Up link despite toggle being ON - -**Diagnosis:** -1. Check if frontend context refetched after save -2. Inspect SiteConfigContext state in React DevTools -3. Verify GET `/api/v1/site-config/public` returns `disable_registration: true` - -**Solution:** Hard refresh browser (Ctrl+Shift+R) or wait for automatic refetch - -#### Issue: /register page still loads instead of 404 - -**Symptoms:** Accessing `/register` shows form instead of 404 error - -**Diagnosis:** -1. Verify RouteGuard is wrapping RegisterPage component -2. Check SiteConfigContext has correct `disable_registration` value -3. Confirm RouteGuard logic handles "registration" route key - -**Solution:** Verify frontend build includes latest changes, restart frontend service - -#### Issue: Backend still accepts registrations despite toggle - -**Symptoms:** POST `/auth/register` returns 200 instead of 403 - -**Diagnosis:** -1. Check database: `SELECT disable_registration FROM site_configuration WHERE id = 1;` -2. Verify `registration_allowed()` function checks database -3. Confirm dependency is applied to register router - -**Solution:** Restart backend service to reload code changes - -### Future Enhancements - -**Potential additions (not yet implemented):** -- [ ] Registration whitelist (allow specific email domains) -- [ ] Invitation-based registration (one-time tokens) -- [ ] Registration queue/approval workflow -- [ ] Auto-disable after N registrations -- [ ] Custom messaging when registration disabled -- [ ] Email notification to admins when disabled -- [ ] Audit log of registration toggle changes -- [ ] Rate limiting for registration attempts - -### Support - -For questions or issues related to registration control: -- Review admin panel documentation: `/dashboard/site-settings` -- Check backend registration logic: `surfsense_backend/app/app.py:registration_allowed()` -- Examine frontend RouteGuard: `surfsense_web/components/RouteGuard.tsx` -- Verify SiteConfig context: `surfsense_web/contexts/SiteConfigContext.tsx` -- Migration file: `surfsense_backend/alembic/versions/39_add_disable_registration_to_site_configuration.py` - ---- - -## Conclusion - -**Overall Security Posture:** โœ… **STRONG** - -SurfSense demonstrates excellent security practices with automated tooling (detect-secrets, pre-commit hooks, bandit), no hardcoded secrets, and comprehensive architecture. The codebase is well-structured, uses modern technologies (Python 3.12, React 19, Next.js 15), and implements advanced RAG techniques with proper security controls (user isolation, search space filtering). - -**Key Strengths:** -- โœ… Automated secret scanning with detect-secrets -- โœ… Pre-commit hooks for security and code quality -- โœ… No real secrets in codebase (all placeholders verified) -- โœ… Modern, actively maintained tech stack -- โœ… Advanced RAG architecture (2-tier hierarchical + hybrid search) -- โœ… Comprehensive LLM support (100+ providers via LiteLLM) -- โœ… Excellent documentation (multi-language support) -- โœ… Apache 2.0 license (permissive, commercial-friendly) - -**Critical Actions Required:** -- None (no critical vulnerabilities found) - -**High-Priority Improvements:** -1. Enhance .gitignore (10 min) -2. Add Docker non-root user (5 min) - -**Medium-Priority Enhancements:** -3. Add SPDX license identifiers (2 hours, automatable) -4. Document upstream sync automation (1 hour) -5. Add resource limits to docker-compose.yml (30 min) - -**Audit Certification:** -This repository is **APPROVED** for continued development and production deployment with implementation of high-priority recommendations. - ---- - -**Audit Completed:** 2025-11-17 -**Next Audit Recommended:** 2025-12-17 (or after major upstream merge) - -**Co-Authors:** -- Ojฤrs Kapteiล†ลก - Audit Requester & Repository Maintainer -- Claude AI Assistant (Anthropic Sonnet 4.5) - Security & Architecture Audit - ---- - -*This audit report is licensed under CC BY 4.0. The audited code remains under Apache License 2.0.* diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index 4a47542aa..467078219 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -1,5 +1,8 @@ DATABASE_URL=postgresql+asyncpg://postgres:postgres@localhost:5432/surfsense +# Uploads directory for file storage +UPLOADS_DIR=./uploads + #Celery Config CELERY_BROKER_URL=redis://localhost:6379/0 CELERY_RESULT_BACKEND=redis://localhost:6379/0 diff --git a/surfsense_backend/app/routes/documents_routes.py b/surfsense_backend/app/routes/documents_routes.py index 4065deeb4..cb11faf80 100644 --- a/surfsense_backend/app/routes/documents_routes.py +++ b/surfsense_backend/app/routes/documents_routes.py @@ -114,7 +114,7 @@ async def create_documents_file_upload( from pathlib import Path # Create uploads directory if it doesn't exist - uploads_dir = Path("/opt/SurfSense/surfsense_backend/uploads") + uploads_dir = Path(os.getenv("UPLOADS_DIR", "./uploads")) uploads_dir.mkdir(parents=True, exist_ok=True) # Create unique filename diff --git a/surfsense_web/app/dashboard/[search_space_id]/documents/webpage/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/documents/webpage/page.tsx index 9da3ad927..fd6617d27 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/documents/webpage/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/documents/webpage/page.tsx @@ -18,8 +18,9 @@ import { import { Label } from "@/components/ui/label"; import { AUTH_TOKEN_KEY } from "@/lib/constants"; -// URL validation regex - updated to support percent-encoded URLs (e.g., Latvian characters) -const urlRegex = /^https?:\/\/[^\s]+$/; +// URL validation regex - requires valid domain structure while supporting international characters +// Accepts: localhost, domain.tld format, and percent-encoded URLs +const urlRegex = /^https?:\/\/(?:localhost|[\w.-]+\.[a-zA-Z]{2,})(?::\d+)?(?:\/[^\s]*)?$/; export default function WebpageCrawler() { const t = useTranslations("add_webpage"); diff --git a/sync-from-production.sh b/sync-from-production.sh deleted file mode 100755 index bc4f67e4c..000000000 --- a/sync-from-production.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -SERVER="root@46.62.230.195" -REMOTE_DIR="/opt/SurfSense/surfsense_backend" -LOCAL_DIR="$HOME/Documents/Kods/SurfSense/surfsense_backend" - -echo "Syncing backend files from production..." -rsync -avz --progress \ - --exclude='.env' \ - --exclude='*.env.local' \ - --exclude='*.env.production' \ - --exclude='node_modules/' \ - --exclude='__pycache__/' \ - --exclude='*.pyc' \ - --exclude='.pytest_cache/' \ - --exclude='venv/' \ - --exclude='.venv/' \ - --exclude='*.log' \ - --exclude='*.db' \ - --exclude='*.sqlite' \ - --exclude='*.pem' \ - --exclude='*.key' \ - --exclude='*.crt' \ - --exclude='.DS_Store' \ - -e "ssh -i ~/.ssh/id_ed25519_surfsense" \ - "$SERVER:$REMOTE_DIR/" "$LOCAL_DIR/" - -echo "Sync complete!"