Added local Speech-to-Text (STT) support using Faster-Whisper

surfsense_backend/.env.example

@@ -31,12 +31,15 @@ TTS_SERVICE_API_KEY=
 # OPTIONAL: TTS Provider API Base
 TTS_SERVICE_API_BASE=
 
-# LiteLLM STT Provider: https://docs.litellm.ai/docs/audio_transcription#supported-providers
-STT_SERVICE=openai/whisper-1
-# Respective STT Service API
-STT_SERVICE_API_KEY=""
-# OPTIONAL: STT Provider API Base
-STT_SERVICE_API_BASE=
+# STT Service Configuration
+# Use 'local' for offline Faster-Whisper or LiteLLM provider
+STT_SERVICE=local
+# For local STT: Whisper model size (tiny, base, small, medium, large-v3)
+LOCAL_STT_MODEL=base
+# For LiteLLM STT Provider: https://docs.litellm.ai/docs/audio_transcription#supported-providers
+# STT_SERVICE=openai/whisper-1
+# STT_SERVICE_API_KEY=""
+# STT_SERVICE_API_BASE=
 
 
 FIRECRAWL_API_KEY=fcr-01J0000000000000000000000

surfsense_backend/app/config/__init__.py

@@ -102,10 +102,13 @@ class Config:
     TTS_SERVICE_API_BASE = os.getenv("TTS_SERVICE_API_BASE")
     TTS_SERVICE_API_KEY = os.getenv("TTS_SERVICE_API_KEY")
 
-    # Litellm STT Configuration
+    # STT Configuration
     STT_SERVICE = os.getenv("STT_SERVICE")
     STT_SERVICE_API_BASE = os.getenv("STT_SERVICE_API_BASE")
     STT_SERVICE_API_KEY = os.getenv("STT_SERVICE_API_KEY")
+    
+    # Local STT Configuration
+    LOCAL_STT_MODEL = os.getenv("LOCAL_STT_MODEL", "base")
 
     # Validation Checks
     # Check embedding dimension

surfsense_backend/app/routes/__init__.py

@@ -17,6 +17,7 @@ from .luma_add_connector_route import router as luma_add_connector_router
 from .podcasts_routes import router as podcasts_router
 from .search_source_connectors_routes import router as search_source_connectors_router
 from .search_spaces_routes import router as search_spaces_router
+from .stt_routes import router as stt_router
 
 router = APIRouter()
 
@@ -31,3 +32,4 @@ router.include_router(airtable_add_connector_router)
 router.include_router(luma_add_connector_router)
 router.include_router(llm_config_router)
 router.include_router(logs_router)
+router.include_router(stt_router)

surfsense_backend/app/routes/stt_routes.py

@@ -0,0 +1,96 @@
+"""Speech-to-Text API routes."""
+
+from fastapi import APIRouter, File, Form, HTTPException, UploadFile
+from fastapi.responses import JSONResponse
+
+from app.services.stt_service import stt_service
+
+router = APIRouter(prefix="/stt", tags=["Speech-to-Text"])
+
+
+@router.post("/transcribe")
+async def transcribe_audio(
+    audio: UploadFile = File(..., description="Audio file to transcribe"),
+    language: str = Form(None, description="Optional language code (e.g., 'en', 'es')"),
+):
+    """Transcribe uploaded audio file to text."""
+    
+    # Validate file type
+    if not audio.content_type or not audio.content_type.startswith("audio/"):
+        raise HTTPException(
+            status_code=400, 
+            detail="File must be an audio file"
+        )
+    
+    try:
+        # Read audio bytes
+        audio_bytes = await audio.read()
+        
+        # Transcribe
+        result = stt_service.transcribe_bytes(
+            audio_bytes, 
+            filename=audio.filename or "audio.wav",
+            language=language if language else None
+        )
+        
+        return JSONResponse(content={
+            "success": True,
+            "transcription": result["text"],
+            "metadata": {
+                "detected_language": result["language"],
+                "language_probability": result["language_probability"],
+                "duration_seconds": result["duration"],
+                "model_size": stt_service.model_size,
+            }
+        })
+        
+    except Exception as e:
+        raise HTTPException(
+            status_code=500,
+            detail=f"Transcription failed: {str(e)}"
+        )
+
+
+@router.get("/models")
+async def get_available_models():
+    """Get list of available Whisper models."""
+    return JSONResponse(content={
+        "models": [
+            {"name": "tiny", "size": "~39 MB", "speed": "fastest", "accuracy": "lowest"},
+            {"name": "base", "size": "~74 MB", "speed": "fast", "accuracy": "good"},
+            {"name": "small", "size": "~244 MB", "speed": "medium", "accuracy": "better"},
+            {"name": "medium", "size": "~769 MB", "speed": "slow", "accuracy": "high"},
+            {"name": "large-v3", "size": "~1550 MB", "speed": "slowest", "accuracy": "highest"},
+        ],
+        "current_model": stt_service.model_size,
+        "note": "Models are downloaded automatically on first use"
+    })
+
+
+@router.post("/change-model")
+async def change_model(model_size: str = Form(...)):
+    """Change the active Whisper model."""
+    
+    valid_models = ["tiny", "base", "small", "medium", "large-v3"]
+    if model_size not in valid_models:
+        raise HTTPException(
+            status_code=400,
+            detail=f"Invalid model. Choose from: {valid_models}"
+        )
+    
+    try:
+        # Create new service instance with different model
+        global stt_service
+        stt_service = type(stt_service)(model_size=model_size)
+        
+        return JSONResponse(content={
+            "success": True,
+            "message": f"Model changed to {model_size}",
+            "note": "Model will be downloaded on next transcription if not cached"
+        })
+        
+    except Exception as e:
+        raise HTTPException(
+            status_code=500,
+            detail=f"Failed to change model: {str(e)}"
+        )

surfsense_backend/app/services/stt_service.py

@@ -0,0 +1,95 @@
+"""Local Speech-to-Text service using Faster-Whisper."""
+
+import os
+import tempfile
+from pathlib import Path
+from typing import Optional
+
+from faster_whisper import WhisperModel
+from app.config import config
+
+
+class STTService:
+    """Local Speech-to-Text service using Faster-Whisper."""
+    
+    def __init__(self, model_size: Optional[str] = None):
+        """Initialize STT service with specified model size.
+        
+        Args:
+            model_size: Whisper model size ("tiny", "base", "small", "medium", "large-v3")
+        """
+        self.model_size = model_size or config.LOCAL_STT_MODEL
+        self._model: Optional[WhisperModel] = None
+        
+    def _get_model(self) -> WhisperModel:
+        """Lazy load the Whisper model."""
+        if self._model is None:
+            # Use CPU with optimizations for better performance
+            self._model = WhisperModel(
+                self.model_size,
+                device="cpu",
+                compute_type="int8",  # Quantization for faster CPU inference
+                num_workers=1,  # Single worker for stability
+            )
+        return self._model
+    
+    def transcribe_file(self, audio_path: str, language: Optional[str] = None) -> dict:
+        """Transcribe audio file to text.
+        
+        Args:
+            audio_path: Path to audio file
+            language: Optional language code (e.g., "en", "es")
+            
+        Returns:
+            Dict with transcription text and metadata
+        """
+        model = self._get_model()
+        
+        # Transcribe with optimized settings
+        segments, info = model.transcribe(
+            audio_path,
+            language=language,
+            beam_size=1,  # Faster inference
+            best_of=1,    # Single pass
+            temperature=0,  # Deterministic output
+            vad_filter=True,  # Voice activity detection
+            vad_parameters=dict(min_silence_duration_ms=500),
+        )
+        
+        # Combine all segments
+        text = " ".join(segment.text.strip() for segment in segments)
+        
+        return {
+            "text": text,
+            "language": info.language,
+            "language_probability": info.language_probability,
+            "duration": info.duration,
+        }
+    
+    def transcribe_bytes(self, audio_bytes: bytes, filename: str = "audio.wav", 
+                        language: Optional[str] = None) -> dict:
+        """Transcribe audio from bytes.
+        
+        Args:
+            audio_bytes: Audio file bytes
+            filename: Original filename for format detection
+            language: Optional language code
+            
+        Returns:
+            Dict with transcription text and metadata
+        """
+        # Save bytes to temporary file
+        suffix = Path(filename).suffix or ".wav"
+        with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as tmp_file:
+            tmp_file.write(audio_bytes)
+            tmp_path = tmp_file.name
+        
+        try:
+            return self.transcribe_file(tmp_path, language)
+        finally:
+            # Clean up temp file
+            os.unlink(tmp_path)
+
+
+# Global STT service instance
+stt_service = STTService()

surfsense_backend/pyproject.toml

@@ -43,6 +43,7 @@ dependencies = [
     "youtube-transcript-api>=1.0.3",
     "litellm>=1.77.5",
     "langchain-litellm>=0.2.3",
+    "faster-whisper>=1.1.0",
 ]
 
 [dependency-groups]