From 70b547c9c9273012a541eb77ed8551d27c0ba98d Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Wed, 15 Oct 2025 14:31:38 -0700 Subject: [PATCH 1/2] chore: updated docs & refactored sst_service.py --- .../app/routes/documents_routes.py | 31 ++++++++----- surfsense_backend/app/services/stt_service.py | 44 ++++++++++--------- .../documents/(manage)/page.tsx | 2 +- .../content/docs/docker-installation.mdx | 6 +-- .../content/docs/manual-installation.mdx | 11 +++-- 5 files changed, 53 insertions(+), 41 deletions(-) diff --git a/surfsense_backend/app/routes/documents_routes.py b/surfsense_backend/app/routes/documents_routes.py index a3c69fe2a..08a352e75 100644 --- a/surfsense_backend/app/routes/documents_routes.py +++ b/surfsense_backend/app/routes/documents_routes.py @@ -785,20 +785,25 @@ async def process_file_in_background( ) # Determine STT service type - stt_service_type = "local" if app_config.STT_SERVICE and app_config.STT_SERVICE.startswith("local/") else "external" - + stt_service_type = ( + "local" + if app_config.STT_SERVICE + and app_config.STT_SERVICE.startswith("local/") + else "external" + ) + # Check if using local STT service if stt_service_type == "local": # Use local Faster-Whisper for transcription from app.services.stt_service import stt_service - + try: result = stt_service.transcribe_file(file_path) transcribed_text = result.get("text", "") - + if not transcribed_text: raise ValueError("Transcription returned empty text") - + # Add metadata about the transcription transcribed_text = ( f"# Transcription of {filename}\n\n{transcribed_text}" @@ -806,9 +811,9 @@ async def process_file_in_background( except Exception as e: raise HTTPException( status_code=422, - detail=f"Failed to transcribe audio file {filename}: {str(e)}" + detail=f"Failed to transcribe audio file {filename}: {e!s}", ) from e - + await task_logger.log_task_progress( log_entry, f"Local STT transcription completed: {filename}", @@ -828,13 +833,17 @@ async def process_file_in_background( "api_key": app_config.STT_SERVICE_API_KEY, } if app_config.STT_SERVICE_API_BASE: - transcription_kwargs["api_base"] = app_config.STT_SERVICE_API_BASE - - transcription_response = await atranscription(**transcription_kwargs) + transcription_kwargs["api_base"] = ( + app_config.STT_SERVICE_API_BASE + ) + + transcription_response = await atranscription( + **transcription_kwargs + ) # Extract the transcribed text transcribed_text = transcription_response.get("text", "") - + if not transcribed_text: raise ValueError("Transcription returned empty text") diff --git a/surfsense_backend/app/services/stt_service.py b/surfsense_backend/app/services/stt_service.py index 273fef05b..ea38480e8 100644 --- a/surfsense_backend/app/services/stt_service.py +++ b/surfsense_backend/app/services/stt_service.py @@ -3,15 +3,15 @@ import os import tempfile from pathlib import Path -from typing import Optional from faster_whisper import WhisperModel + from app.config import config class STTService: """Local Speech-to-Text service using Faster-Whisper.""" - + def __init__(self): """Initialize STT service with model from STT_SERVICE config.""" # Parse model from STT_SERVICE (e.g., "local/base" or "local/tiny") @@ -20,8 +20,8 @@ class STTService: self.model_size = stt_service.split("/", 1)[1] else: self.model_size = "base" # fallback - self._model: Optional[WhisperModel] = None - + self._model: WhisperModel | None = None + def _get_model(self) -> WhisperModel: """Lazy load the Whisper model.""" if self._model is None: @@ -33,49 +33,53 @@ class STTService: num_workers=1, # Single worker for stability ) return self._model - - def transcribe_file(self, audio_path: str, language: Optional[str] = None) -> dict: + + def transcribe_file(self, audio_path: str, language: str | None = None) -> dict: """Transcribe audio file to text. - + Args: audio_path: Path to audio file language: Optional language code (e.g., "en", "es") - + Returns: Dict with transcription text and metadata """ model = self._get_model() - + # Transcribe with optimized settings segments, info = model.transcribe( audio_path, language=language, beam_size=1, # Faster inference - best_of=1, # Single pass + best_of=1, # Single pass temperature=0, # Deterministic output vad_filter=True, # Voice activity detection - vad_parameters=dict(min_silence_duration_ms=500), + vad_parameters={"min_silence_duration_ms": 500}, ) - + # Combine all segments text = " ".join(segment.text.strip() for segment in segments) - + return { "text": text, "language": info.language, "language_probability": info.language_probability, "duration": info.duration, } - - def transcribe_bytes(self, audio_bytes: bytes, filename: str = "audio.wav", - language: Optional[str] = None) -> dict: + + def transcribe_bytes( + self, + audio_bytes: bytes, + filename: str = "audio.wav", + language: str | None = None, + ) -> dict: """Transcribe audio from bytes. - + Args: audio_bytes: Audio file bytes filename: Original filename for format detection language: Optional language code - + Returns: Dict with transcription text and metadata """ @@ -84,7 +88,7 @@ class STTService: with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as tmp_file: tmp_file.write(audio_bytes) tmp_path = tmp_file.name - + try: return self.transcribe_file(tmp_path, language) finally: @@ -93,4 +97,4 @@ class STTService: # Global STT service instance -stt_service = STTService() \ No newline at end of file +stt_service = STTService() diff --git a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx index b63a20367..bf3ca67a0 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx @@ -36,7 +36,7 @@ export default function DocumentsTable() { created_at: true, }); const [pageIndex, setPageIndex] = useState(0); - const [pageSize, setPageSize] = useState(10); + const [pageSize, setPageSize] = useState(50); const [sortKey, setSortKey] = useState("title"); const [sortDesc, setSortDesc] = useState(false); const [selectedIds, setSelectedIds] = useState>(new Set()); diff --git a/surfsense_web/content/docs/docker-installation.mdx b/surfsense_web/content/docs/docker-installation.mdx index ae0ebadfe..4a9a11626 100644 --- a/surfsense_web/content/docs/docker-installation.mdx +++ b/surfsense_web/content/docs/docker-installation.mdx @@ -85,10 +85,10 @@ Before you begin, ensure you have: | RERANKERS_MODEL_NAME | Name of the reranker model (e.g., `ms-marco-MiniLM-L-12-v2`) | | RERANKERS_MODEL_TYPE | Type of reranker model (e.g., `flashrank`) | | TTS_SERVICE | Text-to-Speech API provider for Podcasts (e.g., `local/kokoro`, `openai/tts-1`). See [supported providers](https://docs.litellm.ai/docs/text_to_speech#supported-providers) | -| TTS_SERVICE_API_KEY | API key for the Text-to-Speech service | +| TTS_SERVICE_API_KEY | (Optional if local) API key for the Text-to-Speech service | | TTS_SERVICE_API_BASE | (Optional) Custom API base URL for the Text-to-Speech service | -| STT_SERVICE | Speech-to-Text API provider for Podcasts (e.g., `openai/whisper-1`). See [supported providers](https://docs.litellm.ai/docs/audio_transcription#supported-providers) | -| STT_SERVICE_API_KEY | API key for the Speech-to-Text service | +| STT_SERVICE | Speech-to-Text API provider for Audio Files (e.g., `local/base`, `openai/whisper-1`). See [supported providers](https://docs.litellm.ai/docs/audio_transcription#supported-providers) | +| STT_SERVICE_API_KEY | (Optional if local) API key for the Speech-to-Text service | | STT_SERVICE_API_BASE | (Optional) Custom API base URL for the Speech-to-Text service | | FIRECRAWL_API_KEY | API key for Firecrawl service for web crawling | | ETL_SERVICE | Document parsing service: `UNSTRUCTURED` (supports 34+ formats), `LLAMACLOUD` (supports 50+ formats including legacy document types), or `DOCLING` (local processing, supports PDF, Office docs, images, HTML, CSV) | diff --git a/surfsense_web/content/docs/manual-installation.mdx b/surfsense_web/content/docs/manual-installation.mdx index d1a5a86dc..5ce285348 100644 --- a/surfsense_web/content/docs/manual-installation.mdx +++ b/surfsense_web/content/docs/manual-installation.mdx @@ -62,12 +62,11 @@ Edit the `.env` file and set the following variables: | RERANKERS_MODEL_NAME | Name of the reranker model (e.g., `ms-marco-MiniLM-L-12-v2`) | | RERANKERS_MODEL_TYPE | Type of reranker model (e.g., `flashrank`) | | TTS_SERVICE | Text-to-Speech API provider for Podcasts (e.g., `local/kokoro`, `openai/tts-1`). See [supported providers](https://docs.litellm.ai/docs/text_to_speech#supported-providers) | -| TTS_SERVICE_API_KEY | API key for the Text-to-Speech service | -| TTS_SERVICE_API_BASE | (Optional) Custom API base URL for the Text-to-Speech service | -| STT_SERVICE | Speech-to-Text API provider for Podcasts (e.g., `openai/whisper-1`). See [supported providers](https://docs.litellm.ai/docs/audio_transcription#supported-providers) | -| STT_SERVICE_API_KEY | API key for the Speech-to-Text service | -| STT_SERVICE_API_BASE | (Optional) Custom API base URL for the Speech-to-Text service | -| FIRECRAWL_API_KEY | API key for Firecrawl service for web crawling | +| TTS_SERVICE_API_KEY | (Optional if local) API key for the Text-to-Speech service | +| TTS_SERVICE_API_BASE | (Optional) Custom API base URL for the Text-to-Speech service | +| STT_SERVICE | Speech-to-Text API provider for Audio Files (e.g., `local/base`, `openai/whisper-1`). See [supported providers](https://docs.litellm.ai/docs/audio_transcription#supported-providers) | +| STT_SERVICE_API_KEY | (Optional if local) API key for the Speech-to-Text service | +| STT_SERVICE_API_BASE | (Optional) Custom API base URL for the Speech-to-Text service | | ETL_SERVICE | Document parsing service: `UNSTRUCTURED` (supports 34+ formats), `LLAMACLOUD` (supports 50+ formats including legacy document types), or `DOCLING` (local processing, supports PDF, Office docs, images, HTML, CSV) | | UNSTRUCTURED_API_KEY | API key for Unstructured.io service for document parsing (required if ETL_SERVICE=UNSTRUCTURED) | | LLAMA_CLOUD_API_KEY | API key for LlamaCloud service for document parsing (required if ETL_SERVICE=LLAMACLOUD) | From 3022b255412abb16a0baa79714d232ce97e40741 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Wed, 15 Oct 2025 14:38:26 -0700 Subject: [PATCH 2/2] chore: biome checks --- .../components/chat/ChatInputGroup.tsx | 8 +- .../components/inference-params-editor.tsx | 235 ++++++++++-------- .../components/onboard/add-provider-step.tsx | 3 +- surfsense_web/drizzle.config.ts | 16 +- .../hooks/use-connector-edit-page.ts | 12 +- 5 files changed, 142 insertions(+), 132 deletions(-) diff --git a/surfsense_web/components/chat/ChatInputGroup.tsx b/surfsense_web/components/chat/ChatInputGroup.tsx index c3877c108..866ac7b0f 100644 --- a/surfsense_web/components/chat/ChatInputGroup.tsx +++ b/surfsense_web/components/chat/ChatInputGroup.tsx @@ -334,9 +334,13 @@ ResearchModeSelector.displayName = "ResearchModeSelector"; const LLMSelector = React.memo(() => { const { search_space_id } = useParams(); const searchSpaceId = Number(search_space_id); - + const { llmConfigs, loading: llmLoading, error } = useLLMConfigs(searchSpaceId); - const { preferences, updatePreferences, loading: preferencesLoading } = useLLMPreferences(searchSpaceId); + const { + preferences, + updatePreferences, + loading: preferencesLoading, + } = useLLMPreferences(searchSpaceId); const isLoading = llmLoading || preferencesLoading; diff --git a/surfsense_web/components/inference-params-editor.tsx b/surfsense_web/components/inference-params-editor.tsx index df198cfa0..9ef2ec27a 100644 --- a/surfsense_web/components/inference-params-editor.tsx +++ b/surfsense_web/components/inference-params-editor.tsx @@ -1,138 +1,153 @@ "use client"; -import { useState } from "react"; import { Plus, Trash2 } from "lucide-react"; +import { useState } from "react"; import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; import { - Select, - SelectContent, - SelectItem, - SelectTrigger, - SelectValue, + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, } from "@/components/ui/select"; interface InferenceParamsEditorProps { - params: Record; - setParams: (newParams: Record) => void; + params: Record; + setParams: (newParams: Record) => void; } const PARAM_KEYS = ["temperature", "max_tokens", "top_k", "top_p"] as const; -export default function InferenceParamsEditor({ - params, - setParams, -}: InferenceParamsEditorProps) { - const [selectedKey, setSelectedKey] = useState(""); - const [value, setValue] = useState(""); +export default function InferenceParamsEditor({ params, setParams }: InferenceParamsEditorProps) { + const [selectedKey, setSelectedKey] = useState(""); + const [value, setValue] = useState(""); - const handleAdd = () => { - if (!selectedKey || value === "") return; + const handleAdd = () => { + if (!selectedKey || value === "") return; - if (params[selectedKey]) { - alert(`${selectedKey} already exists`); - return; - } + if (params[selectedKey]) { + alert(`${selectedKey} already exists`); + return; + } - const numericValue = Number(value); + const numericValue = Number(value); - if ((selectedKey === "temperature" || selectedKey === "top_p") && (isNaN(numericValue) || numericValue < 0 || numericValue > 1)) { - alert("Value must be a number between 0 and 1"); - return; - } + if ( + (selectedKey === "temperature" || selectedKey === "top_p") && + (isNaN(numericValue) || numericValue < 0 || numericValue > 1) + ) { + alert("Value must be a number between 0 and 1"); + return; + } - if ((selectedKey === "max_tokens" || selectedKey === "top_k") && (!Number.isInteger(numericValue) || numericValue < 0)) { - alert("Value must be a non-negative integer"); - return; - } + if ( + (selectedKey === "max_tokens" || selectedKey === "top_k") && + (!Number.isInteger(numericValue) || numericValue < 0) + ) { + alert("Value must be a non-negative integer"); + return; + } - setParams({ - ...params, - [selectedKey]: isNaN(numericValue) ? value : numericValue, - }); + setParams({ + ...params, + [selectedKey]: isNaN(numericValue) ? value : numericValue, + }); - setSelectedKey(""); - setValue(""); - }; + setSelectedKey(""); + setValue(""); + }; - const handleDelete = (key: string) => { - const newParams = { ...params }; - delete newParams[key]; - setParams(newParams); - }; + const handleDelete = (key: string) => { + const newParams = { ...params }; + delete newParams[key]; + setParams(newParams); + }; - return ( -
-
-
- - -
+ return ( +
+
+
+ + +
-
- - setValue(e.target.value)} - className="w-full" - /> -
+
+ + setValue(e.target.value)} + className="w-full" + /> +
- -
+ +
-
+
- {Object.keys(params).length > 0 && ( -
- - - - - - - - - - {Object.entries(params).map(([key, val]) => ( - - - - - - ))} - -
KeyValueActions
{key}{val.toString()} - -
-
- )} -
- ); + {Object.keys(params).length > 0 && ( +
+ + + + + + + + + + {Object.entries(params).map(([key, val]) => ( + + + + + + ))} + +
+ Key + + Value + + Actions +
{key}{val.toString()} + +
+
+ )} +
+ ); } diff --git a/surfsense_web/components/onboard/add-provider-step.tsx b/surfsense_web/components/onboard/add-provider-step.tsx index 6517fe0a1..0086674d2 100644 --- a/surfsense_web/components/onboard/add-provider-step.tsx +++ b/surfsense_web/components/onboard/add-provider-step.tsx @@ -17,8 +17,8 @@ import { SelectTrigger, SelectValue, } from "@/components/ui/select"; -import { LLM_PROVIDERS } from "@/contracts/enums/llm-providers"; import { LANGUAGES } from "@/contracts/enums/languages"; +import { LLM_PROVIDERS } from "@/contracts/enums/llm-providers"; import { type CreateLLMConfig, useLLMConfigs } from "@/hooks/use-llm-configs"; import InferenceParamsEditor from "../inference-params-editor"; @@ -223,7 +223,6 @@ export function AddProviderStep({ - {formData.provider === "CUSTOM" && ( diff --git a/surfsense_web/drizzle.config.ts b/surfsense_web/drizzle.config.ts index fbd033a9e..bfc33ec9b 100644 --- a/surfsense_web/drizzle.config.ts +++ b/surfsense_web/drizzle.config.ts @@ -1,11 +1,11 @@ -import 'dotenv/config'; -import { defineConfig } from 'drizzle-kit'; +import "dotenv/config"; +import { defineConfig } from "drizzle-kit"; export default defineConfig({ - out: './drizzle', - schema: './app/db/schema.ts', - dialect: 'postgresql', - dbCredentials: { - url: process.env.DATABASE_URL!, - }, + out: "./drizzle", + schema: "./app/db/schema.ts", + dialect: "postgresql", + dbCredentials: { + url: process.env.DATABASE_URL!, + }, }); diff --git a/surfsense_web/hooks/use-connector-edit-page.ts b/surfsense_web/hooks/use-connector-edit-page.ts index ea6a3fe10..850f7e3e5 100644 --- a/surfsense_web/hooks/use-connector-edit-page.ts +++ b/surfsense_web/hooks/use-connector-edit-page.ts @@ -337,12 +337,7 @@ export function useConnectorEditPage(connectorId: number, searchSpaceId: string) const originalSafesearch = originalConfig.SEARXNG_SAFESEARCH; if (safesearchRaw) { const parsed = Number(safesearchRaw); - if ( - Number.isNaN(parsed) || - !Number.isInteger(parsed) || - parsed < 0 || - parsed > 2 - ) { + if (Number.isNaN(parsed) || !Number.isInteger(parsed) || parsed < 0 || parsed > 2) { toast.error("SearxNG SafeSearch must be 0, 1, or 2."); setIsSaving(false); return; @@ -521,10 +516,7 @@ export function useConnectorEditPage(connectorId: number, searchSpaceId: string) "SEARXNG_CATEGORIES", normalizeListInput(newlySavedConfig.SEARXNG_CATEGORIES).join(", ") ); - editForm.setValue( - "SEARXNG_LANGUAGE", - newlySavedConfig.SEARXNG_LANGUAGE || "" - ); + editForm.setValue("SEARXNG_LANGUAGE", newlySavedConfig.SEARXNG_LANGUAGE || ""); editForm.setValue( "SEARXNG_SAFESEARCH", newlySavedConfig.SEARXNG_SAFESEARCH === null ||