From 215a401475e689191e96dc89cfba71fc2175297d Mon Sep 17 00:00:00 2001 From: Aditya Vaish Date: Thu, 16 Oct 2025 10:45:46 +0530 Subject: [PATCH] print statement replaced with logger --- .../app/connectors/airtable_connector.py | 15 +++--- surfsense_backend/app/routes/chats_routes.py | 3 -- .../app/routes/documents_routes.py | 18 +++---- surfsense_backend/app/tasks/podcast_tasks.py | 53 ++++++++++++++++++- .../app/utils/document_converters.py | 24 ++++++--- 5 files changed, 84 insertions(+), 29 deletions(-) diff --git a/surfsense_backend/app/connectors/airtable_connector.py b/surfsense_backend/app/connectors/airtable_connector.py index 840b2276c..63ce361f5 100644 --- a/surfsense_backend/app/connectors/airtable_connector.py +++ b/surfsense_backend/app/connectors/airtable_connector.py @@ -305,19 +305,18 @@ class AirtableConnector: ) # Create Airtable formula for date filtering - # filter_formula = ( - # f"AND(" - # f"IS_AFTER({{date_field}}, '{start_date}'), " - # f"IS_BEFORE({{date_field}}, '{end_date}')" - # f")" - # ).replace("{date_field}", date_field) - # TODO: Investigate how to properly use filter formula + filter_formula = ( + f"AND(" + f"IS_AFTER({{{date_field}}}, '{start_date}'), " + f"IS_BEFORE({{{date_field}}}, '{end_date}')" + f")" + ) return self.get_all_records( base_id=base_id, table_id=table_id, max_records=max_records, - # filter_by_formula=filter_formula, + filter_by_formula=filter_formula, ) except Exception as e: diff --git a/surfsense_backend/app/routes/chats_routes.py b/surfsense_backend/app/routes/chats_routes.py index e003dc260..cc37799bf 100644 --- a/surfsense_backend/app/routes/chats_routes.py +++ b/surfsense_backend/app/routes/chats_routes.py @@ -54,8 +54,6 @@ async def handle_chat_data( request_data.get("document_ids_to_add_in_context") ) search_mode_str = validate_search_mode(request_data.get("search_mode")) - # print("RESQUEST DATA:", request_data) - # print("SELECTED CONNECTORS:", selected_connectors) # Check if the search space belongs to the current user try: @@ -76,7 +74,6 @@ async def handle_chat_data( ) ) user_preference = language_result.scalars().first() - # print("UserSearchSpacePreference:", user_preference) language = None if ( diff --git a/surfsense_backend/app/routes/documents_routes.py b/surfsense_backend/app/routes/documents_routes.py index 08a352e75..dd133faa3 100644 --- a/surfsense_backend/app/routes/documents_routes.py +++ b/surfsense_backend/app/routes/documents_routes.py @@ -1,5 +1,6 @@ # Force asyncio to use standard event loop before unstructured imports import asyncio +import logging from fastapi import APIRouter, BackgroundTasks, Depends, Form, HTTPException, UploadFile from litellm import atranscription @@ -40,7 +41,7 @@ from app.utils.check_ownership import check_ownership try: asyncio.set_event_loop_policy(asyncio.DefaultEventLoopPolicy()) except RuntimeError as e: - print("Error setting event loop policy", e) + logging.warning(f"Error setting event loop policy: {e}") pass import os @@ -48,6 +49,7 @@ import os os.environ["UNSTRUCTURED_HAS_PATCHED_LOOP"] = "1" +logger = logging.getLogger(__name__) router = APIRouter() @@ -740,7 +742,7 @@ async def process_file_in_background( try: os.unlink(file_path) except Exception as e: - print("Error deleting temp file", e) + logger.warning(f"Error deleting temp file {file_path}: {e}") pass await task_logger.log_task_progress( @@ -865,7 +867,7 @@ async def process_file_in_background( try: os.unlink(file_path) except Exception as e: - print("Error deleting temp file", e) + logger.warning(f"Error deleting temp file {file_path}: {e}") pass # Process transcription as markdown document @@ -931,7 +933,7 @@ async def process_file_in_background( try: os.unlink(file_path) except Exception as e: - print("Error deleting temp file", e) + logger.warning(f"Error deleting temp file {file_path}: {e}") pass # Pass the documents to the existing background task @@ -993,7 +995,7 @@ async def process_file_in_background( try: os.unlink(file_path) except Exception as e: - print("Error deleting temp file", e) + logger.warning(f"Error deleting temp file {file_path}: {e}") pass # Get markdown documents from the result @@ -1071,7 +1073,7 @@ async def process_file_in_background( try: os.unlink(file_path) except Exception as e: - print("Error deleting temp file", e) + logger.warning(f"Error deleting temp file {file_path}: {e}") pass await task_logger.log_task_progress( @@ -1121,7 +1123,5 @@ async def process_file_in_background( str(e), {"error_type": type(e).__name__, "filename": filename}, ) - import logging - - logging.error(f"Error processing file in background: {e!s}") + logger.error(f"Error processing file in background: {e!s}") raise # Re-raise so the wrapper can also handle it diff --git a/surfsense_backend/app/tasks/podcast_tasks.py b/surfsense_backend/app/tasks/podcast_tasks.py index e5f828ef2..b6da3d0df 100644 --- a/surfsense_backend/app/tasks/podcast_tasks.py +++ b/surfsense_backend/app/tasks/podcast_tasks.py @@ -11,8 +11,57 @@ from app.services.task_logging_service import TaskLoggingService async def generate_document_podcast( session: AsyncSession, document_id: int, search_space_id: int, user_id: int ): - # TODO: Need to fetch the document chunks, then concatenate them and pass them to the podcast generation model - pass + """ + Generate a podcast from a document by fetching its chunks and concatenating them. + + Args: + session: Database session + document_id: ID of the document to convert to podcast + search_space_id: Search space ID + user_id: User ID + + Returns: + Podcast object or raises exception + """ + task_logger = TaskLoggingService(session, search_space_id) + + # Log task start + log_entry = await task_logger.log_task_start( + task_name="generate_document_podcast", + source="podcast_task", + message=f"Starting podcast generation for document {document_id}", + metadata={ + "document_id": document_id, + "search_space_id": search_space_id, + "user_id": str(user_id), + }, + ) + + try: + # TODO: Implement document chunk fetching and podcast generation + # This would involve: + # 1. Fetching the document and its chunks + # 2. Concatenating the chunks into readable content + # 3. Passing the content to the podcast generation model + # 4. Creating and saving the podcast + + await task_logger.log_task_failure( + log_entry, + "Document podcast generation not yet implemented", + "NotImplementedError", + {"error_type": "NotImplementedError"}, + ) + raise NotImplementedError("Document podcast generation is not yet implemented") + + except Exception as e: + await session.rollback() + await task_logger.log_task_failure( + log_entry, + f"Error during document podcast generation for document {document_id}", + str(e), + {"error_type": type(e).__name__}, + ) + raise async def generate_chat_podcast( diff --git a/surfsense_backend/app/utils/document_converters.py b/surfsense_backend/app/utils/document_converters.py index 9883a74ed..564121df1 100644 --- a/surfsense_backend/app/utils/document_converters.py +++ b/surfsense_backend/app/utils/document_converters.py @@ -1,7 +1,10 @@ import hashlib +import logging from litellm import get_model_info, token_counter +logger = logging.getLogger(__name__) + from app.config import config from app.db import Chunk, DocumentType from app.prompts import SUMMARY_PROMPT_TEMPLATE @@ -14,8 +17,8 @@ def get_model_context_window(model_name: str) -> int: context_window = model_info.get("max_input_tokens", 4096) # Default fallback return context_window except Exception as e: - print( - f"Warning: Could not get model info for {model_name}, using default 4096 tokens. Error: {e}" + logger.warning( + f"Could not get model info for {model_name}, using default 4096 tokens. Error: {e}" ) return 4096 # Conservative fallback @@ -41,9 +44,16 @@ def optimize_content_for_context_window( context_window = get_model_context_window(model_name) # Reserve tokens for: system prompt, metadata, template overhead, and output - # Conservative estimate: 2000 tokens for prompt + metadata + output buffer - # TODO: Calculate Summary System Prompt Token Count Here - reserved_tokens = 2000 + # Calculate actual system prompt token count + try: + system_prompt_tokens = token_counter( + messages=[{"role": "system", "content": SUMMARY_PROMPT_TEMPLATE}], + model=model_name + ) + reserved_tokens = system_prompt_tokens + 1000 # 1000 tokens for output buffer and overhead + except Exception as e: + logger.warning(f"Could not calculate system prompt tokens, using default: {e}") + reserved_tokens = 2000 # Conservative fallback # Add metadata token cost if present if document_metadata: @@ -58,7 +68,7 @@ def optimize_content_for_context_window( available_tokens = context_window - reserved_tokens if available_tokens <= 100: # Minimum viable content - print(f"Warning: Very limited tokens available for content: {available_tokens}") + logger.warning(f"Very limited tokens available for content: {available_tokens}") return content[:500] # Fallback to first 500 chars # Binary search to find optimal content length @@ -86,7 +96,7 @@ def optimize_content_for_context_window( ) if optimal_length < len(content): - print( + logger.info( f"Content optimized: {len(content)} -> {optimal_length} chars " f"to fit in {available_tokens} available tokens" )