apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-07-02 22:01:05 +02:00
Code Issues Projects Releases Packages Wiki Activity

chore: ran both frontend and backend linting

Browse source
This commit is contained in:
Anish Sarkar 2026-01-14 02:05:40 +05:30
parent 99bd2df463
commit 5bd6bd3d67
21 changed files with 861 additions and 739 deletions
Download patch file Download diff file Expand all files Collapse all files

1
surfsense_backend/alembic/versions/62_add_notifications_table.py
View file

@ -6,6 +6,7 @@ Revises: 61
Note: Electric SQL replication setup (REPLICA IDENTITY FULL and publication)
is handled in app/db.py setup_electric_replication() which runs on app startup.
"""
from collections.abc import Sequence
from alembic import op

17
surfsense_backend/app/db.py
View file

@ -711,15 +711,22 @@ class Notification(BaseModel, TimestampMixin):
__tablename__ = "notifications"
user_id = Column(
UUID(as_uuid=True), ForeignKey("user.id", ondelete="CASCADE"), nullable=False, index=True
UUID(as_uuid=True),
ForeignKey("user.id", ondelete="CASCADE"),
nullable=False,
index=True,
)
search_space_id = Column(
Integer, ForeignKey("searchspaces.id", ondelete="CASCADE"), nullable=True
)
type = Column(String(50), nullable=False) # 'connector_indexing', 'document_processing', etc.
type = Column(
String(50), nullable=False
) # 'connector_indexing', 'document_processing', etc.
title = Column(String(200), nullable=False)
message = Column(Text, nullable=False)
read = Column(Boolean, nullable=False, default=False, server_default=text("false"), index=True)
read = Column(
Boolean, nullable=False, default=False, server_default=text("false"), index=True
)
notification_metadata = Column("metadata", JSONB, nullable=True, default={})
user = relationship("User", back_populates="notifications")
@ -990,7 +997,9 @@ async def setup_electric_replication():
# Set REPLICA IDENTITY FULL (required by Electric SQL for replication)
# This logs full row data for UPDATE/DELETE operations in the WAL
await conn.execute(text("ALTER TABLE notifications REPLICA IDENTITY FULL;"))
await conn.execute(text("ALTER TABLE search_source_connectors REPLICA IDENTITY FULL;"))
await conn.execute(
text("ALTER TABLE search_source_connectors REPLICA IDENTITY FULL;")
)
await conn.execute(text("ALTER TABLE documents REPLICA IDENTITY FULL;"))
# Add tables to Electric SQL publication for replication

31
surfsense_backend/app/routes/search_source_connectors_routes.py
View file

@ -986,21 +986,25 @@ async def _run_indexing_with_notifications(
try:
# Get connector info for notification
connector_result = await session.execute(
select(SearchSourceConnector).where(SearchSourceConnector.id == connector_id)
select(SearchSourceConnector).where(
SearchSourceConnector.id == connector_id
)
)
connector = connector_result.scalar_one_or_none()
if connector:
# Create notification when indexing starts
notification = await NotificationService.connector_indexing.notify_indexing_started(
session=session,
user_id=UUID(user_id),
connector_id=connector_id,
connector_name=connector.name,
connector_type=connector.connector_type.value,
search_space_id=search_space_id,
start_date=start_date,
end_date=end_date,
notification = (
await NotificationService.connector_indexing.notify_indexing_started(
session=session,
user_id=UUID(user_id),
connector_id=connector_id,
connector_name=connector.name,
connector_type=connector.connector_type.value,
search_space_id=search_space_id,
start_date=start_date,
end_date=end_date,
)
)
# Update notification to fetching stage
@ -1640,6 +1644,7 @@ async def run_google_gmail_indexing(
start_date: Start date for indexing
end_date: End date for indexing
"""
# Create a wrapper function that calls index_google_gmail_messages with max_messages
async def gmail_indexing_wrapper(
session: AsyncSession,
@ -1701,7 +1706,9 @@ async def run_google_drive_indexing(
# Get connector info for notification
connector_result = await session.execute(
select(SearchSourceConnector).where(SearchSourceConnector.id == connector_id)
select(SearchSourceConnector).where(
SearchSourceConnector.id == connector_id
)
)
connector = connector_result.scalar_one_or_none()
@ -1813,7 +1820,7 @@ async def run_google_drive_indexing(
f"Critical error in run_google_drive_indexing for connector {connector_id}: {e}",
exc_info=True,
)
# Update notification on exception
if notification:
try:

43
surfsense_backend/app/services/notification_service.py
View file

@ -99,7 +99,9 @@ class BaseNotificationHandler:
flag_modified(notification, "notification_metadata")
await session.commit()
await session.refresh(notification)
logger.info(f"Updated notification {notification.id} for operation {operation_id}")
logger.info(
f"Updated notification {notification.id} for operation {operation_id}"
)
return notification
# Create new notification
@ -119,7 +121,9 @@ class BaseNotificationHandler:
session.add(notification)
await session.commit()
await session.refresh(notification)
logger.info(f"Created notification {notification.id} for operation {operation_id}")
logger.info(
f"Created notification {notification.id} for operation {operation_id}"
)
return notification
async def update_notification(
@ -153,9 +157,9 @@ class BaseNotificationHandler:
if status is not None:
notification.notification_metadata["status"] = status
if status in ("completed", "failed"):
notification.notification_metadata["completed_at"] = (
datetime.now(UTC).isoformat()
)
notification.notification_metadata["completed_at"] = datetime.now(
UTC
).isoformat()
# Mark JSONB column as modified so SQLAlchemy detects the change
flag_modified(notification, "notification_metadata")
@ -180,7 +184,10 @@ class ConnectorIndexingNotificationHandler(BaseNotificationHandler):
super().__init__("connector_indexing")
def _generate_operation_id(
self, connector_id: int, start_date: str | None = None, end_date: str | None = None
self,
connector_id: int,
start_date: str | None = None,
end_date: str | None = None,
) -> str:
"""
Generate a unique operation ID for a connector indexing operation.
@ -298,7 +305,7 @@ class ConnectorIndexingNotificationHandler(BaseNotificationHandler):
"processing": "Preparing for search",
"storing": "Almost done",
}
# Use stage-based message if stage provided, otherwise fallback
if stage or stage_message:
progress_msg = stage_message or stage_messages.get(stage, "Processing")
@ -341,7 +348,9 @@ class ConnectorIndexingNotificationHandler(BaseNotificationHandler):
Returns:
Updated notification
"""
connector_name = notification.notification_metadata.get("connector_name", "Connector")
connector_name = notification.notification_metadata.get(
"connector_name", "Connector"
)
if error_message:
title = f"Failed: {connector_name}"
@ -414,7 +423,7 @@ class ConnectorIndexingNotificationHandler(BaseNotificationHandler):
"indexed_count": 0,
"sync_stage": "connecting",
}
if folder_names:
metadata["folder_names"] = folder_names
if file_names:
@ -454,6 +463,7 @@ class DocumentProcessingNotificationHandler(BaseNotificationHandler):
timestamp = datetime.now(UTC).strftime("%Y%m%d_%H%M%S_%f")
# Create a short hash of filename to ensure uniqueness
import hashlib
filename_hash = hashlib.md5(filename.encode()).hexdigest()[:8]
return f"doc_{document_type}_{search_space_id}_{timestamp}_{filename_hash}"
@ -480,7 +490,9 @@ class DocumentProcessingNotificationHandler(BaseNotificationHandler):
Returns:
Notification: The created notification
"""
operation_id = self._generate_operation_id(document_type, document_name, search_space_id)
operation_id = self._generate_operation_id(
document_type, document_name, search_space_id
)
title = f"Processing: {document_name}"
message = "Waiting in queue"
@ -489,7 +501,7 @@ class DocumentProcessingNotificationHandler(BaseNotificationHandler):
"document_name": document_name,
"processing_stage": "queued",
}
if file_size is not None:
metadata["file_size"] = file_size
@ -531,7 +543,7 @@ class DocumentProcessingNotificationHandler(BaseNotificationHandler):
"embedding": "Preparing for search",
"storing": "Finalizing",
}
message = stage_message or stage_messages.get(stage, "Processing")
metadata_updates = {"processing_stage": stage}
@ -568,7 +580,9 @@ class DocumentProcessingNotificationHandler(BaseNotificationHandler):
Returns:
Updated notification
"""
document_name = notification.notification_metadata.get("document_name", "Document")
document_name = notification.notification_metadata.get(
"document_name", "Document"
)
if error_message:
title = f"Failed: {document_name}"
@ -583,7 +597,7 @@ class DocumentProcessingNotificationHandler(BaseNotificationHandler):
"processing_stage": "completed" if not error_message else "failed",
"error_message": error_message,
}
if document_id is not None:
metadata_updates["document_id"] = document_id
# Store chunks_count in metadata for debugging, but don't show to user
@ -645,4 +659,3 @@ class NotificationService:
await session.refresh(notification)
logger.info(f"Created notification {notification.id} for user {user_id}")
return notification

153
surfsense_backend/app/tasks/celery_tasks/document_tasks.py
View file

@ -92,12 +92,14 @@ async def _process_extension_document(
page_title += "..."
# Create notification for document processing
notification = await NotificationService.document_processing.notify_processing_started(
session=session,
user_id=UUID(user_id),
document_type="EXTENSION",
document_name=page_title,
search_space_id=search_space_id,
notification = (
await NotificationService.document_processing.notify_processing_started(
session=session,
user_id=UUID(user_id),
document_type="EXTENSION",
document_name=page_title,
search_space_id=search_space_id,
)
)
log_entry = await task_logger.log_task_start(
@ -115,7 +117,10 @@ async def _process_extension_document(
try:
# Update notification: parsing stage
await NotificationService.document_processing.notify_processing_progress(
session, notification, stage="parsing", stage_message="Reading page content"
session,
notification,
stage="parsing",
stage_message="Reading page content",
)
result = await add_extension_received_document(
@ -130,11 +135,13 @@ async def _process_extension_document(
)
# Update notification on success
await NotificationService.document_processing.notify_processing_completed(
session=session,
notification=notification,
document_id=result.id,
chunks_count=None,
await (
NotificationService.document_processing.notify_processing_completed(
session=session,
notification=notification,
document_id=result.id,
chunks_count=None,
)
)
else:
await task_logger.log_task_success(
@ -144,10 +151,12 @@ async def _process_extension_document(
)
# Update notification for duplicate
await NotificationService.document_processing.notify_processing_completed(
session=session,
notification=notification,
error_message="Page already saved (duplicate)",
await (
NotificationService.document_processing.notify_processing_completed(
session=session,
notification=notification,
error_message="Page already saved (duplicate)",
)
)
except Exception as e:
await task_logger.log_task_failure(
@ -198,12 +207,14 @@ async def _process_youtube_video(url: str, search_space_id: int, user_id: str):
video_name = url.split("v=")[-1][:11] if "v=" in url else url
# Create notification for document processing
notification = await NotificationService.document_processing.notify_processing_started(
session=session,
user_id=UUID(user_id),
document_type="YOUTUBE_VIDEO",
document_name=f"YouTube: {video_name}",
search_space_id=search_space_id,
notification = (
await NotificationService.document_processing.notify_processing_started(
session=session,
user_id=UUID(user_id),
document_type="YOUTUBE_VIDEO",
document_name=f"YouTube: {video_name}",
search_space_id=search_space_id,
)
)
log_entry = await task_logger.log_task_start(
@ -216,7 +227,10 @@ async def _process_youtube_video(url: str, search_space_id: int, user_id: str):
try:
# Update notification: parsing (fetching transcript)
await NotificationService.document_processing.notify_processing_progress(
session, notification, stage="parsing", stage_message="Fetching video transcript"
session,
notification,
stage="parsing",
stage_message="Fetching video transcript",
)
result = await add_youtube_video_document(
@ -235,11 +249,13 @@ async def _process_youtube_video(url: str, search_space_id: int, user_id: str):
)
# Update notification on success
await NotificationService.document_processing.notify_processing_completed(
session=session,
notification=notification,
document_id=result.id,
chunks_count=None,
await (
NotificationService.document_processing.notify_processing_completed(
session=session,
notification=notification,
document_id=result.id,
chunks_count=None,
)
)
else:
await task_logger.log_task_success(
@ -249,10 +265,12 @@ async def _process_youtube_video(url: str, search_space_id: int, user_id: str):
)
# Update notification for duplicate
await NotificationService.document_processing.notify_processing_completed(
session=session,
notification=notification,
error_message="Video already exists (duplicate)",
await (
NotificationService.document_processing.notify_processing_completed(
session=session,
notification=notification,
error_message="Video already exists (duplicate)",
)
)
except Exception as e:
await task_logger.log_task_failure(
@ -317,13 +335,15 @@ async def _process_file_upload(
file_size = None
# Create notification for document processing
notification = await NotificationService.document_processing.notify_processing_started(
session=session,
user_id=UUID(user_id),
document_type="FILE",
document_name=filename,
search_space_id=search_space_id,
file_size=file_size,
notification = (
await NotificationService.document_processing.notify_processing_started(
session=session,
user_id=UUID(user_id),
document_type="FILE",
document_name=filename,
search_space_id=search_space_id,
file_size=file_size,
)
)
log_entry = await task_logger.log_task_start(
@ -352,18 +372,22 @@ async def _process_file_upload(
# Update notification on success
if result:
await NotificationService.document_processing.notify_processing_completed(
session=session,
notification=notification,
document_id=result.id,
chunks_count=None,
await (
NotificationService.document_processing.notify_processing_completed(
session=session,
notification=notification,
document_id=result.id,
chunks_count=None,
)
)
else:
# Duplicate detected
await NotificationService.document_processing.notify_processing_completed(
session=session,
notification=notification,
error_message="Document already exists (duplicate)",
await (
NotificationService.document_processing.notify_processing_completed(
session=session,
notification=notification,
error_message="Document already exists (duplicate)",
)
)
except Exception as e:
@ -456,12 +480,14 @@ async def _process_circleback_meeting(
# Create notification if user_id is available
notification = None
if user_id:
notification = await NotificationService.document_processing.notify_processing_started(
session=session,
user_id=UUID(user_id),
document_type="CIRCLEBACK",
document_name=f"Meeting: {meeting_name[:40]}",
search_space_id=search_space_id,
notification = (
await NotificationService.document_processing.notify_processing_started(
session=session,
user_id=UUID(user_id),
document_type="CIRCLEBACK",
document_name=f"Meeting: {meeting_name[:40]}",
search_space_id=search_space_id,
)
)
log_entry = await task_logger.log_task_start(
@ -479,8 +505,13 @@ async def _process_circleback_meeting(
try:
# Update notification: parsing stage
if notification:
await NotificationService.document_processing.notify_processing_progress(
session, notification, stage="parsing", stage_message="Reading meeting notes"
await (
NotificationService.document_processing.notify_processing_progress(
session,
notification,
stage="parsing",
stage_message="Reading meeting notes",
)
)
result = await add_circleback_meeting_document(
@ -535,10 +566,12 @@ async def _process_circleback_meeting(
# Update notification on failure
if notification:
await NotificationService.document_processing.notify_processing_completed(
session=session,
notification=notification,
error_message=str(e)[:100],
await (
NotificationService.document_processing.notify_processing_completed(
session=session,
notification=notification,
error_message=str(e)[:100],
)
)
logger.error(f"Error processing Circleback meeting: {e!s}")

53
surfsense_backend/app/tasks/document_processors/file_processors.py
View file

@ -476,15 +476,21 @@ async def process_file_in_background(
log_entry: Log,
connector: dict
| None = None, # Optional: {"type": "GOOGLE_DRIVE_FILE", "metadata": {...}}
notification: Notification | None = None, # Optional notification for progress updates
notification: Notification
| None = None, # Optional notification for progress updates
) -> Document | None:
try:
# Check if the file is a markdown or text file
if filename.lower().endswith((".md", ".markdown", ".txt")):
# Update notification: parsing stage
if notification:
await NotificationService.document_processing.notify_processing_progress(
session, notification, stage="parsing", stage_message="Reading file"
await (
NotificationService.document_processing.notify_processing_progress(
session,
notification,
stage="parsing",
stage_message="Reading file",
)
)
await task_logger.log_task_progress(
@ -508,8 +514,10 @@ async def process_file_in_background(
# Update notification: chunking stage
if notification:
await NotificationService.document_processing.notify_processing_progress(
session, notification, stage="chunking"
await (
NotificationService.document_processing.notify_processing_progress(
session, notification, stage="chunking"
)
)
await task_logger.log_task_progress(
@ -554,8 +562,13 @@ async def process_file_in_background(
):
# Update notification: parsing stage (transcription)
if notification:
await NotificationService.document_processing.notify_processing_progress(
session, notification, stage="parsing", stage_message="Transcribing audio"
await (
NotificationService.document_processing.notify_processing_progress(
session,
notification,
stage="parsing",
stage_message="Transcribing audio",
)
)
await task_logger.log_task_progress(
@ -643,8 +656,10 @@ async def process_file_in_background(
# Update notification: chunking stage
if notification:
await NotificationService.document_processing.notify_processing_progress(
session, notification, stage="chunking"
await (
NotificationService.document_processing.notify_processing_progress(
session, notification, stage="chunking"
)
)
# Clean up the temp file
@ -749,7 +764,10 @@ async def process_file_in_background(
# Update notification: parsing stage
if notification:
await NotificationService.document_processing.notify_processing_progress(
session, notification, stage="parsing", stage_message="Extracting content"
session,
notification,
stage="parsing",
stage_message="Extracting content",
)
await task_logger.log_task_progress(
@ -859,7 +877,10 @@ async def process_file_in_background(
# Update notification: parsing stage
if notification:
await NotificationService.document_processing.notify_processing_progress(
session, notification, stage="parsing", stage_message="Extracting content"
session,
notification,
stage="parsing",
stage_message="Extracting content",
)
await task_logger.log_task_progress(
@ -904,7 +925,10 @@ async def process_file_in_background(
# Update notification: chunking stage
if notification:
await NotificationService.document_processing.notify_processing_progress(
session, notification, stage="chunking", chunks_count=len(markdown_documents)
session,
notification,
stage="chunking",
chunks_count=len(markdown_documents),
)
await task_logger.log_task_progress(
@ -1018,7 +1042,10 @@ async def process_file_in_background(
# Update notification: parsing stage
if notification:
await NotificationService.document_processing.notify_processing_progress(
session, notification, stage="parsing", stage_message="Extracting content"
session,
notification,
stage="parsing",
stage_message="Extracting content",
)
await task_logger.log_task_progress(