apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-04-25 16:56:22 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: implement Redis heartbeat mechanism for document processing tasks and enhance stale notification cleanup

Browse source
This commit is contained in:
Anish Sarkar 2026-02-06 18:09:05 +05:30
parent 652eb6ece8
commit 72205ce11b
3 changed files with 350 additions and 30 deletions
Download patch file Download diff file Expand all files Collapse all files

18
surfsense_backend/app/tasks/document_processors/youtube_processor.py
View file

@ -61,7 +61,11 @@ def get_youtube_video_id(url: str) -> str | None:
async def add_youtube_video_document(
session: AsyncSession, url: str, search_space_id: int, user_id: str
session: AsyncSession,
url: str,
search_space_id: int,
user_id: str,
notification=None,
) -> Document:
"""
Process a YouTube video URL, extract transcripts, and store as a document.
@ -75,6 +79,9 @@ async def add_youtube_video_document(
url: YouTube video URL (supports standard, shortened, and embed formats)
search_space_id: ID of the search space to add the document to
user_id: ID of the user
notification: Optional notification object — if provided, the document_id
is stored in its metadata right after document creation so the stale
cleanup task can identify stuck documents.
Returns:
Document: The created document object
@ -182,6 +189,15 @@ async def add_youtube_video_document(
await session.commit() # Document visible in UI now with pending status!
is_new_document = True
# Store document_id in notification metadata so stale cleanup task
# can identify this document if the worker crashes.
if notification and notification.notification_metadata is not None:
from sqlalchemy.orm.attributes import flag_modified
notification.notification_metadata["document_id"] = document.id
flag_modified(notification, "notification_metadata")
await session.commit()
logging.info(f"Created pending document for YouTube video {video_id}")
# =======================================================================