mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-05-01 03:46:25 +02:00
feat: implement background processing for binary attachments in Obsidian plugin
- Added a new Celery task for indexing non-markdown attachments. - Enhanced the Obsidian plugin schema to support binary attachments. - Updated routes to enqueue binary attachments for background processing. - Improved metadata handling for binary attachments during indexing. - Added tests for binary attachment processing and validation.
This commit is contained in:
parent
5047527b47
commit
6ac5256431
11 changed files with 519 additions and 68 deletions
59
surfsense_backend/app/tasks/celery_tasks/obsidian_tasks.py
Normal file
59
surfsense_backend/app/tasks/celery_tasks/obsidian_tasks.py
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
"""Celery tasks for Obsidian plugin background processing."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from app.celery_app import celery_app
|
||||
from app.db import SearchSourceConnector
|
||||
from app.schemas.obsidian_plugin import NotePayload
|
||||
from app.services.obsidian_plugin_indexer import upsert_note
|
||||
from app.tasks.celery_tasks import get_celery_session_maker
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@celery_app.task(name="index_obsidian_attachment", bind=True)
|
||||
def index_obsidian_attachment_task(
|
||||
self,
|
||||
connector_id: int,
|
||||
payload_data: dict,
|
||||
user_id: str,
|
||||
) -> None:
|
||||
"""Process one Obsidian non-markdown attachment asynchronously."""
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
try:
|
||||
loop.run_until_complete(
|
||||
_index_obsidian_attachment(
|
||||
connector_id=connector_id,
|
||||
payload_data=payload_data,
|
||||
user_id=user_id,
|
||||
)
|
||||
)
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
|
||||
async def _index_obsidian_attachment(
|
||||
*,
|
||||
connector_id: int,
|
||||
payload_data: dict,
|
||||
user_id: str,
|
||||
) -> None:
|
||||
async with get_celery_session_maker()() as session:
|
||||
connector = await session.get(SearchSourceConnector, connector_id)
|
||||
if connector is None:
|
||||
logger.warning(
|
||||
"obsidian attachment task skipped: connector %s not found", connector_id
|
||||
)
|
||||
return
|
||||
|
||||
payload = NotePayload.model_validate(payload_data)
|
||||
await upsert_note(
|
||||
session,
|
||||
connector=connector,
|
||||
payload=payload,
|
||||
user_id=user_id,
|
||||
)
|
||||
Loading…
Add table
Add a link
Reference in a new issue