mirror of
https://github.com/MODSetter/SurfSense.git
synced 2026-04-26 17:26:23 +02:00
656e061f84
Some checks are pending
Build and Push Docker Images / tag_release (push) Waiting to run
Build and Push Docker Images / build (./surfsense_backend, ./surfsense_backend/Dockerfile, backend, surfsense-backend, ubuntu-24.04-arm, linux/arm64, arm64) (push) Blocked by required conditions
Build and Push Docker Images / build (./surfsense_backend, ./surfsense_backend/Dockerfile, backend, surfsense-backend, ubuntu-latest, linux/amd64, amd64) (push) Blocked by required conditions
Build and Push Docker Images / build (./surfsense_web, ./surfsense_web/Dockerfile, web, surfsense-web, ubuntu-24.04-arm, linux/arm64, arm64) (push) Blocked by required conditions
Build and Push Docker Images / build (./surfsense_web, ./surfsense_web/Dockerfile, web, surfsense-web, ubuntu-latest, linux/amd64, amd64) (push) Blocked by required conditions
Build and Push Docker Images / create_manifest (backend, surfsense-backend) (push) Blocked by required conditions
Build and Push Docker Images / create_manifest (web, surfsense-web) (push) Blocked by required conditions
- Introduced a `ProcessingMode` enum to differentiate between basic and premium processing modes. - Updated `EtlRequest` to include a `processing_mode` field, defaulting to basic. - Enhanced ETL pipeline services to utilize the selected processing mode for Azure Document Intelligence and LlamaCloud parsing. - Modified various routes and services to handle processing mode, affecting document upload and indexing tasks. - Improved error handling and logging to include processing mode details. - Added tests to validate processing mode functionality and its impact on ETL operations.
59 lines
1.6 KiB
Python
59 lines
1.6 KiB
Python
"""Task dispatcher abstraction for background document processing.
|
|
|
|
Decouples the upload endpoint from Celery so tests can swap in a
|
|
synchronous (inline) implementation that needs only PostgreSQL.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Protocol
|
|
|
|
|
|
class TaskDispatcher(Protocol):
|
|
async def dispatch_file_processing(
|
|
self,
|
|
*,
|
|
document_id: int,
|
|
temp_path: str,
|
|
filename: str,
|
|
search_space_id: int,
|
|
user_id: str,
|
|
should_summarize: bool = False,
|
|
use_vision_llm: bool = False,
|
|
processing_mode: str = "basic",
|
|
) -> None: ...
|
|
|
|
|
|
class CeleryTaskDispatcher:
|
|
"""Production dispatcher — fires Celery tasks via Redis broker."""
|
|
|
|
async def dispatch_file_processing(
|
|
self,
|
|
*,
|
|
document_id: int,
|
|
temp_path: str,
|
|
filename: str,
|
|
search_space_id: int,
|
|
user_id: str,
|
|
should_summarize: bool = False,
|
|
use_vision_llm: bool = False,
|
|
processing_mode: str = "basic",
|
|
) -> None:
|
|
from app.tasks.celery_tasks.document_tasks import (
|
|
process_file_upload_with_document_task,
|
|
)
|
|
|
|
process_file_upload_with_document_task.delay(
|
|
document_id=document_id,
|
|
temp_path=temp_path,
|
|
filename=filename,
|
|
search_space_id=search_space_id,
|
|
user_id=user_id,
|
|
should_summarize=should_summarize,
|
|
use_vision_llm=use_vision_llm,
|
|
processing_mode=processing_mode,
|
|
)
|
|
|
|
|
|
async def get_task_dispatcher() -> TaskDispatcher:
|
|
return CeleryTaskDispatcher()
|