Body
", encoding="utf-8") + + result = await EtlPipelineService().extract( + EtlRequest(file_path=str(html_file), filename="page.html") + ) + + assert "Title" in result.markdown_content + assert "Body" in result.markdown_content + assert result.etl_service == "DIRECT_CONVERT" + + +async def test_extract_mp3_returns_transcription(tmp_path, mocker): + """An .mp3 audio file is transcribed via litellm.atranscription.""" + audio_file = tmp_path / "recording.mp3" + audio_file.write_bytes(b"\x00" * 100) + + mocker.patch("app.config.config.STT_SERVICE", "openai/whisper-1") + mocker.patch("app.config.config.STT_SERVICE_API_KEY", "fake-key") + mocker.patch("app.config.config.STT_SERVICE_API_BASE", None) + + mock_transcription = mocker.patch( + "app.etl_pipeline.parsers.audio.atranscription", + return_value={"text": "Hello from audio"}, + ) + + result = await EtlPipelineService().extract( + EtlRequest(file_path=str(audio_file), filename="recording.mp3") + ) + + assert "Hello from audio" in result.markdown_content + assert result.etl_service == "AUDIO" + assert result.content_type == "audio" + mock_transcription.assert_called_once() + + +# --------------------------------------------------------------------------- +# Slice 7 - DOCLING document parsing +# --------------------------------------------------------------------------- + + +async def test_extract_pdf_with_docling(tmp_path, mocker): + """A .pdf file with ETL_SERVICE=DOCLING returns parsed markdown.""" + pdf_file = tmp_path / "report.pdf" + pdf_file.write_bytes(b"%PDF-1.4 fake") + + mocker.patch("app.config.config.ETL_SERVICE", "DOCLING") + + fake_docling = mocker.AsyncMock() + fake_docling.process_document.return_value = {"content": "# Parsed PDF"} + mocker.patch( + "app.services.docling_service.create_docling_service", + return_value=fake_docling, + ) + + result = await EtlPipelineService().extract( + EtlRequest(file_path=str(pdf_file), filename="report.pdf") + ) + + assert result.markdown_content == "# Parsed PDF" + assert result.etl_service == "DOCLING" + assert result.content_type == "document" + + +# --------------------------------------------------------------------------- +# Slice 8 - UNSTRUCTURED document parsing +# --------------------------------------------------------------------------- + + +async def test_extract_pdf_with_unstructured(tmp_path, mocker): + """A .pdf file with ETL_SERVICE=UNSTRUCTURED returns parsed markdown.""" + pdf_file = tmp_path / "report.pdf" + pdf_file.write_bytes(b"%PDF-1.4 fake") + + mocker.patch("app.config.config.ETL_SERVICE", "UNSTRUCTURED") + + class FakeDoc: + def __init__(self, text): + self.page_content = text + + fake_loader_instance = mocker.AsyncMock() + fake_loader_instance.aload.return_value = [ + FakeDoc("Page 1 content"), + FakeDoc("Page 2 content"), + ] + mocker.patch( + "langchain_unstructured.UnstructuredLoader", + return_value=fake_loader_instance, + ) + + result = await EtlPipelineService().extract( + EtlRequest(file_path=str(pdf_file), filename="report.pdf") + ) + + assert "Page 1 content" in result.markdown_content + assert "Page 2 content" in result.markdown_content + assert result.etl_service == "UNSTRUCTURED" + assert result.content_type == "document" + + +# --------------------------------------------------------------------------- +# Slice 9 - LLAMACLOUD document parsing +# --------------------------------------------------------------------------- + + +async def test_extract_pdf_with_llamacloud(tmp_path, mocker): + """A .pdf file with ETL_SERVICE=LLAMACLOUD returns parsed markdown.""" + pdf_file = tmp_path / "report.pdf" + pdf_file.write_bytes(b"%PDF-1.4 fake content " * 10) + + mocker.patch("app.config.config.ETL_SERVICE", "LLAMACLOUD") + mocker.patch("app.config.config.LLAMA_CLOUD_API_KEY", "fake-key", create=True) + + class FakeDoc: + text = "# LlamaCloud parsed" + + class FakeJobResult: + pages = [] + + def get_markdown_documents(self, split_by_page=True): + return [FakeDoc()] + + fake_parser = mocker.AsyncMock() + fake_parser.aparse.return_value = FakeJobResult() + mocker.patch( + "llama_cloud_services.LlamaParse", + return_value=fake_parser, + ) + mocker.patch( + "llama_cloud_services.parse.utils.ResultType", + mocker.MagicMock(MD="md"), + ) + + result = await EtlPipelineService().extract( + EtlRequest(file_path=str(pdf_file), filename="report.pdf", estimated_pages=5) + ) + + assert result.markdown_content == "# LlamaCloud parsed" + assert result.etl_service == "LLAMACLOUD" + assert result.content_type == "document" + + +# --------------------------------------------------------------------------- +# Slice 10 - unknown extension falls through to document ETL +# --------------------------------------------------------------------------- + + +async def test_unknown_extension_uses_document_etl(tmp_path, mocker): + """An allowlisted document extension (.docx) routes to the document ETL path.""" + docx_file = tmp_path / "doc.docx" + docx_file.write_bytes(b"PK fake docx") + + mocker.patch("app.config.config.ETL_SERVICE", "DOCLING") + + fake_docling = mocker.AsyncMock() + fake_docling.process_document.return_value = {"content": "Docx content"} + mocker.patch( + "app.services.docling_service.create_docling_service", + return_value=fake_docling, + ) + + result = await EtlPipelineService().extract( + EtlRequest(file_path=str(docx_file), filename="doc.docx") + ) + + assert result.markdown_content == "Docx content" + assert result.content_type == "document" + + +# --------------------------------------------------------------------------- +# Slice 11 - EtlRequest validation +# --------------------------------------------------------------------------- + + +def test_etl_request_requires_filename(): + """EtlRequest rejects missing filename.""" + with pytest.raises(ValueError, match="filename must not be empty"): + EtlRequest(file_path="/tmp/some.txt", filename="") + + +# --------------------------------------------------------------------------- +# Slice 12 - unknown ETL_SERVICE raises EtlServiceUnavailableError +# --------------------------------------------------------------------------- + + +async def test_unknown_etl_service_raises(tmp_path, mocker): + """An unknown ETL_SERVICE raises EtlServiceUnavailableError.""" + from app.etl_pipeline.exceptions import EtlServiceUnavailableError + + pdf_file = tmp_path / "report.pdf" + pdf_file.write_bytes(b"%PDF fake") + + mocker.patch("app.config.config.ETL_SERVICE", "NONEXISTENT") + + with pytest.raises(EtlServiceUnavailableError, match="Unknown ETL_SERVICE"): + await EtlPipelineService().extract( + EtlRequest(file_path=str(pdf_file), filename="report.pdf") + ) + + +# --------------------------------------------------------------------------- +# Slice 13 - unsupported file types are rejected before reaching any parser +# --------------------------------------------------------------------------- + + +def test_unknown_extension_classified_as_unsupported(): + """An unknown extension defaults to UNSUPPORTED (allowlist behaviour).""" + from app.etl_pipeline.file_classifier import FileCategory, classify_file + + assert classify_file("random.xyz") == FileCategory.UNSUPPORTED + + +@pytest.mark.parametrize( + "filename", + [ + "malware.exe", + "archive.zip", + "video.mov", + "font.woff2", + "model.blend", + "data.parquet", + "package.deb", + "firmware.bin", + ], +) +def test_unsupported_extensions_classified_correctly(filename): + """Extensions not in any allowlist are classified as UNSUPPORTED.""" + from app.etl_pipeline.file_classifier import FileCategory, classify_file + + assert classify_file(filename) == FileCategory.UNSUPPORTED + + +@pytest.mark.parametrize( + "filename,expected", + [ + ("report.pdf", "document"), + ("doc.docx", "document"), + ("slides.pptx", "document"), + ("sheet.xlsx", "document"), + ("photo.png", "document"), + ("photo.jpg", "document"), + ("book.epub", "document"), + ("letter.odt", "document"), + ("readme.md", "plaintext"), + ("data.csv", "direct_convert"), + ], +) +def test_parseable_extensions_classified_correctly(filename, expected): + """Parseable files are classified into their correct category.""" + from app.etl_pipeline.file_classifier import FileCategory, classify_file + + result = classify_file(filename) + assert result != FileCategory.UNSUPPORTED + assert result.value == expected + + +async def test_extract_unsupported_file_raises_error(tmp_path): + """EtlPipelineService.extract() raises EtlUnsupportedFileError for .exe files.""" + from app.etl_pipeline.exceptions import EtlUnsupportedFileError + + exe_file = tmp_path / "program.exe" + exe_file.write_bytes(b"\x00" * 10) + + with pytest.raises(EtlUnsupportedFileError, match="not supported"): + await EtlPipelineService().extract( + EtlRequest(file_path=str(exe_file), filename="program.exe") + ) + + +async def test_extract_zip_raises_unsupported_error(tmp_path): + """EtlPipelineService.extract() raises EtlUnsupportedFileError for .zip archives.""" + from app.etl_pipeline.exceptions import EtlUnsupportedFileError + + zip_file = tmp_path / "archive.zip" + zip_file.write_bytes(b"PK\x03\x04") + + with pytest.raises(EtlUnsupportedFileError, match="not supported"): + await EtlPipelineService().extract( + EtlRequest(file_path=str(zip_file), filename="archive.zip") + ) + + +# --------------------------------------------------------------------------- +# Slice 14 - should_skip_for_service (per-parser document filtering) +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "filename,etl_service,expected_skip", + [ + ("file.eml", "DOCLING", True), + ("file.eml", "UNSTRUCTURED", False), + ("file.docm", "LLAMACLOUD", False), + ("file.docm", "DOCLING", True), + ("file.txt", "DOCLING", False), + ("file.csv", "LLAMACLOUD", False), + ("file.mp3", "UNSTRUCTURED", False), + ("file.exe", "LLAMACLOUD", True), + ("file.pdf", "DOCLING", False), + ("file.webp", "DOCLING", False), + ("file.webp", "UNSTRUCTURED", True), + ("file.gif", "LLAMACLOUD", False), + ("file.gif", "DOCLING", True), + ("file.heic", "UNSTRUCTURED", False), + ("file.heic", "DOCLING", True), + ("file.svg", "LLAMACLOUD", False), + ("file.svg", "DOCLING", True), + ("file.p7s", "UNSTRUCTURED", False), + ("file.p7s", "LLAMACLOUD", True), + ], +) +def test_should_skip_for_service(filename, etl_service, expected_skip): + from app.etl_pipeline.file_classifier import should_skip_for_service + + assert should_skip_for_service(filename, etl_service) is expected_skip, ( + f"{filename} with {etl_service}: expected skip={expected_skip}" + ) + + +# --------------------------------------------------------------------------- +# Slice 14b - ETL pipeline rejects per-parser incompatible documents +# --------------------------------------------------------------------------- + + +async def test_extract_docm_with_docling_raises_unsupported(tmp_path, mocker): + """Docling cannot parse .docm -- pipeline should reject before dispatching.""" + from app.etl_pipeline.exceptions import EtlUnsupportedFileError + + mocker.patch("app.config.config.ETL_SERVICE", "DOCLING") + + docm_file = tmp_path / "macro.docm" + docm_file.write_bytes(b"\x00" * 10) + + with pytest.raises(EtlUnsupportedFileError, match="not supported by DOCLING"): + await EtlPipelineService().extract( + EtlRequest(file_path=str(docm_file), filename="macro.docm") + ) + + +async def test_extract_eml_with_docling_raises_unsupported(tmp_path, mocker): + """Docling cannot parse .eml -- pipeline should reject before dispatching.""" + from app.etl_pipeline.exceptions import EtlUnsupportedFileError + + mocker.patch("app.config.config.ETL_SERVICE", "DOCLING") + + eml_file = tmp_path / "mail.eml" + eml_file.write_bytes(b"From: test@example.com") + + with pytest.raises(EtlUnsupportedFileError, match="not supported by DOCLING"): + await EtlPipelineService().extract( + EtlRequest(file_path=str(eml_file), filename="mail.eml") + ) diff --git a/surfsense_backend/tests/unit/services/__init__.py b/surfsense_backend/tests/unit/services/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/services/test_docling_image_support.py b/surfsense_backend/tests/unit/services/test_docling_image_support.py new file mode 100644 index 000000000..11ffc0ed1 --- /dev/null +++ b/surfsense_backend/tests/unit/services/test_docling_image_support.py @@ -0,0 +1,70 @@ +"""Test that DoclingService does NOT restrict allowed_formats, letting Docling +accept all its supported formats (PDF, DOCX, PPTX, XLSX, IMAGE, etc.).""" + +from enum import Enum +from unittest.mock import MagicMock, patch + +import pytest + +pytestmark = pytest.mark.unit + + +class _FakeInputFormat(Enum): + PDF = "pdf" + IMAGE = "image" + DOCX = "docx" + PPTX = "pptx" + XLSX = "xlsx" + + +def test_docling_service_does_not_restrict_allowed_formats(): + """DoclingService should NOT pass allowed_formats to DocumentConverter, + so Docling defaults to accepting every InputFormat it supports.""" + + mock_converter_cls = MagicMock() + mock_backend = MagicMock() + + fake_pipeline_options_cls = MagicMock() + fake_pipeline_options = MagicMock() + fake_pipeline_options_cls.return_value = fake_pipeline_options + + fake_pdf_format_option_cls = MagicMock() + + with patch.dict( + "sys.modules", + { + "docling": MagicMock(), + "docling.backend": MagicMock(), + "docling.backend.pypdfium2_backend": MagicMock( + PyPdfiumDocumentBackend=mock_backend + ), + "docling.datamodel": MagicMock(), + "docling.datamodel.base_models": MagicMock(InputFormat=_FakeInputFormat), + "docling.datamodel.pipeline_options": MagicMock( + PdfPipelineOptions=fake_pipeline_options_cls + ), + "docling.document_converter": MagicMock( + DocumentConverter=mock_converter_cls, + PdfFormatOption=fake_pdf_format_option_cls, + ), + }, + ): + from importlib import reload + + import app.services.docling_service as mod + + reload(mod) + + mod.DoclingService() + + call_kwargs = mock_converter_cls.call_args + assert call_kwargs is not None, "DocumentConverter was never called" + + _, kwargs = call_kwargs + assert "allowed_formats" not in kwargs, ( + f"allowed_formats should not be passed — let Docling accept all formats. " + f"Got: {kwargs.get('allowed_formats')}" + ) + assert _FakeInputFormat.PDF in kwargs.get("format_options", {}), ( + "format_options should still configure PDF pipeline options" + ) diff --git a/surfsense_backend/tests/unit/utils/__init__.py b/surfsense_backend/tests/unit/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/surfsense_backend/tests/unit/utils/test_file_extensions.py b/surfsense_backend/tests/unit/utils/test_file_extensions.py new file mode 100644 index 000000000..c33b39f05 --- /dev/null +++ b/surfsense_backend/tests/unit/utils/test_file_extensions.py @@ -0,0 +1,154 @@ +"""Tests for the DOCUMENT_EXTENSIONS allowlist module.""" + +import pytest + +pytestmark = pytest.mark.unit + + +def test_pdf_is_supported_document(): + from app.utils.file_extensions import is_supported_document_extension + + assert is_supported_document_extension("report.pdf") is True + + +def test_exe_is_not_supported_document(): + from app.utils.file_extensions import is_supported_document_extension + + assert is_supported_document_extension("malware.exe") is False + + +@pytest.mark.parametrize( + "filename", + [ + "report.pdf", + "doc.docx", + "old.doc", + "sheet.xlsx", + "legacy.xls", + "slides.pptx", + "deck.ppt", + "macro.docm", + "macro.xlsm", + "macro.pptm", + "photo.png", + "photo.jpg", + "photo.jpeg", + "scan.bmp", + "scan.tiff", + "scan.tif", + "photo.webp", + "anim.gif", + "iphone.heic", + "manual.rtf", + "book.epub", + "letter.odt", + "data.ods", + "presentation.odp", + "inbox.eml", + "outlook.msg", + "korean.hwpx", + "korean.hwp", + "template.dot", + "template.dotm", + "template.pot", + "template.potx", + "binary.xlsb", + "workspace.xlw", + "vector.svg", + "signature.p7s", + ], +) +def test_document_extensions_are_supported(filename): + from app.utils.file_extensions import is_supported_document_extension + + assert is_supported_document_extension(filename) is True, ( + f"{filename} should be supported" + ) + + +@pytest.mark.parametrize( + "filename", + [ + "malware.exe", + "archive.zip", + "video.mov", + "font.woff2", + "model.blend", + "random.xyz", + "data.parquet", + "package.deb", + ], +) +def test_non_document_extensions_are_not_supported(filename): + from app.utils.file_extensions import is_supported_document_extension + + assert is_supported_document_extension(filename) is False, ( + f"{filename} should NOT be supported" + ) + + +# --------------------------------------------------------------------------- +# Per-parser extension sets +# --------------------------------------------------------------------------- + + +def test_union_equals_all_three_sets(): + from app.utils.file_extensions import ( + DOCLING_DOCUMENT_EXTENSIONS, + DOCUMENT_EXTENSIONS, + LLAMAPARSE_DOCUMENT_EXTENSIONS, + UNSTRUCTURED_DOCUMENT_EXTENSIONS, + ) + + expected = ( + DOCLING_DOCUMENT_EXTENSIONS + | LLAMAPARSE_DOCUMENT_EXTENSIONS + | UNSTRUCTURED_DOCUMENT_EXTENSIONS + ) + assert expected == DOCUMENT_EXTENSIONS + + +def test_get_extensions_for_docling(): + from app.utils.file_extensions import get_document_extensions_for_service + + exts = get_document_extensions_for_service("DOCLING") + assert ".pdf" in exts + assert ".webp" in exts + assert ".docx" in exts + assert ".eml" not in exts + assert ".docm" not in exts + assert ".gif" not in exts + assert ".heic" not in exts + + +def test_get_extensions_for_llamacloud(): + from app.utils.file_extensions import get_document_extensions_for_service + + exts = get_document_extensions_for_service("LLAMACLOUD") + assert ".docm" in exts + assert ".gif" in exts + assert ".svg" in exts + assert ".hwp" in exts + assert ".eml" not in exts + assert ".heic" not in exts + + +def test_get_extensions_for_unstructured(): + from app.utils.file_extensions import get_document_extensions_for_service + + exts = get_document_extensions_for_service("UNSTRUCTURED") + assert ".eml" in exts + assert ".heic" in exts + assert ".p7s" in exts + assert ".docm" not in exts + assert ".gif" not in exts + assert ".svg" not in exts + + +def test_get_extensions_for_none_returns_union(): + from app.utils.file_extensions import ( + DOCUMENT_EXTENSIONS, + get_document_extensions_for_service, + ) + + assert get_document_extensions_for_service(None) == DOCUMENT_EXTENSIONS diff --git a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsFilters.tsx b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsFilters.tsx index b85af13b7..a795b61c7 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsFilters.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/components/DocumentsFilters.tsx @@ -8,6 +8,7 @@ import { Button } from "@/components/ui/button"; import { Checkbox } from "@/components/ui/checkbox"; import { Input } from "@/components/ui/input"; import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; +import { ToggleGroup, ToggleGroupItem } from "@/components/ui/toggle-group"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; import type { DocumentTypeEnum } from "@/contracts/types/document.types"; import { getDocumentTypeIcon, getDocumentTypeLabel } from "./DocumentTypeIcon"; @@ -63,109 +64,113 @@ export function DocumentsFilters({ return (- If SurfSense doesn't appear in the list, click + and select it from Applications. + If SurfSense doesn't appear in the list, click + and + select it from Applications.
{suggestion}
{thread.title || "New Chat"}
-- {relativeTime} -
+{relativeTime}