apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-06-10 20:35:17 +02:00
Code Issues Projects Releases Packages Wiki Activity

chore: ran linting

Browse source
This commit is contained in:
Anish Sarkar 2026-04-07 05:55:39 +05:30
parent 5803fe79da
commit 0a26a6c5bb
54 changed files with 1015 additions and 672 deletions
Download patch file Download diff file Expand all files Collapse all files

14
surfsense_backend/app/connectors/dropbox/client.py
View file

@ -225,9 +225,7 @@ class DropboxClient:
return all_items, None
async def get_latest_cursor(
self, path: str = ""
) -> tuple[str | None, str | None]:
async def get_latest_cursor(self, path: str = "") -> tuple[str | None, str | None]:
"""Get a cursor representing the current state of a folder.
Uses /2/files/list_folder/get_latest_cursor so we can later call
@ -251,9 +249,7 @@ class DropboxClient:
"""
all_entries: list[dict[str, Any]] = []
resp = await self._request(
"/2/files/list_folder/continue", {"cursor": cursor}
)
resp = await self._request("/2/files/list_folder/continue", {"cursor": cursor})
if resp.status_code == 401:
return [], None, "Dropbox authentication expired (401)"
if resp.status_code != 200:
@ -268,7 +264,11 @@ class DropboxClient:
"/2/files/list_folder/continue", {"cursor": cursor}
)
if resp.status_code != 200:
return all_entries, data.get("cursor"), f"Pagination failed: {resp.status_code}"
return (
all_entries,
data.get("cursor"),
f"Pagination failed: {resp.status_code}",
)
data = resp.json()
all_entries.extend(data.get("entries", []))

8
surfsense_backend/app/connectors/google_drive/content_extractor.py
View file

@ -100,7 +100,9 @@ async def download_and_extract_content(
if error:
return None, drive_metadata, error
etl_filename = file_name + extension if is_google_workspace_file(mime_type) else file_name
etl_filename = (
file_name + extension if is_google_workspace_file(mime_type) else file_name
)
markdown = await _parse_file_to_markdown(temp_file_path, etl_filename)
return markdown, drive_metadata, None
@ -233,7 +235,9 @@ async def download_and_process_file(
"."
)[-1]
etl_filename = file_name + extension if is_google_workspace_file(mime_type) else file_name
etl_filename = (
file_name + extension if is_google_workspace_file(mime_type) else file_name
)
logger.info(f"Processing {file_name} with Surfsense's file processor")
await process_file_in_background(
file_path=temp_file_path,

9
surfsense_backend/app/etl_pipeline/etl_pipeline_service.py
View file

@ -1,6 +1,9 @@
from app.config import config as app_config
from app.etl_pipeline.etl_document import EtlRequest, EtlResult
from app.etl_pipeline.exceptions import EtlServiceUnavailableError, EtlUnsupportedFileError
from app.etl_pipeline.exceptions import (
EtlServiceUnavailableError,
EtlUnsupportedFileError,
)
from app.etl_pipeline.file_classifier import FileCategory, classify_file
from app.etl_pipeline.parsers.audio import transcribe_audio
from app.etl_pipeline.parsers.direct_convert import convert_file_directly
@ -78,9 +81,7 @@ class EtlPipelineService:
request.file_path, request.estimated_pages
)
else:
raise EtlServiceUnavailableError(
f"Unknown ETL_SERVICE: {etl_service}"
)
raise EtlServiceUnavailableError(f"Unknown ETL_SERVICE: {etl_service}")
return EtlResult(
markdown_content=content,

105
surfsense_backend/app/etl_pipeline/file_classifier.py
View file

@ -1,27 +1,96 @@
from enum import Enum
from pathlib import PurePosixPath
from app.utils.file_extensions import DOCUMENT_EXTENSIONS, get_document_extensions_for_service
from app.utils.file_extensions import (
DOCUMENT_EXTENSIONS,
get_document_extensions_for_service,
)
PLAINTEXT_EXTENSIONS = frozenset(
{
".md", ".markdown", ".txt", ".text",
".json", ".jsonl", ".yaml", ".yml", ".toml", ".ini", ".cfg", ".conf", ".xml",
".css", ".scss", ".less", ".sass",
".py", ".pyw", ".pyi", ".pyx",
".js", ".jsx", ".ts", ".tsx", ".mjs", ".cjs",
".java", ".kt", ".kts", ".scala", ".groovy",
".c", ".h", ".cpp", ".cxx", ".cc", ".hpp", ".hxx",
".cs", ".fs", ".fsx",
".go", ".rs", ".rb", ".php", ".pl", ".pm", ".lua", ".swift",
".m", ".mm", ".r", ".jl",
".sh", ".bash", ".zsh", ".fish", ".bat", ".cmd", ".ps1",
".sql", ".graphql", ".gql",
".env", ".gitignore", ".dockerignore", ".editorconfig",
".makefile", ".cmake",
".log", ".rst", ".tex", ".bib", ".org", ".adoc", ".asciidoc",
".vue", ".svelte", ".astro",
".tf", ".hcl", ".proto",
".md",
".markdown",
".txt",
".text",
".json",
".jsonl",
".yaml",
".yml",
".toml",
".ini",
".cfg",
".conf",
".xml",
".css",
".scss",
".less",
".sass",
".py",
".pyw",
".pyi",
".pyx",
".js",
".jsx",
".ts",
".tsx",
".mjs",
".cjs",
".java",
".kt",
".kts",
".scala",
".groovy",
".c",
".h",
".cpp",
".cxx",
".cc",
".hpp",
".hxx",
".cs",
".fs",
".fsx",
".go",
".rs",
".rb",
".php",
".pl",
".pm",
".lua",
".swift",
".m",
".mm",
".r",
".jl",
".sh",
".bash",
".zsh",
".fish",
".bat",
".cmd",
".ps1",
".sql",
".graphql",
".gql",
".env",
".gitignore",
".dockerignore",
".editorconfig",
".makefile",
".cmake",
".log",
".rst",
".tex",
".bib",
".org",
".adoc",
".asciidoc",
".vue",
".svelte",
".astro",
".tf",
".hcl",
".proto",
}
)

12
surfsense_backend/app/etl_pipeline/parsers/llamacloud.py
View file

@ -66,16 +66,12 @@ async def parse_with_llamacloud(file_path: str, estimated_pages: int) -> str:
)
if hasattr(result, "get_markdown_documents"):
markdown_docs = result.get_markdown_documents(
split_by_page=False
)
markdown_docs = result.get_markdown_documents(split_by_page=False)
if markdown_docs and hasattr(markdown_docs[0], "text"):
return markdown_docs[0].text
if hasattr(result, "pages") and result.pages:
return "\n\n".join(
p.md
for p in result.pages
if hasattr(p, "md") and p.md
p.md for p in result.pages if hasattr(p, "md") and p.md
)
return str(result)
@ -83,9 +79,7 @@ async def parse_with_llamacloud(file_path: str, estimated_pages: int) -> str:
if result and hasattr(result[0], "text"):
return result[0].text
return "\n\n".join(
doc.page_content
if hasattr(doc, "page_content")
else str(doc)
doc.page_content if hasattr(doc, "page_content") else str(doc)
for doc in result
)

9
surfsense_backend/app/routes/autocomplete_routes.py
View file

@ -1,4 +1,4 @@
from fastapi import APIRouter, Depends, HTTPException
from fastapi import APIRouter, Depends
from fastapi.responses import StreamingResponse
from pydantic import BaseModel, Field
from sqlalchemy.ext.asyncio import AsyncSession
@ -31,8 +31,11 @@ async def vision_autocomplete_stream(
return StreamingResponse(
stream_vision_autocomplete(
body.screenshot, body.search_space_id, session,
app_name=body.app_name, window_title=body.window_title,
body.screenshot,
body.search_space_id,
session,
app_name=body.app_name,
window_title=body.window_title,
),
media_type="text/event-stream",
headers={

14
surfsense_backend/app/routes/search_source_connectors_routes.py
View file

@ -2647,7 +2647,12 @@ async def run_onedrive_indexing(
stage="fetching",
)
total_indexed, total_skipped, error_message, total_unsupported = await index_onedrive_files(
(
total_indexed,
total_skipped,
error_message,
total_unsupported,
) = await index_onedrive_files(
session,
connector_id,
search_space_id,
@ -2756,7 +2761,12 @@ async def run_dropbox_indexing(
stage="fetching",
)
total_indexed, total_skipped, error_message, total_unsupported = await index_dropbox_files(
(
total_indexed,
total_skipped,
error_message,
total_unsupported,
) = await index_dropbox_files(
session,
connector_id,
search_space_id,

65
surfsense_backend/app/services/vision_autocomplete_service.py
View file

@ -1,5 +1,5 @@
import logging
from typing import AsyncGenerator
from collections.abc import AsyncGenerator
from langchain_core.messages import HumanMessage, SystemMessage
from sqlalchemy.ext.asyncio import AsyncSession
@ -68,8 +68,10 @@ def _is_vision_unsupported_error(e: Exception) -> bool:
async def _extract_query_from_screenshot(
llm, screenshot_data_url: str,
app_name: str = "", window_title: str = "",
llm,
screenshot_data_url: str,
app_name: str = "",
window_title: str = "",
) -> str | None:
"""Ask the Vision LLM to describe what the user is working on.
@ -78,18 +80,26 @@ async def _extract_query_from_screenshot(
"""
if app_name:
prompt_text = EXTRACT_QUERY_PROMPT_WITH_APP.format(
app_name=app_name, window_title=window_title,
app_name=app_name,
window_title=window_title,
)
else:
prompt_text = EXTRACT_QUERY_PROMPT
try:
response = await llm.ainvoke([
HumanMessage(content=[
{"type": "text", "text": prompt_text},
{"type": "image_url", "image_url": {"url": screenshot_data_url}},
]),
])
response = await llm.ainvoke(
[
HumanMessage(
content=[
{"type": "text", "text": prompt_text},
{
"type": "image_url",
"image_url": {"url": screenshot_data_url},
},
]
),
]
)
query = response.content.strip() if hasattr(response, "content") else ""
return query if query else None
except Exception as e:
@ -167,10 +177,15 @@ async def stream_vision_autocomplete(
kb_context = ""
try:
query = await _extract_query_from_screenshot(
llm, screenshot_data_url, app_name=app_name, window_title=window_title,
llm,
screenshot_data_url,
app_name=app_name,
window_title=window_title,
)
except Exception as e:
logger.warning(f"Vision autocomplete: selected model does not support vision: {e}")
logger.warning(
f"Vision autocomplete: selected model does not support vision: {e}"
)
yield streaming.format_message_start()
yield streaming.format_error(vision_error_msg)
yield streaming.format_done()
@ -183,16 +198,18 @@ async def stream_vision_autocomplete(
messages = [
SystemMessage(content=system_prompt),
HumanMessage(content=[
{
"type": "text",
"text": "Analyze this screenshot. Understand the full context of what the user is working on, then generate the text they most likely want to write in the active text area.",
},
{
"type": "image_url",
"image_url": {"url": screenshot_data_url},
},
]),
HumanMessage(
content=[
{
"type": "text",
"text": "Analyze this screenshot. Understand the full context of what the user is working on, then generate the text they most likely want to write in the active text area.",
},
{
"type": "image_url",
"image_url": {"url": screenshot_data_url},
},
]
),
]
text_started = False
@ -217,7 +234,9 @@ async def stream_vision_autocomplete(
yield streaming.format_text_end(text_id)
if _is_vision_unsupported_error(e):
logger.warning(f"Vision autocomplete: selected model does not support vision: {e}")
logger.warning(
f"Vision autocomplete: selected model does not support vision: {e}"
)
yield streaming.format_error(vision_error_msg)
else:
logger.error(f"Vision autocomplete streaming error: {e}", exc_info=True)

17
surfsense_backend/app/tasks/connector_indexers/dropbox_indexer.py
View file

@ -254,9 +254,7 @@ async def _download_and_index(
return batch_indexed, download_failed + batch_failed
async def _remove_document(
session: AsyncSession, file_id: str, search_space_id: int
):
async def _remove_document(session: AsyncSession, file_id: str, search_space_id: int):
"""Remove a document that was deleted in Dropbox."""
primary_hash = compute_identifier_hash(
DocumentType.DROPBOX_FILE.value, file_id, search_space_id
@ -268,8 +266,7 @@ async def _remove_document(
select(Document).where(
Document.search_space_id == search_space_id,
Document.document_type == DocumentType.DROPBOX_FILE,
cast(Document.document_metadata["dropbox_file_id"], String)
== file_id,
cast(Document.document_metadata["dropbox_file_id"], String) == file_id,
)
)
existing = result.scalar_one_or_none()
@ -671,9 +668,7 @@ async def index_dropbox_files(
saved_cursor = folder_cursors.get(folder_path)
can_use_delta = (
use_delta_sync
and saved_cursor
and connector.last_indexed_at
use_delta_sync and saved_cursor and connector.last_indexed_at
)
if can_use_delta:
@ -739,7 +734,11 @@ async def index_dropbox_files(
await task_logger.log_task_success(
log_entry,
f"Successfully completed Dropbox indexing for connector {connector_id}",
{"files_processed": total_indexed, "files_skipped": total_skipped, "files_unsupported": total_unsupported},
{
"files_processed": total_indexed,
"files_skipped": total_skipped,
"files_unsupported": total_unsupported,
},
)
logger.info(
f"Dropbox indexing completed: {total_indexed} indexed, "

13
surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py
View file

@ -1010,7 +1010,11 @@ async def index_google_drive_files(
documents_unsupported += ru
else:
logger.info(f"Using full scan for connector {connector_id}")
documents_indexed, documents_skipped, documents_unsupported = await _index_full_scan(
(
documents_indexed,
documents_skipped,
documents_unsupported,
) = await _index_full_scan(
drive_client,
session,
connector,
@ -1301,7 +1305,12 @@ async def index_google_drive_selected_files(
log_entry,
f"Batch file indexing completed with {len(errors)} error(s)",
"; ".join(errors),
{"indexed": indexed, "skipped": skipped, "unsupported": unsupported, "error_count": len(errors)},
{
"indexed": indexed,
"skipped": skipped,
"unsupported": unsupported,
"error_count": len(errors),
},
)
else:
await task_logger.log_task_success(

3
surfsense_backend/app/tasks/connector_indexers/local_folder_indexer.py
View file

@ -23,7 +23,6 @@ from sqlalchemy import select
from sqlalchemy.exc import IntegrityError, SQLAlchemyError
from sqlalchemy.ext.asyncio import AsyncSession
from app.config import config
from app.db import (
Document,
DocumentStatus,
@ -153,8 +152,6 @@ def scan_folder(
return files
async def _read_file_content(file_path: str, filename: str) -> str:
"""Read file content via the unified ETL pipeline.

6
surfsense_backend/app/tasks/connector_indexers/onedrive_indexer.py
View file

@ -762,7 +762,11 @@ async def index_onedrive_files(
await task_logger.log_task_success(
log_entry,
f"Successfully completed OneDrive indexing for connector {connector_id}",
{"files_processed": total_indexed, "files_skipped": total_skipped, "files_unsupported": total_unsupported},
{
"files_processed": total_indexed,
"files_skipped": total_skipped,
"files_unsupported": total_unsupported,
},
)
logger.info(
f"OneDrive indexing completed: {total_indexed} indexed, "

12
surfsense_backend/app/tasks/document_processors/file_processors.py
View file

@ -292,8 +292,10 @@ async def process_file_in_background(
)
try:
from app.etl_pipeline.file_classifier import FileCategory as EtlFileCategory
from app.etl_pipeline.file_classifier import classify_file as etl_classify
from app.etl_pipeline.file_classifier import (
FileCategory as EtlFileCategory,
classify_file as etl_classify,
)
category = etl_classify(filename)
@ -345,8 +347,10 @@ async def _extract_file_content(
"""
from app.etl_pipeline.etl_document import EtlRequest
from app.etl_pipeline.etl_pipeline_service import EtlPipelineService
from app.etl_pipeline.file_classifier import FileCategory
from app.etl_pipeline.file_classifier import classify_file as etl_classify
from app.etl_pipeline.file_classifier import (
FileCategory,
classify_file as etl_classify,
)
category = etl_classify(filename)
estimated_pages = 0

97
surfsense_backend/app/utils/file_extensions.py
View file

@ -15,30 +15,83 @@ from pathlib import PurePosixPath
# Per-parser document extension sets (from official documentation)
# ---------------------------------------------------------------------------
DOCLING_DOCUMENT_EXTENSIONS: frozenset[str] = frozenset({
".pdf",
".docx", ".xlsx", ".pptx",
".png", ".jpg", ".jpeg", ".tiff", ".tif", ".bmp", ".webp",
})
DOCLING_DOCUMENT_EXTENSIONS: frozenset[str] = frozenset(
{
".pdf",
".docx",
".xlsx",
".pptx",
".png",
".jpg",
".jpeg",
".tiff",
".tif",
".bmp",
".webp",
}
)
LLAMAPARSE_DOCUMENT_EXTENSIONS: frozenset[str] = frozenset({
".pdf",
".docx", ".doc", ".xlsx", ".xls", ".pptx", ".ppt",
".docm", ".dot", ".dotm", ".pptm", ".pot", ".potx",
".xlsm", ".xlsb", ".xlw",
".rtf", ".epub",
".png", ".jpg", ".jpeg", ".gif", ".bmp", ".tiff", ".tif", ".webp", ".svg",
".odt", ".ods", ".odp",
".hwp", ".hwpx",
})
LLAMAPARSE_DOCUMENT_EXTENSIONS: frozenset[str] = frozenset(
{
".pdf",
".docx",
".doc",
".xlsx",
".xls",
".pptx",
".ppt",
".docm",
".dot",
".dotm",
".pptm",
".pot",
".potx",
".xlsm",
".xlsb",
".xlw",
".rtf",
".epub",
".png",
".jpg",
".jpeg",
".gif",
".bmp",
".tiff",
".tif",
".webp",
".svg",
".odt",
".ods",
".odp",
".hwp",
".hwpx",
}
)
UNSTRUCTURED_DOCUMENT_EXTENSIONS: frozenset[str] = frozenset({
".pdf",
".docx", ".doc", ".xlsx", ".xls", ".pptx", ".ppt",
".png", ".jpg", ".jpeg", ".bmp", ".tiff", ".tif", ".heic",
".rtf", ".epub", ".odt",
".eml", ".msg", ".p7s",
})
UNSTRUCTURED_DOCUMENT_EXTENSIONS: frozenset[str] = frozenset(
{
".pdf",
".docx",
".doc",
".xlsx",
".xls",
".pptx",
".ppt",
".png",
".jpg",
".jpeg",
".bmp",
".tiff",
".tif",
".heic",
".rtf",
".epub",
".odt",
".eml",
".msg",
".p7s",
}
)
# ---------------------------------------------------------------------------
# Union (used by classify_file for routing) + service lookup

20
surfsense_backend/tests/unit/connector_indexers/test_content_extraction.py
View file

@ -6,7 +6,6 @@ real so we know the full path from "cloud gives us bytes" to "we get markdown
back" actually works.
"""
import os
from unittest.mock import AsyncMock, MagicMock
import pytest
@ -21,6 +20,7 @@ _CSV_CONTENT = "name,age\nAlice,30\nBob,25\n"
# Helpers
# ---------------------------------------------------------------------------
async def _write_file(dest_path: str, content: str) -> None:
"""Simulate a cloud client writing downloaded bytes to disk."""
with open(dest_path, "w", encoding="utf-8") as f:
@ -43,8 +43,8 @@ def _make_download_side_effect(content: str):
# Google Drive
# ===================================================================
class TestGoogleDriveContentExtraction:
class TestGoogleDriveContentExtraction:
async def test_txt_file_returns_markdown(self):
from app.connectors.google_drive.content_extractor import (
download_and_extract_content,
@ -76,7 +76,7 @@ class TestGoogleDriveContentExtraction:
file = {"id": "f2", "name": "data.csv", "mimeType": "text/csv"}
markdown, metadata, error = await download_and_extract_content(client, file)
markdown, _metadata, error = await download_and_extract_content(client, file)
assert error is None
assert "Alice" in markdown
@ -93,7 +93,7 @@ class TestGoogleDriveContentExtraction:
file = {"id": "f3", "name": "doc.txt", "mimeType": "text/plain"}
markdown, metadata, error = await download_and_extract_content(client, file)
markdown, _metadata, error = await download_and_extract_content(client, file)
assert markdown is None
assert error == "Network timeout"
@ -103,8 +103,8 @@ class TestGoogleDriveContentExtraction:
# OneDrive
# ===================================================================
class TestOneDriveContentExtraction:
class TestOneDriveContentExtraction:
async def test_txt_file_returns_markdown(self):
from app.connectors.onedrive.content_extractor import (
download_and_extract_content,
@ -144,7 +144,7 @@ class TestOneDriveContentExtraction:
"file": {"mimeType": "text/csv"},
}
markdown, metadata, error = await download_and_extract_content(client, file)
markdown, _metadata, error = await download_and_extract_content(client, file)
assert error is None
assert "Alice" in markdown
@ -164,7 +164,7 @@ class TestOneDriveContentExtraction:
"file": {"mimeType": "text/plain"},
}
markdown, metadata, error = await download_and_extract_content(client, file)
markdown, _metadata, error = await download_and_extract_content(client, file)
assert markdown is None
assert error == "403 Forbidden"
@ -174,8 +174,8 @@ class TestOneDriveContentExtraction:
# Dropbox
# ===================================================================
class TestDropboxContentExtraction:
class TestDropboxContentExtraction:
async def test_txt_file_returns_markdown(self):
from app.connectors.dropbox.content_extractor import (
download_and_extract_content,
@ -217,7 +217,7 @@ class TestDropboxContentExtraction:
"path_lower": "/data.csv",
}
markdown, metadata, error = await download_and_extract_content(client, file)
markdown, _metadata, error = await download_and_extract_content(client, file)
assert error is None
assert "Alice" in markdown
@ -238,7 +238,7 @@ class TestDropboxContentExtraction:
"path_lower": "/big.txt",
}
markdown, metadata, error = await download_and_extract_content(client, file)
markdown, _metadata, error = await download_and_extract_content(client, file)
assert markdown is None
assert error == "Rate limited"

59
surfsense_backend/tests/unit/connector_indexers/test_dropbox_parallel.py
View file

@ -265,6 +265,7 @@ def full_scan_mocks(mock_dropbox_client, monkeypatch):
async def _fake_skip(session, file, search_space_id):
from app.connectors.dropbox.file_types import should_skip_file as _skip
item_skip, unsup_ext = _skip(file)
if item_skip:
if unsup_ext:
@ -468,7 +469,11 @@ async def test_selected_files_fetch_failure_isolation(selected_files_mocks):
indexed, skipped, _unsupported, errors = await _run_selected(
selected_files_mocks,
[("/first.txt", "first.txt"), ("/mid.txt", "mid.txt"), ("/third.txt", "third.txt")],
[
("/first.txt", "first.txt"),
("/mid.txt", "mid.txt"),
("/third.txt", "third.txt"),
],
)
assert indexed == 2
@ -526,8 +531,18 @@ async def test_delta_sync_deletions_call_remove_document(monkeypatch):
import app.tasks.connector_indexers.dropbox_indexer as _mod
entries = [
{".tag": "deleted", "name": "gone.txt", "path_lower": "/gone.txt", "id": "id:del1"},
{".tag": "deleted", "name": "also_gone.pdf", "path_lower": "/also_gone.pdf", "id": "id:del2"},
{
".tag": "deleted",
"name": "gone.txt",
"path_lower": "/gone.txt",
"id": "id:del1",
},
{
".tag": "deleted",
"name": "also_gone.pdf",
"path_lower": "/also_gone.pdf",
"id": "id:del2",
},
]
mock_client = MagicMock()
@ -544,7 +559,7 @@ async def test_delta_sync_deletions_call_remove_document(monkeypatch):
mock_task_logger = MagicMock()
mock_task_logger.log_task_progress = AsyncMock()
indexed, skipped, unsupported, cursor = await _index_with_delta_sync(
_indexed, _skipped, _unsupported, cursor = await _index_with_delta_sync(
mock_client,
AsyncMock(),
_CONNECTOR_ID,
@ -573,7 +588,9 @@ async def test_delta_sync_upserts_filtered_and_downloaded(monkeypatch):
mock_client = MagicMock()
mock_client.get_changes = AsyncMock(return_value=(entries, "cursor-v2", None))
monkeypatch.setattr(_mod, "_should_skip_file", AsyncMock(return_value=(False, None)))
monkeypatch.setattr(
_mod, "_should_skip_file", AsyncMock(return_value=(False, None))
)
download_mock = AsyncMock(return_value=(2, 0))
monkeypatch.setattr(_mod, "_download_and_index", download_mock)
@ -581,7 +598,7 @@ async def test_delta_sync_upserts_filtered_and_downloaded(monkeypatch):
mock_task_logger = MagicMock()
mock_task_logger.log_task_progress = AsyncMock()
indexed, skipped, unsupported, cursor = await _index_with_delta_sync(
indexed, skipped, _unsupported, cursor = await _index_with_delta_sync(
mock_client,
AsyncMock(),
_CONNECTOR_ID,
@ -608,8 +625,18 @@ async def test_delta_sync_mix_deletions_and_upserts(monkeypatch):
import app.tasks.connector_indexers.dropbox_indexer as _mod
entries = [
{".tag": "deleted", "name": "removed.txt", "path_lower": "/removed.txt", "id": "id:del1"},
{".tag": "deleted", "name": "trashed.pdf", "path_lower": "/trashed.pdf", "id": "id:del2"},
{
".tag": "deleted",
"name": "removed.txt",
"path_lower": "/removed.txt",
"id": "id:del1",
},
{
".tag": "deleted",
"name": "trashed.pdf",
"path_lower": "/trashed.pdf",
"id": "id:del2",
},
_make_file_dict("mod1", "updated.txt"),
_make_file_dict("new1", "brandnew.docx"),
]
@ -623,7 +650,9 @@ async def test_delta_sync_mix_deletions_and_upserts(monkeypatch):
remove_calls.append(file_id)
monkeypatch.setattr(_mod, "_remove_document", _fake_remove)
monkeypatch.setattr(_mod, "_should_skip_file", AsyncMock(return_value=(False, None)))
monkeypatch.setattr(
_mod, "_should_skip_file", AsyncMock(return_value=(False, None))
)
download_mock = AsyncMock(return_value=(2, 0))
monkeypatch.setattr(_mod, "_download_and_index", download_mock)
@ -631,7 +660,7 @@ async def test_delta_sync_mix_deletions_and_upserts(monkeypatch):
mock_task_logger = MagicMock()
mock_task_logger.log_task_progress = AsyncMock()
indexed, skipped, unsupported, cursor = await _index_with_delta_sync(
indexed, skipped, _unsupported, cursor = await _index_with_delta_sync(
mock_client,
AsyncMock(),
_CONNECTOR_ID,
@ -665,7 +694,7 @@ async def test_delta_sync_returns_new_cursor(monkeypatch):
mock_task_logger = MagicMock()
mock_task_logger.log_task_progress = AsyncMock()
indexed, skipped, unsupported, cursor = await _index_with_delta_sync(
indexed, skipped, _unsupported, cursor = await _index_with_delta_sync(
mock_client,
AsyncMock(),
_CONNECTOR_ID,
@ -723,9 +752,7 @@ def orchestrator_mocks(monkeypatch):
mock_client = MagicMock()
mock_client.get_latest_cursor = AsyncMock(return_value=("latest-cursor-abc", None))
monkeypatch.setattr(
_mod, "DropboxClient", MagicMock(return_value=mock_client)
)
monkeypatch.setattr(_mod, "DropboxClient", MagicMock(return_value=mock_client))
return {
"connector": mock_connector,
@ -751,7 +778,7 @@ async def test_orchestrator_uses_delta_sync_when_cursor_and_last_indexed(
mock_session = AsyncMock()
mock_session.commit = AsyncMock()
indexed, skipped, error, _unsupported = await index_dropbox_files(
_indexed, _skipped, error, _unsupported = await index_dropbox_files(
mock_session,
_CONNECTOR_ID,
_SEARCH_SPACE_ID,
@ -779,7 +806,7 @@ async def test_orchestrator_falls_back_to_full_scan_without_cursor(
mock_session = AsyncMock()
mock_session.commit = AsyncMock()
indexed, skipped, error, _unsupported = await index_dropbox_files(
_indexed, _skipped, error, _unsupported = await index_dropbox_files(
mock_session,
_CONNECTOR_ID,
_SEARCH_SPACE_ID,

10
surfsense_backend/tests/unit/connector_indexers/test_google_drive_parallel.py
View file

@ -366,7 +366,7 @@ async def test_full_scan_three_phase_counts(full_scan_mocks, monkeypatch):
full_scan_mocks["download_mock"].return_value = (mock_docs, 0)
full_scan_mocks["batch_mock"].return_value = ([], 2, 0)
indexed, skipped, unsupported = await _run_full_scan(full_scan_mocks)
indexed, skipped, _unsupported = await _run_full_scan(full_scan_mocks)
assert indexed == 3 # 1 renamed + 2 from batch
assert skipped == 1 # 1 unchanged
@ -497,7 +497,7 @@ async def test_delta_sync_removals_serial_rest_parallel(monkeypatch):
mock_task_logger = MagicMock()
mock_task_logger.log_task_progress = AsyncMock()
indexed, skipped, unsupported = await _index_with_delta_sync(
indexed, skipped, _unsupported = await _index_with_delta_sync(
MagicMock(),
mock_session,
MagicMock(),
@ -589,7 +589,7 @@ async def test_selected_files_single_file_indexed(selected_files_mocks):
)
selected_files_mocks["download_and_index_mock"].return_value = (1, 0)
indexed, skipped, unsup, errors = await _run_selected(
indexed, skipped, _unsup, errors = await _run_selected(
selected_files_mocks,
[("f1", "report.pdf")],
)
@ -613,7 +613,7 @@ async def test_selected_files_fetch_failure_isolation(selected_files_mocks):
)
selected_files_mocks["download_and_index_mock"].return_value = (2, 0)
indexed, skipped, unsup, errors = await _run_selected(
indexed, skipped, _unsup, errors = await _run_selected(
selected_files_mocks,
[("f1", "first.txt"), ("f2", "mid.txt"), ("f3", "third.txt")],
)
@ -647,7 +647,7 @@ async def test_selected_files_skip_rename_counting(selected_files_mocks):
selected_files_mocks["download_and_index_mock"].return_value = (2, 0)
indexed, skipped, unsup, errors = await _run_selected(
indexed, skipped, _unsup, errors = await _run_selected(
selected_files_mocks,
[
("s1", "unchanged.txt"),

8
surfsense_backend/tests/unit/connector_indexers/test_page_limits.py
View file

@ -219,7 +219,9 @@ async def test_gdrive_files_exceeding_quota_rejected(gdrive_selected_mocks):
None,
)
indexed, _skipped, _unsup, errors = await _run_gdrive_selected(m, [("big", "huge.pdf")])
indexed, _skipped, _unsup, errors = await _run_gdrive_selected(
m, [("big", "huge.pdf")]
)
assert indexed == 0
assert len(errors) == 1
@ -552,7 +554,9 @@ async def test_onedrive_over_quota_rejected(onedrive_selected_mocks):
None,
)
indexed, _skipped, _unsup, errors = await _run_onedrive_selected(m, [("big", "huge.pdf")])
indexed, _skipped, _unsup, errors = await _run_onedrive_selected(
m, [("big", "huge.pdf")]
)
assert indexed == 0
assert len(errors) == 1

10
surfsense_backend/tests/unit/connectors/test_dropbox_client.py
View file

@ -19,6 +19,7 @@ def _make_client() -> DropboxClient:
# ---------- C1: get_latest_cursor ----------
async def test_get_latest_cursor_returns_cursor_string(monkeypatch):
client = _make_client()
@ -34,12 +35,17 @@ async def test_get_latest_cursor_returns_cursor_string(monkeypatch):
assert error is None
client._request.assert_called_once_with(
"/2/files/list_folder/get_latest_cursor",
{"path": "/my-folder", "recursive": False, "include_non_downloadable_files": True},
{
"path": "/my-folder",
"recursive": False,
"include_non_downloadable_files": True,
},
)
# ---------- C2: get_changes returns entries and new cursor ----------
async def test_get_changes_returns_entries_and_cursor(monkeypatch):
client = _make_client()
@ -66,6 +72,7 @@ async def test_get_changes_returns_entries_and_cursor(monkeypatch):
# ---------- C3: get_changes handles pagination ----------
async def test_get_changes_handles_pagination(monkeypatch):
client = _make_client()
@ -98,6 +105,7 @@ async def test_get_changes_handles_pagination(monkeypatch):
# ---------- C4: get_changes raises on 401 ----------
async def test_get_changes_returns_error_on_401(monkeypatch):
client = _make_client()

105
surfsense_backend/tests/unit/connectors/test_dropbox_file_types.py
View file

@ -41,15 +41,40 @@ def test_non_downloadable_item_is_skipped():
@pytest.mark.parametrize(
"filename",
[
"archive.zip", "backup.tar", "data.gz", "stuff.rar", "pack.7z",
"program.exe", "lib.dll", "module.so", "image.dmg", "disk.iso",
"movie.mov", "clip.avi", "video.mkv", "film.wmv", "stream.flv",
"archive.zip",
"backup.tar",
"data.gz",
"stuff.rar",
"pack.7z",
"program.exe",
"lib.dll",
"module.so",
"image.dmg",
"disk.iso",
"movie.mov",
"clip.avi",
"video.mkv",
"film.wmv",
"stream.flv",
"favicon.ico",
"raw.cr2", "photo.nef", "image.arw", "pic.dng",
"design.psd", "vector.ai", "mockup.sketch", "proto.fig",
"font.ttf", "font.otf", "font.woff", "font.woff2",
"model.stl", "scene.fbx", "mesh.blend",
"local.db", "data.sqlite", "access.mdb",
"raw.cr2",
"photo.nef",
"image.arw",
"pic.dng",
"design.psd",
"vector.ai",
"mockup.sketch",
"proto.fig",
"font.ttf",
"font.otf",
"font.woff",
"font.woff2",
"model.stl",
"scene.fbx",
"mesh.blend",
"local.db",
"data.sqlite",
"access.mdb",
],
)
def test_non_parseable_extensions_are_skipped(filename, mocker):
@ -63,9 +88,16 @@ def test_non_parseable_extensions_are_skipped(filename, mocker):
@pytest.mark.parametrize(
"filename",
[
"report.pdf", "document.docx", "sheet.xlsx", "slides.pptx",
"readme.txt", "data.csv", "page.html", "notes.md",
"config.json", "feed.xml",
"report.pdf",
"document.docx",
"sheet.xlsx",
"slides.pptx",
"readme.txt",
"data.csv",
"page.html",
"notes.md",
"config.json",
"feed.xml",
],
)
def test_parseable_documents_are_not_skipped(filename, mocker):
@ -92,30 +124,33 @@ def test_universal_images_are_not_skipped(filename, mocker):
assert ext is None
@pytest.mark.parametrize("filename,service,expected_skip", [
("old.doc", "DOCLING", True),
("old.doc", "LLAMACLOUD", False),
("old.doc", "UNSTRUCTURED", False),
("legacy.xls", "DOCLING", True),
("legacy.xls", "LLAMACLOUD", False),
("legacy.xls", "UNSTRUCTURED", False),
("deck.ppt", "DOCLING", True),
("deck.ppt", "LLAMACLOUD", False),
("deck.ppt", "UNSTRUCTURED", False),
("icon.svg", "DOCLING", True),
("icon.svg", "LLAMACLOUD", False),
("anim.gif", "DOCLING", True),
("anim.gif", "LLAMACLOUD", False),
("photo.webp", "DOCLING", False),
("photo.webp", "LLAMACLOUD", False),
("photo.webp", "UNSTRUCTURED", True),
("live.heic", "DOCLING", True),
("live.heic", "UNSTRUCTURED", False),
("macro.docm", "DOCLING", True),
("macro.docm", "LLAMACLOUD", False),
("mail.eml", "DOCLING", True),
("mail.eml", "UNSTRUCTURED", False),
])
@pytest.mark.parametrize(
"filename,service,expected_skip",
[
("old.doc", "DOCLING", True),
("old.doc", "LLAMACLOUD", False),
("old.doc", "UNSTRUCTURED", False),
("legacy.xls", "DOCLING", True),
("legacy.xls", "LLAMACLOUD", False),
("legacy.xls", "UNSTRUCTURED", False),
("deck.ppt", "DOCLING", True),
("deck.ppt", "LLAMACLOUD", False),
("deck.ppt", "UNSTRUCTURED", False),
("icon.svg", "DOCLING", True),
("icon.svg", "LLAMACLOUD", False),
("anim.gif", "DOCLING", True),
("anim.gif", "LLAMACLOUD", False),
("photo.webp", "DOCLING", False),
("photo.webp", "LLAMACLOUD", False),
("photo.webp", "UNSTRUCTURED", True),
("live.heic", "DOCLING", True),
("live.heic", "UNSTRUCTURED", False),
("macro.docm", "DOCLING", True),
("macro.docm", "LLAMACLOUD", False),
("mail.eml", "DOCLING", True),
("mail.eml", "UNSTRUCTURED", False),
],
)
def test_parser_specific_extensions(filename, service, expected_skip, mocker):
mocker.patch("app.config.config.ETL_SERVICE", service)
item = {".tag": "file", "name": filename}

55
surfsense_backend/tests/unit/connectors/test_google_drive_file_types.py
View file

@ -7,21 +7,37 @@ from app.connectors.google_drive.file_types import should_skip_by_extension
pytestmark = pytest.mark.unit
@pytest.mark.parametrize("filename", [
"malware.exe", "archive.zip", "video.mov", "font.woff2", "model.blend",
])
@pytest.mark.parametrize(
"filename",
[
"malware.exe",
"archive.zip",
"video.mov",
"font.woff2",
"model.blend",
],
)
def test_unsupported_extensions_are_skipped_regardless_of_service(filename, mocker):
"""Truly unsupported files are skipped no matter which ETL service is configured."""
for service in ("DOCLING", "LLAMACLOUD", "UNSTRUCTURED"):
mocker.patch("app.config.config.ETL_SERVICE", service)
skip, ext = should_skip_by_extension(filename)
skip, _ext = should_skip_by_extension(filename)
assert skip is True
@pytest.mark.parametrize("filename", [
"report.pdf", "doc.docx", "sheet.xlsx", "slides.pptx",
"readme.txt", "data.csv", "photo.png", "notes.md",
])
@pytest.mark.parametrize(
"filename",
[
"report.pdf",
"doc.docx",
"sheet.xlsx",
"slides.pptx",
"readme.txt",
"data.csv",
"photo.png",
"notes.md",
],
)
def test_universal_extensions_are_not_skipped(filename, mocker):
"""Files supported by all parsers (or handled by plaintext/direct_convert) are never skipped."""
for service in ("DOCLING", "LLAMACLOUD", "UNSTRUCTURED"):
@ -31,16 +47,19 @@ def test_universal_extensions_are_not_skipped(filename, mocker):
assert ext is None
@pytest.mark.parametrize("filename,service,expected_skip", [
("macro.docm", "DOCLING", True),
("macro.docm", "LLAMACLOUD", False),
("mail.eml", "DOCLING", True),
("mail.eml", "UNSTRUCTURED", False),
("photo.gif", "DOCLING", True),
("photo.gif", "LLAMACLOUD", False),
("photo.heic", "UNSTRUCTURED", False),
("photo.heic", "DOCLING", True),
])
@pytest.mark.parametrize(
"filename,service,expected_skip",
[
("macro.docm", "DOCLING", True),
("macro.docm", "LLAMACLOUD", False),
("mail.eml", "DOCLING", True),
("mail.eml", "UNSTRUCTURED", False),
("photo.gif", "DOCLING", True),
("photo.gif", "LLAMACLOUD", False),
("photo.heic", "UNSTRUCTURED", False),
("photo.heic", "DOCLING", True),
],
)
def test_parser_specific_extensions(filename, service, expected_skip, mocker):
mocker.patch("app.config.config.ETL_SERVICE", service)
skip, ext = should_skip_by_extension(filename)

49
surfsense_backend/tests/unit/connectors/test_onedrive_file_types.py
View file

@ -45,9 +45,16 @@ def test_onenote_is_skipped():
# ---------------------------------------------------------------------------
@pytest.mark.parametrize("filename", [
"malware.exe", "archive.zip", "video.mov", "font.woff2", "model.blend",
])
@pytest.mark.parametrize(
"filename",
[
"malware.exe",
"archive.zip",
"video.mov",
"font.woff2",
"model.blend",
],
)
def test_unsupported_extensions_are_skipped(filename, mocker):
mocker.patch("app.config.config.ETL_SERVICE", "DOCLING")
item = {"name": filename, "file": {"mimeType": "application/octet-stream"}}
@ -56,10 +63,19 @@ def test_unsupported_extensions_are_skipped(filename, mocker):
assert ext is not None
@pytest.mark.parametrize("filename", [
"report.pdf", "doc.docx", "sheet.xlsx", "slides.pptx",
"readme.txt", "data.csv", "photo.png", "notes.md",
])
@pytest.mark.parametrize(
"filename",
[
"report.pdf",
"doc.docx",
"sheet.xlsx",
"slides.pptx",
"readme.txt",
"data.csv",
"photo.png",
"notes.md",
],
)
def test_universal_files_are_not_skipped(filename, mocker):
for service in ("DOCLING", "LLAMACLOUD", "UNSTRUCTURED"):
mocker.patch("app.config.config.ETL_SERVICE", service)
@ -69,14 +85,17 @@ def test_universal_files_are_not_skipped(filename, mocker):
assert ext is None
@pytest.mark.parametrize("filename,service,expected_skip", [
("macro.docm", "DOCLING", True),
("macro.docm", "LLAMACLOUD", False),
("mail.eml", "DOCLING", True),
("mail.eml", "UNSTRUCTURED", False),
("photo.heic", "UNSTRUCTURED", False),
("photo.heic", "DOCLING", True),
])
@pytest.mark.parametrize(
"filename,service,expected_skip",
[
("macro.docm", "DOCLING", True),
("macro.docm", "LLAMACLOUD", False),
("mail.eml", "DOCLING", True),
("mail.eml", "UNSTRUCTURED", False),
("photo.heic", "UNSTRUCTURED", False),
("photo.heic", "DOCLING", True),
],
)
def test_parser_specific_extensions(filename, service, expected_skip, mocker):
mocker.patch("app.config.config.ETL_SERVICE", service)
item = {"name": filename, "file": {"mimeType": "application/octet-stream"}}

4
surfsense_backend/tests/unit/etl_pipeline/conftest.py
View file

@ -24,6 +24,4 @@ def _stub_package(dotted: str, fs_dir: Path) -> None:
_stub_package("app", _BACKEND / "app")
_stub_package("app.etl_pipeline", _BACKEND / "app" / "etl_pipeline")
_stub_package(
"app.etl_pipeline.parsers", _BACKEND / "app" / "etl_pipeline" / "parsers"
)
_stub_package("app.etl_pipeline.parsers", _BACKEND / "app" / "etl_pipeline" / "parsers")

113
surfsense_backend/tests/unit/etl_pipeline/test_etl_pipeline_service.py
View file

@ -144,7 +144,7 @@ async def test_extract_mp3_returns_transcription(tmp_path, mocker):
# ---------------------------------------------------------------------------
# Slice 7 – DOCLING document parsing
# Slice 7 - DOCLING document parsing
# ---------------------------------------------------------------------------
@ -172,7 +172,7 @@ async def test_extract_pdf_with_docling(tmp_path, mocker):
# ---------------------------------------------------------------------------
# Slice 8 – UNSTRUCTURED document parsing
# Slice 8 - UNSTRUCTURED document parsing
# ---------------------------------------------------------------------------
@ -208,7 +208,7 @@ async def test_extract_pdf_with_unstructured(tmp_path, mocker):
# ---------------------------------------------------------------------------
# Slice 9 – LLAMACLOUD document parsing
# Slice 9 - LLAMACLOUD document parsing
# ---------------------------------------------------------------------------
@ -241,9 +241,7 @@ async def test_extract_pdf_with_llamacloud(tmp_path, mocker):
)
result = await EtlPipelineService().extract(
EtlRequest(
file_path=str(pdf_file), filename="report.pdf", estimated_pages=5
)
EtlRequest(file_path=str(pdf_file), filename="report.pdf", estimated_pages=5)
)
assert result.markdown_content == "# LlamaCloud parsed"
@ -252,7 +250,7 @@ async def test_extract_pdf_with_llamacloud(tmp_path, mocker):
# ---------------------------------------------------------------------------
# Slice 10 – unknown extension falls through to document ETL
# Slice 10 - unknown extension falls through to document ETL
# ---------------------------------------------------------------------------
@ -279,18 +277,18 @@ async def test_unknown_extension_uses_document_etl(tmp_path, mocker):
# ---------------------------------------------------------------------------
# Slice 11 – EtlRequest validation
# Slice 11 - EtlRequest validation
# ---------------------------------------------------------------------------
def test_etl_request_requires_filename():
"""EtlRequest rejects missing filename."""
with pytest.raises(Exception):
with pytest.raises(ValueError, match="filename must not be empty"):
EtlRequest(file_path="/tmp/some.txt", filename="")
# ---------------------------------------------------------------------------
# Slice 12 – unknown ETL_SERVICE raises EtlServiceUnavailableError
# Slice 12 - unknown ETL_SERVICE raises EtlServiceUnavailableError
# ---------------------------------------------------------------------------
@ -310,7 +308,7 @@ async def test_unknown_etl_service_raises(tmp_path, mocker):
# ---------------------------------------------------------------------------
# Slice 13 – unsupported file types are rejected before reaching any parser
# Slice 13 - unsupported file types are rejected before reaching any parser
# ---------------------------------------------------------------------------
@ -321,10 +319,19 @@ def test_unknown_extension_classified_as_unsupported():
assert classify_file("random.xyz") == FileCategory.UNSUPPORTED
@pytest.mark.parametrize("filename", [
"malware.exe", "archive.zip", "video.mov", "font.woff2",
"model.blend", "data.parquet", "package.deb", "firmware.bin",
])
@pytest.mark.parametrize(
"filename",
[
"malware.exe",
"archive.zip",
"video.mov",
"font.woff2",
"model.blend",
"data.parquet",
"package.deb",
"firmware.bin",
],
)
def test_unsupported_extensions_classified_correctly(filename):
"""Extensions not in any allowlist are classified as UNSUPPORTED."""
from app.etl_pipeline.file_classifier import FileCategory, classify_file
@ -332,18 +339,21 @@ def test_unsupported_extensions_classified_correctly(filename):
assert classify_file(filename) == FileCategory.UNSUPPORTED
@pytest.mark.parametrize("filename,expected", [
("report.pdf", "document"),
("doc.docx", "document"),
("slides.pptx", "document"),
("sheet.xlsx", "document"),
("photo.png", "document"),
("photo.jpg", "document"),
("book.epub", "document"),
("letter.odt", "document"),
("readme.md", "plaintext"),
("data.csv", "direct_convert"),
])
@pytest.mark.parametrize(
"filename,expected",
[
("report.pdf", "document"),
("doc.docx", "document"),
("slides.pptx", "document"),
("sheet.xlsx", "document"),
("photo.png", "document"),
("photo.jpg", "document"),
("book.epub", "document"),
("letter.odt", "document"),
("readme.md", "plaintext"),
("data.csv", "direct_convert"),
],
)
def test_parseable_extensions_classified_correctly(filename, expected):
"""Parseable files are classified into their correct category."""
from app.etl_pipeline.file_classifier import FileCategory, classify_file
@ -380,31 +390,34 @@ async def test_extract_zip_raises_unsupported_error(tmp_path):
# ---------------------------------------------------------------------------
# Slice 14 – should_skip_for_service (per-parser document filtering)
# Slice 14 - should_skip_for_service (per-parser document filtering)
# ---------------------------------------------------------------------------
@pytest.mark.parametrize("filename,etl_service,expected_skip", [
("file.eml", "DOCLING", True),
("file.eml", "UNSTRUCTURED", False),
("file.docm", "LLAMACLOUD", False),
("file.docm", "DOCLING", True),
("file.txt", "DOCLING", False),
("file.csv", "LLAMACLOUD", False),
("file.mp3", "UNSTRUCTURED", False),
("file.exe", "LLAMACLOUD", True),
("file.pdf", "DOCLING", False),
("file.webp", "DOCLING", False),
("file.webp", "UNSTRUCTURED", True),
("file.gif", "LLAMACLOUD", False),
("file.gif", "DOCLING", True),
("file.heic", "UNSTRUCTURED", False),
("file.heic", "DOCLING", True),
("file.svg", "LLAMACLOUD", False),
("file.svg", "DOCLING", True),
("file.p7s", "UNSTRUCTURED", False),
("file.p7s", "LLAMACLOUD", True),
])
@pytest.mark.parametrize(
"filename,etl_service,expected_skip",
[
("file.eml", "DOCLING", True),
("file.eml", "UNSTRUCTURED", False),
("file.docm", "LLAMACLOUD", False),
("file.docm", "DOCLING", True),
("file.txt", "DOCLING", False),
("file.csv", "LLAMACLOUD", False),
("file.mp3", "UNSTRUCTURED", False),
("file.exe", "LLAMACLOUD", True),
("file.pdf", "DOCLING", False),
("file.webp", "DOCLING", False),
("file.webp", "UNSTRUCTURED", True),
("file.gif", "LLAMACLOUD", False),
("file.gif", "DOCLING", True),
("file.heic", "UNSTRUCTURED", False),
("file.heic", "DOCLING", True),
("file.svg", "LLAMACLOUD", False),
("file.svg", "DOCLING", True),
("file.p7s", "UNSTRUCTURED", False),
("file.p7s", "LLAMACLOUD", True),
],
)
def test_should_skip_for_service(filename, etl_service, expected_skip):
from app.etl_pipeline.file_classifier import should_skip_for_service
@ -414,7 +427,7 @@ def test_should_skip_for_service(filename, etl_service, expected_skip):
# ---------------------------------------------------------------------------
# Slice 14b – ETL pipeline rejects per-parser incompatible documents
# Slice 14b - ETL pipeline rejects per-parser incompatible documents
# ---------------------------------------------------------------------------

41
surfsense_backend/tests/unit/services/test_docling_image_support.py
View file

@ -30,26 +30,29 @@ def test_docling_service_does_not_restrict_allowed_formats():
fake_pdf_format_option_cls = MagicMock()
with patch.dict("sys.modules", {
"docling": MagicMock(),
"docling.backend": MagicMock(),
"docling.backend.pypdfium2_backend": MagicMock(
PyPdfiumDocumentBackend=mock_backend
),
"docling.datamodel": MagicMock(),
"docling.datamodel.base_models": MagicMock(
InputFormat=_FakeInputFormat
),
"docling.datamodel.pipeline_options": MagicMock(
PdfPipelineOptions=fake_pipeline_options_cls
),
"docling.document_converter": MagicMock(
DocumentConverter=mock_converter_cls,
PdfFormatOption=fake_pdf_format_option_cls,
),
}):
import app.services.docling_service as mod
with patch.dict(
"sys.modules",
{
"docling": MagicMock(),
"docling.backend": MagicMock(),
"docling.backend.pypdfium2_backend": MagicMock(
PyPdfiumDocumentBackend=mock_backend
),
"docling.datamodel": MagicMock(),
"docling.datamodel.base_models": MagicMock(InputFormat=_FakeInputFormat),
"docling.datamodel.pipeline_options": MagicMock(
PdfPipelineOptions=fake_pipeline_options_cls
),
"docling.document_converter": MagicMock(
DocumentConverter=mock_converter_cls,
PdfFormatOption=fake_pdf_format_option_cls,
),
},
):
from importlib import reload
import app.services.docling_service as mod
reload(mod)
mod.DoclingService()

84
surfsense_backend/tests/unit/utils/test_file_extensions.py
View file

@ -17,36 +17,74 @@ def test_exe_is_not_supported_document():
assert is_supported_document_extension("malware.exe") is False
@pytest.mark.parametrize("filename", [
"report.pdf", "doc.docx", "old.doc",
"sheet.xlsx", "legacy.xls",
"slides.pptx", "deck.ppt",
"macro.docm", "macro.xlsm", "macro.pptm",
"photo.png", "photo.jpg", "photo.jpeg", "scan.bmp", "scan.tiff", "scan.tif",
"photo.webp", "anim.gif", "iphone.heic",
"manual.rtf", "book.epub",
"letter.odt", "data.ods", "presentation.odp",
"inbox.eml", "outlook.msg",
"korean.hwpx", "korean.hwp",
"template.dot", "template.dotm",
"template.pot", "template.potx",
"binary.xlsb", "workspace.xlw",
"vector.svg", "signature.p7s",
])
@pytest.mark.parametrize(
"filename",
[
"report.pdf",
"doc.docx",
"old.doc",
"sheet.xlsx",
"legacy.xls",
"slides.pptx",
"deck.ppt",
"macro.docm",
"macro.xlsm",
"macro.pptm",
"photo.png",
"photo.jpg",
"photo.jpeg",
"scan.bmp",
"scan.tiff",
"scan.tif",
"photo.webp",
"anim.gif",
"iphone.heic",
"manual.rtf",
"book.epub",
"letter.odt",
"data.ods",
"presentation.odp",
"inbox.eml",
"outlook.msg",
"korean.hwpx",
"korean.hwp",
"template.dot",
"template.dotm",
"template.pot",
"template.potx",
"binary.xlsb",
"workspace.xlw",
"vector.svg",
"signature.p7s",
],
)
def test_document_extensions_are_supported(filename):
from app.utils.file_extensions import is_supported_document_extension
assert is_supported_document_extension(filename) is True, f"{filename} should be supported"
assert is_supported_document_extension(filename) is True, (
f"{filename} should be supported"
)
@pytest.mark.parametrize("filename", [
"malware.exe", "archive.zip", "video.mov", "font.woff2",
"model.blend", "random.xyz", "data.parquet", "package.deb",
])
@pytest.mark.parametrize(
"filename",
[
"malware.exe",
"archive.zip",
"video.mov",
"font.woff2",
"model.blend",
"random.xyz",
"data.parquet",
"package.deb",
],
)
def test_non_document_extensions_are_not_supported(filename):
from app.utils.file_extensions import is_supported_document_extension
assert is_supported_document_extension(filename) is False, f"{filename} should NOT be supported"
assert is_supported_document_extension(filename) is False, (
f"{filename} should NOT be supported"
)
# ---------------------------------------------------------------------------
@ -67,7 +105,7 @@ def test_union_equals_all_three_sets():
| LLAMAPARSE_DOCUMENT_EXTENSIONS
| UNSTRUCTURED_DOCUMENT_EXTENSIONS
)
assert DOCUMENT_EXTENSIONS == expected
assert expected == DOCUMENT_EXTENSIONS
def test_get_extensions_for_docling():

8
surfsense_web/app/dashboard/[search_space_id]/user-settings/components/DesktopContent.tsx
View file

@ -3,8 +3,8 @@
import { useEffect, useState } from "react";
import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";
import { Label } from "@/components/ui/label";
import { Switch } from "@/components/ui/switch";
import { Spinner } from "@/components/ui/spinner";
import { Switch } from "@/components/ui/switch";
export function DesktopContent() {
const [isElectron, setIsElectron] = useState(false);
@ -66,11 +66,7 @@ export function DesktopContent() {
Show suggestions while typing in other applications.
</p>
</div>
<Switch
id="autocomplete-toggle"
checked={enabled}
onCheckedChange={handleToggle}
/>
<Switch id="autocomplete-toggle" checked={enabled} onCheckedChange={handleToggle} />
</div>
</CardContent>
</Card>

15
surfsense_web/app/desktop/permissions/page.tsx
View file

@ -1,7 +1,7 @@
"use client";
import { useEffect, useState } from "react";
import { useRouter } from "next/navigation";
import { useEffect, useState } from "react";
import { Logo } from "@/components/Logo";
import { Button } from "@/components/ui/button";
import { Spinner } from "@/components/ui/spinner";
@ -17,7 +17,8 @@ const STEPS = [
{
id: "screen-recording",
title: "Screen Recording",
description: "Lets SurfSense capture your screen to understand context and provide smart writing suggestions.",
description:
"Lets SurfSense capture your screen to understand context and provide smart writing suggestions.",
action: "requestScreenRecording",
field: "screenRecording" as const,
},
@ -79,7 +80,9 @@ export default function DesktopPermissionsPage() {
poll();
interval = setInterval(poll, 2000);
return () => { if (interval) clearInterval(interval); };
return () => {
if (interval) clearInterval(interval);
};
}, []);
if (!isElectron) {
@ -98,7 +101,8 @@ export default function DesktopPermissionsPage() {
);
}
const allGranted = permissions.accessibility === "authorized" && permissions.screenRecording === "authorized";
const allGranted =
permissions.accessibility === "authorized" && permissions.screenRecording === "authorized";
const handleRequest = async (action: string) => {
if (action === "requestScreenRecording") {
@ -175,7 +179,8 @@ export default function DesktopPermissionsPage() {
</p>
)}
<p className="text-xs text-muted-foreground">
If SurfSense doesn&apos;t appear in the list, click <strong>+</strong> and select it from Applications.
If SurfSense doesn&apos;t appear in the list, click <strong>+</strong> and
select it from Applications.
</p>
</div>
)}

6
surfsense_web/app/desktop/suggestion/layout.tsx
View file

@ -4,10 +4,6 @@ export const metadata = {
title: "SurfSense Suggestion",
};
export default function SuggestionLayout({
children,
}: {
children: React.ReactNode;
}) {
export default function SuggestionLayout({ children }: { children: React.ReactNode }) {
return <div className="suggestion-body">{children}</div>;
}

50
surfsense_web/app/desktop/suggestion/page.tsx
View file

@ -72,27 +72,23 @@ export default function SuggestionPage() {
return;
}
const backendUrl =
process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL || "http://localhost:8000";
const backendUrl = process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL || "http://localhost:8000";
try {
const response = await fetch(
`${backendUrl}/api/v1/autocomplete/vision/stream`,
{
method: "POST",
headers: {
Authorization: `Bearer ${token}`,
"Content-Type": "application/json",
},
body: JSON.stringify({
screenshot,
search_space_id: parseInt(searchSpaceId, 10),
app_name: appName || "",
window_title: windowTitle || "",
}),
signal: controller.signal,
const response = await fetch(`${backendUrl}/api/v1/autocomplete/vision/stream`, {
method: "POST",
headers: {
Authorization: `Bearer ${token}`,
"Content-Type": "application/json",
},
);
body: JSON.stringify({
screenshot,
search_space_id: parseInt(searchSpaceId, 10),
app_name: appName || "",
window_title: windowTitle || "",
}),
signal: controller.signal,
});
if (!response.ok) {
setError(friendlyError(response.status));
@ -132,9 +128,7 @@ export default function SuggestionPage() {
} else if (parsed.type === "error") {
setError(friendlyError(parsed.errorText));
}
} catch {
continue;
}
} catch {}
}
}
}
@ -145,7 +139,7 @@ export default function SuggestionPage() {
setIsLoading(false);
}
},
[],
[]
);
useEffect(() => {
@ -207,10 +201,18 @@ export default function SuggestionPage() {
<div className="suggestion-tooltip">
<p className="suggestion-text">{suggestion}</p>
<div className="suggestion-actions">
<button className="suggestion-btn suggestion-btn-accept" onClick={handleAccept}>
<button
type="button"
className="suggestion-btn suggestion-btn-accept"
onClick={handleAccept}
>
Accept
</button>
<button className="suggestion-btn suggestion-btn-dismiss" onClick={handleDismiss}>
<button
type="button"
className="suggestion-btn suggestion-btn-dismiss"
onClick={handleDismiss}
>
Dismiss
</button>
</div>

144
surfsense_web/app/desktop/suggestion/suggestion.css
View file

@ -1,121 +1,125 @@
html:has(.suggestion-body),
body:has(.suggestion-body) {
margin: 0 !important;
padding: 0 !important;
background: transparent !important;
overflow: hidden !important;
height: auto !important;
width: 100% !important;
margin: 0 !important;
padding: 0 !important;
background: transparent !important;
overflow: hidden !important;
height: auto !important;
width: 100% !important;
}
.suggestion-body {
margin: 0;
padding: 0;
background: transparent;
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
-webkit-font-smoothing: antialiased;
user-select: none;
-webkit-app-region: no-drag;
margin: 0;
padding: 0;
background: transparent;
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
-webkit-font-smoothing: antialiased;
user-select: none;
-webkit-app-region: no-drag;
}
.suggestion-tooltip {
background: #1e1e1e;
border: 1px solid #3c3c3c;
border-radius: 8px;
padding: 8px 12px;
margin: 4px;
max-width: 400px;
box-shadow: 0 4px 16px rgba(0, 0, 0, 0.5);
background: #1e1e1e;
border: 1px solid #3c3c3c;
border-radius: 8px;
padding: 8px 12px;
margin: 4px;
max-width: 400px;
box-shadow: 0 4px 16px rgba(0, 0, 0, 0.5);
}
.suggestion-text {
color: #d4d4d4;
font-size: 13px;
line-height: 1.45;
margin: 0 0 6px 0;
word-wrap: break-word;
white-space: pre-wrap;
color: #d4d4d4;
font-size: 13px;
line-height: 1.45;
margin: 0 0 6px 0;
word-wrap: break-word;
white-space: pre-wrap;
}
.suggestion-actions {
display: flex;
justify-content: flex-end;
gap: 4px;
border-top: 1px solid #2a2a2a;
padding-top: 6px;
display: flex;
justify-content: flex-end;
gap: 4px;
border-top: 1px solid #2a2a2a;
padding-top: 6px;
}
.suggestion-btn {
padding: 2px 8px;
border-radius: 3px;
border: 1px solid #3c3c3c;
font-family: inherit;
font-size: 10px;
font-weight: 500;
cursor: pointer;
line-height: 16px;
transition: background 0.15s, border-color 0.15s;
padding: 2px 8px;
border-radius: 3px;
border: 1px solid #3c3c3c;
font-family: inherit;
font-size: 10px;
font-weight: 500;
cursor: pointer;
line-height: 16px;
transition:
background 0.15s,
border-color 0.15s;
}
.suggestion-btn-accept {
background: #2563eb;
border-color: #3b82f6;
color: #fff;
background: #2563eb;
border-color: #3b82f6;
color: #fff;
}
.suggestion-btn-accept:hover {
background: #1d4ed8;
background: #1d4ed8;
}
.suggestion-btn-dismiss {
background: #2a2a2a;
color: #999;
background: #2a2a2a;
color: #999;
}
.suggestion-btn-dismiss:hover {
background: #333;
color: #ccc;
background: #333;
color: #ccc;
}
.suggestion-error {
border-color: #5c2626;
border-color: #5c2626;
}
.suggestion-error-text {
color: #f48771;
font-size: 12px;
color: #f48771;
font-size: 12px;
}
.suggestion-loading {
display: flex;
gap: 5px;
padding: 2px 0;
justify-content: center;
display: flex;
gap: 5px;
padding: 2px 0;
justify-content: center;
}
.suggestion-dot {
width: 4px;
height: 4px;
border-radius: 50%;
background: #666;
animation: suggestion-pulse 1.2s infinite ease-in-out;
width: 4px;
height: 4px;
border-radius: 50%;
background: #666;
animation: suggestion-pulse 1.2s infinite ease-in-out;
}
.suggestion-dot:nth-child(2) {
animation-delay: 0.15s;
animation-delay: 0.15s;
}
.suggestion-dot:nth-child(3) {
animation-delay: 0.3s;
animation-delay: 0.3s;
}
@keyframes suggestion-pulse {
0%, 80%, 100% {
opacity: 0.3;
transform: scale(0.8);
}
40% {
opacity: 1;
transform: scale(1.1);
}
0%,
80%,
100% {
opacity: 0.3;
transform: scale(0.8);
}
40% {
opacity: 1;
transform: scale(1.1);
}
}

4
surfsense_web/components/assistant-ui/connector-popup/views/connector-accounts-list-view.tsx
View file

@ -173,9 +173,7 @@ export const ConnectorAccountsListView: FC<ConnectorAccountsListViewProps> = ({
<Plus className="size-3 text-primary" />
)}
</div>
<span className="text-xs sm:text-sm font-medium">
{buttonText}
</span>
<span className="text-xs sm:text-sm font-medium">{buttonText}</span>
</button>
</div>
</div>

4
surfsense_web/components/assistant-ui/connector-popup/views/youtube-crawler-view.tsx
View file

@ -337,9 +337,7 @@ export const YouTubeCrawlerView: FC<YouTubeCrawlerViewProps> = ({ searchSpaceId,
disabled={isSubmitting || isFetchingPlaylist || videoTags.length === 0}
className="relative text-xs sm:text-sm min-w-[140px] disabled:opacity-50 disabled:cursor-not-allowed disabled:pointer-events-none"
>
<span className={isSubmitting ? "opacity-0" : ""}>
{t("submit")}
</span>
<span className={isSubmitting ? "opacity-0" : ""}>{t("submit")}</span>
{isSubmitting && <Spinner size="sm" className="absolute" />}
</Button>
</div>

4
surfsense_web/components/assistant-ui/document-upload-popup.tsx
View file

@ -132,9 +132,7 @@ const DocumentUploadPopupContent: FC<{
<div className="flex-1 min-h-0 overflow-y-auto overscroll-contain">
<div className="sticky top-0 z-20 bg-muted px-4 sm:px-6 pt-6 sm:pt-8 pb-10">
<div className="flex items-center gap-2 mb-1 pr-8 sm:pr-0">
<h2 className="text-xl sm:text-3xl font-semibold tracking-tight">
Upload Documents
</h2>
<h2 className="text-xl sm:text-3xl font-semibold tracking-tight">Upload Documents</h2>
</div>
<p className="text-xs sm:text-base text-muted-foreground/80 line-clamp-1">
Upload and sync your documents to your search space

134
surfsense_web/components/assistant-ui/image.tsx
View file

@ -3,10 +3,10 @@
import type { ImageMessagePartComponent } from "@assistant-ui/react";
import { cva, type VariantProps } from "class-variance-authority";
import { ImageIcon, ImageOffIcon } from "lucide-react";
import NextImage from "next/image";
import { memo, type PropsWithChildren, useEffect, useRef, useState } from "react";
import { createPortal } from "react-dom";
import { cn } from "@/lib/utils";
import NextImage from 'next/image';
const imageVariants = cva("aui-image-root relative overflow-hidden rounded-lg", {
variants: {
@ -88,23 +88,23 @@ function ImagePreview({
<ImageOffIcon className="size-8 text-muted-foreground" />
</div>
) : isDataOrBlobUrl(src) ? (
// biome-ignore lint/performance/noImgElement: data/blob URLs need plain img
<img
ref={imgRef}
src={src}
alt={alt}
className={cn("block h-auto w-full object-contain", !loaded && "invisible", className)}
onLoad={(e) => {
if (typeof src === "string") setLoadedSrc(src);
onLoad?.(e);
}}
onError={(e) => {
if (typeof src === "string") setErrorSrc(src);
onError?.(e);
}}
{...props}
/>
) : (
// biome-ignore lint/performance/noImgElement: data/blob URLs need plain img
<img
ref={imgRef}
src={src}
alt={alt}
className={cn("block h-auto w-full object-contain", !loaded && "invisible", className)}
onLoad={(e) => {
if (typeof src === "string") setLoadedSrc(src);
onLoad?.(e);
}}
onError={(e) => {
if (typeof src === "string") setErrorSrc(src);
onError?.(e);
}}
{...props}
/>
) : (
// biome-ignore lint/performance/noImgElement: intentional for dynamic external URLs
// <img
// ref={imgRef}
@ -122,22 +122,22 @@ function ImagePreview({
// {...props}
// />
<NextImage
fill
src={src || ""}
alt={alt}
sizes="(max-width: 768px) 100vw, (max-width: 1200px) 80vw, 60vw"
className={cn("block object-contain", !loaded && "invisible", className)}
onLoad={() => {
if (typeof src === "string") setLoadedSrc(src);
onLoad?.();
}}
onError={() => {
if (typeof src === "string") setErrorSrc(src);
onError?.();
}}
unoptimized={false}
{...props}
/>
fill
src={src || ""}
alt={alt}
sizes="(max-width: 768px) 100vw, (max-width: 1200px) 80vw, 60vw"
className={cn("block object-contain", !loaded && "invisible", className)}
onLoad={() => {
if (typeof src === "string") setLoadedSrc(src);
onLoad?.();
}}
onError={() => {
if (typeof src === "string") setErrorSrc(src);
onError?.();
}}
unoptimized={false}
{...props}
/>
)}
</div>
);
@ -162,8 +162,8 @@ type ImageZoomProps = PropsWithChildren<{
alt?: string;
}>;
function isDataOrBlobUrl(src: string | undefined): boolean {
if (!src || typeof src !== "string") return false;
return src.startsWith("data:") || src.startsWith("blob:");
if (!src || typeof src !== "string") return false;
return src.startsWith("data:") || src.startsWith("blob:");
}
function ImageZoom({ src, alt = "Image preview", children }: ImageZoomProps) {
const [isMounted, setIsMounted] = useState(false);
@ -216,38 +216,38 @@ function ImageZoom({ src, alt = "Image preview", children }: ImageZoomProps) {
>
{/** biome-ignore lint/performance/noImgElement: <explanation> */}
{isDataOrBlobUrl(src) ? (
// biome-ignore lint/performance/noImgElement: data/blob URLs need plain img
<img
data-slot="image-zoom-content"
src={src}
alt={alt}
className="aui-image-zoom-content fade-in zoom-in-95 max-h-[90vh] max-w-[90vw] animate-in object-contain duration-200"
onClick={(e) => {
e.stopPropagation();
handleClose();
}}
onKeyDown={(e) => {
if (e.key === "Enter") {
e.stopPropagation();
handleClose();
}
}}
/>
) : (
// biome-ignore lint/performance/noImgElement: data/blob URLs need plain img
<img
data-slot="image-zoom-content"
src={src}
alt={alt}
className="aui-image-zoom-content fade-in zoom-in-95 max-h-[90vh] max-w-[90vw] animate-in object-contain duration-200"
onClick={(e) => {
e.stopPropagation();
handleClose();
}}
onKeyDown={(e) => {
if (e.key === "Enter") {
e.stopPropagation();
handleClose();
}
}}
/>
) : (
<NextImage
data-slot="image-zoom-content"
fill
src={src}
alt={alt}
sizes="90vw"
className="aui-image-zoom-content fade-in zoom-in-95 object-contain duration-200"
onClick={(e) => {
e.stopPropagation();
handleClose();
}}
unoptimized={false}
/>
)}
data-slot="image-zoom-content"
fill
src={src}
alt={alt}
sizes="90vw"
className="aui-image-zoom-content fade-in zoom-in-95 object-contain duration-200"
onClick={(e) => {
e.stopPropagation();
handleClose();
}}
unoptimized={false}
/>
)}
</button>,
document.body
)}

4
surfsense_web/components/assistant-ui/thread-list.tsx
View file

@ -241,9 +241,7 @@ const ThreadListItemComponent = memo(function ThreadListItemComponent({
<MessageSquareIcon className="size-4 shrink-0 text-muted-foreground" />
<div className="flex-1 min-w-0">
<p className="truncate text-sm font-medium">{thread.title || "New Chat"}</p>
<p className="truncate text-xs text-muted-foreground">
{relativeTime}
</p>
<p className="truncate text-xs text-muted-foreground">{relativeTime}</p>
</div>
<DropdownMenu>
<DropdownMenuTrigger asChild>

3
surfsense_web/components/assistant-ui/tool-fallback.tsx
View file

@ -26,7 +26,8 @@ export const ToolFallback: ToolCallMessagePartComponent = ({
);
const serializedResult = useMemo(
() => (result !== undefined && typeof result !== "string" ? JSON.stringify(result, null, 2) : null),
() =>
result !== undefined && typeof result !== "string" ? JSON.stringify(result, null, 2) : null,
[result]
);

24
surfsense_web/components/chat-comments/comment-composer/comment-composer.tsx
View file

@ -300,15 +300,15 @@ export function CommentComposer({
<div className={cn("flex items-center gap-2", !compact && "justify-end")}>
{onCancel && (
<Button
type="button"
variant="ghost"
size="sm"
onClick={onCancel}
disabled={isSubmitting}
>
Cancel
</Button>
<Button
type="button"
variant="ghost"
size="sm"
onClick={onCancel}
disabled={isSubmitting}
>
Cancel
</Button>
)}
<Button
type="button"
@ -317,11 +317,7 @@ export function CommentComposer({
disabled={!canSubmit}
className={cn(!canSubmit && "opacity-50", compact && "size-8 shrink-0 rounded-full")}
>
{compact ? (
<ArrowUp className="size-4" />
) : (
submitLabel
)}
{compact ? <ArrowUp className="size-4" /> : submitLabel}
</Button>
</div>
</div>

20
surfsense_web/components/documents/DocumentNode.tsx
View file

@ -207,9 +207,15 @@ export const DocumentNode = React.memo(function DocumentNode({
);
})()}
<Tooltip delayDuration={600} open={titleTooltipOpen} onOpenChange={handleTitleTooltipOpenChange}>
<Tooltip
delayDuration={600}
open={titleTooltipOpen}
onOpenChange={handleTitleTooltipOpenChange}
>
<TooltipTrigger asChild>
<span ref={titleRef} className="flex-1 min-w-0 truncate">{doc.title}</span>
<span ref={titleRef} className="flex-1 min-w-0 truncate">
{doc.title}
</span>
</TooltipTrigger>
<TooltipContent side="bottom" className="max-w-xs break-words">
{doc.title}
@ -276,10 +282,7 @@ export const DocumentNode = React.memo(function DocumentNode({
Versions
</DropdownMenuItem>
)}
<DropdownMenuItem
disabled={isProcessing}
onClick={() => onDelete(doc)}
>
<DropdownMenuItem disabled={isProcessing} onClick={() => onDelete(doc)}>
<Trash2 className="mr-2 h-4 w-4" />
Delete
</DropdownMenuItem>
@ -321,10 +324,7 @@ export const DocumentNode = React.memo(function DocumentNode({
Versions
</ContextMenuItem>
)}
<ContextMenuItem
disabled={isProcessing}
onClick={() => onDelete(doc)}
>
<ContextMenuItem disabled={isProcessing} onClick={() => onDelete(doc)}>
<Trash2 className="mr-2 h-4 w-4" />
Delete
</ContextMenuItem>

15
surfsense_web/components/documents/FolderTreeView.tsx
View file

@ -97,7 +97,10 @@ export function FolderTreeView({
const handleCancelRename = useCallback(() => setRenamingFolderId(null), [setRenamingFolderId]);
const effectiveActiveTypes = useMemo(() => {
if (activeTypes.includes("FILE" as DocumentTypeEnum) && !activeTypes.includes("LOCAL_FOLDER_FILE" as DocumentTypeEnum)) {
if (
activeTypes.includes("FILE" as DocumentTypeEnum) &&
!activeTypes.includes("LOCAL_FOLDER_FILE" as DocumentTypeEnum)
) {
return [...activeTypes, "LOCAL_FOLDER_FILE" as DocumentTypeEnum];
}
return activeTypes;
@ -110,7 +113,9 @@ export function FolderTreeView({
function check(folderId: number): boolean {
if (match[folderId] !== undefined) return match[folderId];
const childDocs = (docsByFolder[folderId] ?? []).some(
(d) => effectiveActiveTypes.length === 0 || effectiveActiveTypes.includes(d.document_type as DocumentTypeEnum)
(d) =>
effectiveActiveTypes.length === 0 ||
effectiveActiveTypes.includes(d.document_type as DocumentTypeEnum)
);
if (childDocs) {
match[folderId] = true;
@ -201,7 +206,9 @@ export function FolderTreeView({
? childFolders.filter((f) => hasDescendantMatch[f.id])
: childFolders;
const childDocs = (docsByFolder[key] ?? []).filter(
(d) => effectiveActiveTypes.length === 0 || effectiveActiveTypes.includes(d.document_type as DocumentTypeEnum)
(d) =>
effectiveActiveTypes.length === 0 ||
effectiveActiveTypes.includes(d.document_type as DocumentTypeEnum)
);
const nodes: React.ReactNode[] = [];
@ -223,7 +230,7 @@ export function FolderTreeView({
depth={depth}
isExpanded={isExpanded}
isRenaming={renamingFolderId === f.id}
selectionState={folderSelectionStates[f.id] ?? "none"}
selectionState={folderSelectionStates[f.id] ?? "none"}
processingState={folderProcessingStates[f.id] ?? "idle"}
onToggleSelect={onToggleFolderSelect}
onToggleExpand={onToggleExpand}

19
surfsense_web/components/editor/plate-editor.tsx
View file

@ -158,17 +158,18 @@ export function PlateEditor({
// When not forced read-only, the user can toggle between editing/viewing.
const canToggleMode = !readOnly;
const contextProviderValue = useMemo(()=> ({
onSave,
hasUnsavedChanges,
isSaving,
canToggleMode,
}), [onSave, hasUnsavedChanges, isSaving, canToggleMode]);
const contextProviderValue = useMemo(
() => ({
onSave,
hasUnsavedChanges,
isSaving,
canToggleMode,
}),
[onSave, hasUnsavedChanges, isSaving, canToggleMode]
);
return (
<EditorSaveContext.Provider
value={contextProviderValue}
>
<EditorSaveContext.Provider value={contextProviderValue}>
<Plate
editor={editor}
// Only pass readOnly as a controlled prop when forced (permanently read-only).

16
surfsense_web/components/homepage/use-cases-grid.tsx
View file

@ -1,7 +1,7 @@
"use client";
import Image from 'next/image';
import { AnimatePresence, motion } from "motion/react";
import Image from "next/image";
import { ExpandedGifOverlay, useExpandedGif } from "@/components/ui/expanded-gif-overlay";
const useCases = [
@ -83,13 +83,13 @@ function UseCaseCard({
className="w-full rounded-xl object-cover transition-transform duration-500 group-hover:scale-[1.02]"
/>
<div className="relative w-full h-48">
<Image
src={src}
alt={title}
fill
className="rounded-xl object-cover transition-transform duration-500 group-hover:scale-[1.02]"
unoptimized={src.endsWith('.gif')}
/>
<Image
src={src}
alt={title}
fill
className="rounded-xl object-cover transition-transform duration-500 group-hover:scale-[1.02]"
unoptimized={src.endsWith(".gif")}
/>
</div>
</div>
<div className="px-5 py-4">

3
surfsense_web/components/layout/providers/LayoutDataProvider.tsx
View file

@ -370,7 +370,8 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid
url: "#announcements",
icon: Megaphone,
isActive: isAnnouncementsSidebarOpen,
badge: announcementUnreadCount > 0 ? formatInboxCount(announcementUnreadCount) : undefined,
badge:
announcementUnreadCount > 0 ? formatInboxCount(announcementUnreadCount) : undefined,
},
] as (NavItem | null)[]
).filter((item): item is NavItem => item !== null),

36
surfsense_web/components/layout/ui/sidebar/AllPrivateChatsSidebar.tsx
View file

@ -376,24 +376,24 @@ export function AllPrivateChatsSidebarContent({
<span className="truncate">{thread.title || "New Chat"}</span>
</button>
) : (
<Tooltip delayDuration={600}>
<TooltipTrigger asChild>
<button
type="button"
onClick={() => handleThreadClick(thread.id)}
disabled={isBusy}
className="flex items-center gap-2 flex-1 min-w-0 text-left overflow-hidden"
>
<span className="truncate">{thread.title || "New Chat"}</span>
</button>
</TooltipTrigger>
<TooltipContent side="bottom" align="start">
<p>
{t("updated") || "Updated"}:{" "}
{format(new Date(thread.updatedAt), "MMM d, yyyy 'at' h:mm a")}
</p>
</TooltipContent>
</Tooltip>
<Tooltip delayDuration={600}>
<TooltipTrigger asChild>
<button
type="button"
onClick={() => handleThreadClick(thread.id)}
disabled={isBusy}
className="flex items-center gap-2 flex-1 min-w-0 text-left overflow-hidden"
>
<span className="truncate">{thread.title || "New Chat"}</span>
</button>
</TooltipTrigger>
<TooltipContent side="bottom" align="start">
<p>
{t("updated") || "Updated"}:{" "}
{format(new Date(thread.updatedAt), "MMM d, yyyy 'at' h:mm a")}
</p>
</TooltipContent>
</Tooltip>
)}
<DropdownMenu

36
surfsense_web/components/layout/ui/sidebar/AllSharedChatsSidebar.tsx
View file

@ -375,24 +375,24 @@ export function AllSharedChatsSidebarContent({
<span className="truncate">{thread.title || "New Chat"}</span>
</button>
) : (
<Tooltip delayDuration={600}>
<TooltipTrigger asChild>
<button
type="button"
onClick={() => handleThreadClick(thread.id)}
disabled={isBusy}
className="flex items-center gap-2 flex-1 min-w-0 text-left overflow-hidden"
>
<span className="truncate">{thread.title || "New Chat"}</span>
</button>
</TooltipTrigger>
<TooltipContent side="bottom" align="start">
<p>
{t("updated") || "Updated"}:{" "}
{format(new Date(thread.updatedAt), "MMM d, yyyy 'at' h:mm a")}
</p>
</TooltipContent>
</Tooltip>
<Tooltip delayDuration={600}>
<TooltipTrigger asChild>
<button
type="button"
onClick={() => handleThreadClick(thread.id)}
disabled={isBusy}
className="flex items-center gap-2 flex-1 min-w-0 text-left overflow-hidden"
>
<span className="truncate">{thread.title || "New Chat"}</span>
</button>
</TooltipTrigger>
<TooltipContent side="bottom" align="start">
<p>
{t("updated") || "Updated"}:{" "}
{format(new Date(thread.updatedAt), "MMM d, yyyy 'at' h:mm a")}
</p>
</TooltipContent>
</Tooltip>
)}
<DropdownMenu

51
surfsense_web/components/markdown-viewer.tsx
View file

@ -2,9 +2,9 @@ import { createCodePlugin } from "@streamdown/code";
import { createMathPlugin } from "@streamdown/math";
import { Streamdown, type StreamdownProps } from "streamdown";
import "katex/dist/katex.min.css";
import { cn } from "@/lib/utils";
import Image from 'next/image';
import { is } from "drizzle-orm";
import Image from "next/image";
import { cn } from "@/lib/utils";
const code = createCodePlugin({
themes: ["nord", "nord"],
@ -130,30 +130,31 @@ export function MarkdownViewer({ content, className, maxLength }: MarkdownViewer
),
hr: ({ ...props }) => <hr className="my-4 border-muted" {...props} />,
img: ({ src, alt, width: _w, height: _h, ...props }) => {
const isDataOrUnknownUrl = typeof src === "string" && (src.startsWith("data:") || !src.startsWith("http"));
const isDataOrUnknownUrl =
typeof src === "string" && (src.startsWith("data:") || !src.startsWith("http"));
return isDataOrUnknownUrl ? (
// eslint-disable-next-line @next/next/no-img-element
<img
className="max-w-full h-auto my-4 rounded"
alt={alt || "markdown image"}
src={src}
loading="lazy"
{...props}
/>
) : (
<Image
className="max-w-full h-auto my-4 rounded"
alt={alt || "markdown image"}
src={typeof src === "string" ? src : ""}
width={_w || 800}
height={_h || 600}
sizes="(max-width: 768px) 100vw, (max-width: 1200px) 75vw, 60vw"
unoptimized={isDataOrUnknownUrl}
{...props}
/>
);
},
return isDataOrUnknownUrl ? (
// eslint-disable-next-line @next/next/no-img-element
<img
className="max-w-full h-auto my-4 rounded"
alt={alt || "markdown image"}
src={src}
loading="lazy"
{...props}
/>
) : (
<Image
className="max-w-full h-auto my-4 rounded"
alt={alt || "markdown image"}
src={typeof src === "string" ? src : ""}
width={_w || 800}
height={_h || 600}
sizes="(max-width: 768px) 100vw, (max-width: 1200px) 75vw, 60vw"
unoptimized={isDataOrUnknownUrl}
{...props}
/>
);
},
table: ({ ...props }) => (
<div className="overflow-x-auto my-4 rounded-lg border border-border w-full">
<table className="w-full divide-y divide-border" {...props} />

2
surfsense_web/components/settings/user-settings-dialog.tsx
View file

@ -5,10 +5,10 @@ import { Globe, KeyRound, Monitor, Receipt, Sparkles, User } from "lucide-react"
import { useTranslations } from "next-intl";
import { ApiKeyContent } from "@/app/dashboard/[search_space_id]/user-settings/components/ApiKeyContent";
import { CommunityPromptsContent } from "@/app/dashboard/[search_space_id]/user-settings/components/CommunityPromptsContent";
import { DesktopContent } from "@/app/dashboard/[search_space_id]/user-settings/components/DesktopContent";
import { ProfileContent } from "@/app/dashboard/[search_space_id]/user-settings/components/ProfileContent";
import { PromptsContent } from "@/app/dashboard/[search_space_id]/user-settings/components/PromptsContent";
import { PurchaseHistoryContent } from "@/app/dashboard/[search_space_id]/user-settings/components/PurchaseHistoryContent";
import { DesktopContent } from "@/app/dashboard/[search_space_id]/user-settings/components/DesktopContent";
import { userSettingsDialogAtom } from "@/atoms/settings/settings-dialog.atoms";
import { SettingsDialog } from "@/components/settings/settings-dialog";

49
surfsense_web/components/sources/DocumentUploadTab.tsx
View file

@ -471,13 +471,13 @@ export function DocumentUploadTab({
</button>
))
) : (
<button
type="button"
className="flex flex-col items-center gap-4 py-12 px-4 cursor-pointer w-full bg-transparent border-none"
onClick={() => {
if (!isElectron) fileInputRef.current?.click();
}}
>
<button
type="button"
className="flex flex-col items-center gap-4 py-12 px-4 cursor-pointer w-full bg-transparent border-none"
onClick={() => {
if (!isElectron) fileInputRef.current?.click();
}}
>
<Upload className="h-10 w-10 text-muted-foreground" />
<div className="text-center space-y-1.5">
<p className="text-base font-medium">
@ -485,10 +485,15 @@ export function DocumentUploadTab({
</p>
<p className="text-sm text-muted-foreground">{t("file_size_limit")}</p>
</div>
{/* biome-ignore lint/a11y/useSemanticElements: wrapper to stop click propagation to parent button */}
<div className="w-full mt-1" onClick={(e) => e.stopPropagation()} onKeyDown={(e) => e.stopPropagation()} role="group">
{renderBrowseButton({ fullWidth: true })}
</div>
{/* biome-ignore lint/a11y/useSemanticElements: wrapper to stop click propagation to parent button */}
<div
className="w-full mt-1"
onClick={(e) => e.stopPropagation()}
onKeyDown={(e) => e.stopPropagation()}
role="group"
>
{renderBrowseButton({ fullWidth: true })}
</div>
</button>
)}
</div>
@ -684,17 +689,17 @@ export function DocumentUploadTab({
</span>
</AccordionTrigger>
<AccordionContent className="px-3 pb-3">
<div className="flex flex-wrap gap-1.5">
{supportedExtensions.map((ext) => (
<Badge
key={ext}
variant="secondary"
className="rounded border-0 bg-neutral-200/80 dark:bg-neutral-700/60 text-muted-foreground text-[10px] px-2 py-0.5 font-normal"
>
{ext}
</Badge>
))}
</div>
<div className="flex flex-wrap gap-1.5">
{supportedExtensions.map((ext) => (
<Badge
key={ext}
variant="secondary"
className="rounded border-0 bg-neutral-200/80 dark:bg-neutral-700/60 text-muted-foreground text-[10px] px-2 py-0.5 font-normal"
>
{ext}
</Badge>
))}
</div>
</AccordionContent>
</AccordionItem>
</Accordion>

31
surfsense_web/components/tool-ui/citation/citation-list.tsx
View file

@ -2,13 +2,12 @@
import type { LucideIcon } from "lucide-react";
import { Code2, Database, ExternalLink, File, FileText, Globe, Newspaper } from "lucide-react";
import NextImage from "next/image";
import * as React from "react";
import { openSafeNavigationHref, resolveSafeNavigationHref } from "../shared/media";
import { cn, Popover, PopoverContent, PopoverTrigger } from "./_adapter";
import { Citation } from "./citation";
import type { CitationType, CitationVariant, SerializableCitation } from "./schema";
import NextImage from 'next/image';
const TYPE_ICONS: Record<CitationType, LucideIcon> = {
webpage: Globe,
@ -264,9 +263,9 @@ function OverflowItem({ citation, onClick }: OverflowItemProps) {
className="size-4.5 rounded-full object-cover"
unoptimized={true}
/>
) : (
) : (
<TypeIcon className="text-muted-foreground size-3" aria-hidden="true" />
)}
)}
<div className="min-w-0 flex-1">
<p className="group-hover:decoration-foreground/30 truncate text-sm font-medium group-hover:underline group-hover:underline-offset-2">
{citation.title}
@ -341,18 +340,18 @@ function StackedCitations({ id, citations, className, onNavigate }: StackedCitat
style={{ zIndex: maxIcons - index }}
>
{citation.favicon ? (
<NextImage
src={citation.favicon}
alt=""
aria-hidden="true"
width={18}
height={18}
className="size-4.5 rounded-full object-cover"
unoptimized={true}
/>
) : (
<TypeIcon className="text-muted-foreground size-3" aria-hidden="true" />
)}
<NextImage
src={citation.favicon}
alt=""
aria-hidden="true"
width={18}
height={18}
className="size-4.5 rounded-full object-cover"
unoptimized={true}
/>
) : (
<TypeIcon className="text-muted-foreground size-3" aria-hidden="true" />
)}
</div>
);
})}

26
surfsense_web/components/tool-ui/citation/citation.tsx
View file

@ -2,11 +2,11 @@
import type { LucideIcon } from "lucide-react";
import { Code2, Database, ExternalLink, File, FileText, Globe, Newspaper } from "lucide-react";
import NextImage from "next/image";
import * as React from "react";
import { openSafeNavigationHref, sanitizeHref } from "../shared/media";
import { cn, Popover, PopoverContent, PopoverTrigger } from "./_adapter";
import type { CitationType, CitationVariant, SerializableCitation } from "./schema";
import NextImage from 'next/image';
const FALLBACK_LOCALE = "en-US";
@ -115,18 +115,18 @@ export function Citation(props: CitationProps) {
};
const iconElement = favicon ? (
<NextImage
src={favicon}
alt=""
aria-hidden="true"
width={16}
height={16}
className="bg-muted size-3.5 shrink-0 rounded object-cover"
unoptimized={true}
/>
) : (
<TypeIcon className="size-3.5 shrink-0 opacity-60" aria-hidden="true" />
);
<NextImage
src={favicon}
alt=""
aria-hidden="true"
width={16}
height={16}
className="bg-muted size-3.5 shrink-0 rounded object-cover"
unoptimized={true}
/>
) : (
<TypeIcon className="size-3.5 shrink-0 opacity-60" aria-hidden="true" />
);
const { open, handleMouseEnter, handleMouseLeave } = useHoverPopover();

5
surfsense_web/components/ui/animated-tabs.tsx
View file

@ -202,7 +202,10 @@ const Tabs = forwardRef<
},
[onValueChange, value]
);
const contextValue = useMemo(() => ({ activeValue, onValueChange: handleValueChange }), [activeValue, handleValueChange]);
const contextValue = useMemo(
() => ({ activeValue, onValueChange: handleValueChange }),
[activeValue, handleValueChange]
);
return (
<TabsContext.Provider value={contextValue}>
<div ref={ref} className={cn("tabs-container", className)} {...props}>

10
surfsense_web/components/ui/toggle-group.tsx
View file

@ -3,9 +3,9 @@
import * as ToggleGroupPrimitive from "@radix-ui/react-toggle-group";
import type { VariantProps } from "class-variance-authority";
import * as React from "react";
import { useMemo } from "react";
import { toggleVariants } from "@/components/ui/toggle";
import { cn } from "@/lib/utils";
import { useMemo } from "react";
const ToggleGroupContext = React.createContext<
VariantProps<typeof toggleVariants> & {
@ -28,8 +28,8 @@ function ToggleGroup({
VariantProps<typeof toggleVariants> & {
spacing?: number;
}) {
const contextValue = useMemo(() => ({variant, size, spacing }), [variant, size, spacing]);
const contextValue = useMemo(() => ({ variant, size, spacing }), [variant, size, spacing]);
return (
<ToggleGroupPrimitive.Root
data-slot="toggle-group"
@ -43,9 +43,7 @@ function ToggleGroup({
)}
{...props}
>
<ToggleGroupContext.Provider value={contextValue}>
{children}
</ToggleGroupContext.Provider>
<ToggleGroupContext.Provider value={contextValue}>{children}</ToggleGroupContext.Provider>
</ToggleGroupPrimitive.Root>
);
}

13
surfsense_web/contexts/LocaleContext.tsx
View file

@ -2,12 +2,12 @@
import type React from "react";
import { createContext, useCallback, useContext, useEffect, useMemo, useState } from "react";
import { set } from "zod";
import enMessages from "../messages/en.json";
import esMessages from "../messages/es.json";
import hiMessages from "../messages/hi.json";
import ptMessages from "../messages/pt.json";
import zhMessages from "../messages/zh.json";
import { set } from "zod";
type Locale = "en" | "es" | "pt" | "hi" | "zh";
@ -66,13 +66,12 @@ export function LocaleProvider({ children }: { children: React.ReactNode }) {
}
}, [locale, mounted]);
const contextValue = useMemo(() => ({ locale, messages, setLocale }), [locale, messages, setLocale]);
return (
<LocaleContext.Provider value={contextValue}>
{children}
</LocaleContext.Provider>
const contextValue = useMemo(
() => ({ locale, messages, setLocale }),
[locale, messages, setLocale]
);
return <LocaleContext.Provider value={contextValue}>{children}</LocaleContext.Provider>;
}
export function useLocaleContext() {

13
surfsense_web/types/window.d.ts vendored
View file

@ -50,14 +50,21 @@ interface ElectronAPI {
replaceText: (text: string) => Promise<void>;
// Permissions
getPermissionsStatus: () => Promise<{
accessibility: 'authorized' | 'denied' | 'not determined' | 'restricted' | 'limited';
screenRecording: 'authorized' | 'denied' | 'not determined' | 'restricted' | 'limited';
accessibility: "authorized" | "denied" | "not determined" | "restricted" | "limited";
screenRecording: "authorized" | "denied" | "not determined" | "restricted" | "limited";
}>;
requestAccessibility: () => Promise<void>;
requestScreenRecording: () => Promise<void>;
restartApp: () => Promise<void>;
// Autocomplete
onAutocompleteContext: (callback: (data: { screenshot: string; searchSpaceId?: string; appName?: string; windowTitle?: string }) => void) => () => void;
onAutocompleteContext: (
callback: (data: {
screenshot: string;
searchSpaceId?: string;
appName?: string;
windowTitle?: string;
}) => void
) => () => void;
acceptSuggestion: (text: string) => Promise<void>;
dismissSuggestion: () => Promise<void>;
setAutocompleteEnabled: (enabled: boolean) => Promise<void>;