apunkt/SurfSense
1
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2026-07-06 22:12:12 +02:00
Code Issues Projects Releases Packages Wiki Activity

refactor(backend): rename search_space -> workspace across app bulk (Phase 2 Wave D)

Browse source 
Scoped codemod over surfsense_backend/app (excluding routes/, Wave E): renames
search_space_id -> workspace_id, search_space -> workspace, SearchSpace -> Workspace
across services, utils, tasks, agents, gateway, event_bus, notifications, podcasts,
automations, observability params, and prompt .md files. Also flips the camelCase
payload key searchSpaceId -> workspaceId (no backend reader; hard cutover).

Preserved carve-outs (verbatim): Celery task names "delete_search_space_background"
and "ai_sort_search_space" (wire names), and the OTel/metric key "search_space.id"
(dashboards depend on it). Enum values 'SEARCH_SPACE' and SearchSourceConnector
untouched.
This commit is contained in:
CREDO23 2026-06-26 18:30:47 +02:00
parent 0c53d884eb
commit 7fb0707933
259 changed files with 1996 additions and 1996 deletions
Download patch file Download diff file Expand all files Collapse all files

8
surfsense_backend/app/tasks/document_processors/_helpers.py
View file

@ -24,7 +24,7 @@ from .base import (
def get_google_drive_unique_identifier(
connector: dict | None,
filename: str,
search_space_id: int,
workspace_id: int,
) -> tuple[str, str | None]:
"""
Get unique identifier hash, using file_id for Google Drive (stable across renames).
@ -40,15 +40,15 @@ def get_google_drive_unique_identifier(
if file_id:
primary_hash = generate_unique_identifier_hash(
DocumentType.GOOGLE_DRIVE_FILE, file_id, search_space_id
DocumentType.GOOGLE_DRIVE_FILE, file_id, workspace_id
)
legacy_hash = generate_unique_identifier_hash(
DocumentType.GOOGLE_DRIVE_FILE, filename, search_space_id
DocumentType.GOOGLE_DRIVE_FILE, filename, workspace_id
)
return primary_hash, legacy_hash
primary_hash = generate_unique_identifier_hash(
DocumentType.FILE, filename, search_space_id
DocumentType.FILE, filename, workspace_id
)
return primary_hash, None

10
surfsense_backend/app/tasks/document_processors/_save.py
View file

@ -28,7 +28,7 @@ async def save_file_document(
session: AsyncSession,
file_name: str,
markdown_content: str,
search_space_id: int,
workspace_id: int,
user_id: str,
etl_service: str,
connector: dict | None = None,
@ -43,7 +43,7 @@ async def save_file_document(
session: Database session
file_name: Name of the processed file
markdown_content: Markdown content to store
search_space_id: ID of the search space
workspace_id: ID of the workspace
user_id: ID of the user
etl_service: Name of the ETL service (UNSTRUCTURED, LLAMACLOUD, DOCLING)
connector: Optional connector info for Google Drive files
@ -53,9 +53,9 @@ async def save_file_document(
"""
try:
primary_hash, legacy_hash = get_google_drive_unique_identifier(
connector, file_name, search_space_id
connector, file_name, workspace_id
)
content_hash = generate_content_hash(markdown_content, search_space_id)
content_hash = generate_content_hash(markdown_content, workspace_id)
existing_document = await find_existing_document_with_migration(
session, primary_hash, legacy_hash, content_hash
@ -99,7 +99,7 @@ async def save_file_document(
doc_type = DocumentType.GOOGLE_DRIVE_FILE
document = Document(
search_space_id=search_space_id,
workspace_id=workspace_id,
title=file_name,
document_type=doc_type,
document_metadata=doc_metadata,

32
surfsense_backend/app/tasks/document_processors/circleback_processor.py
View file

@ -23,7 +23,7 @@ from app.db import (
DocumentType,
SearchSourceConnector,
SearchSourceConnectorType,
SearchSpace,
Workspace,
)
from app.utils.document_converters import (
create_document_chunks,
@ -47,7 +47,7 @@ async def add_circleback_meeting_document(
meeting_name: str,
markdown_content: str,
metadata: dict[str, Any],
search_space_id: int,
workspace_id: int,
connector_id: int | None = None,
) -> Document | None:
"""
@ -64,7 +64,7 @@ async def add_circleback_meeting_document(
meeting_name: Name of the meeting
markdown_content: Meeting content formatted as markdown
metadata: Meeting metadata dictionary
search_space_id: ID of the search space
workspace_id: ID of the workspace
connector_id: ID of the Circleback connector (for deletion support)
Returns:
@ -75,11 +75,11 @@ async def add_circleback_meeting_document(
# Generate unique identifier hash using Circleback meeting ID
unique_identifier = f"circleback_{meeting_id}"
unique_identifier_hash = generate_unique_identifier_hash(
DocumentType.CIRCLEBACK, unique_identifier, search_space_id
DocumentType.CIRCLEBACK, unique_identifier, workspace_id
)
# Generate content hash
content_hash = generate_content_hash(markdown_content, search_space_id)
content_hash = generate_content_hash(markdown_content, workspace_id)
# Check if document with this unique identifier already exists
existing_document = await check_document_by_unique_identifier(
@ -113,13 +113,13 @@ async def add_circleback_meeting_document(
# =======================================================================
# Fetch the user who set up the Circleback connector (preferred)
# or fall back to search space owner if no connector found
# or fall back to workspace owner if no connector found
created_by_user_id = None
# Try to find the Circleback connector for this search space
# Try to find the Circleback connector for this workspace
connector_result = await session.execute(
select(SearchSourceConnector.user_id).where(
SearchSourceConnector.search_space_id == search_space_id,
SearchSourceConnector.workspace_id == workspace_id,
SearchSourceConnector.connector_type
== SearchSourceConnectorType.CIRCLEBACK_CONNECTOR,
)
@ -130,15 +130,15 @@ async def add_circleback_meeting_document(
# Use the user who set up the Circleback connector
created_by_user_id = connector_user
else:
# Fallback: use search space owner if no connector found
search_space_result = await session.execute(
select(SearchSpace.user_id).where(SearchSpace.id == search_space_id)
# Fallback: use workspace owner if no connector found
workspace_result = await session.execute(
select(Workspace.user_id).where(Workspace.id == workspace_id)
)
created_by_user_id = search_space_result.scalar_one_or_none()
created_by_user_id = workspace_result.scalar_one_or_none()
# Create new document with PENDING status (visible in UI immediately)
document = Document(
search_space_id=search_space_id,
workspace_id=workspace_id,
title=meeting_name,
document_type=DocumentType.CIRCLEBACK,
document_metadata={
@ -162,7 +162,7 @@ async def add_circleback_meeting_document(
# Commit immediately so document appears in UI with pending status
await session.commit()
logger.info(
f"Created pending Circleback meeting document {meeting_id} in search space {search_space_id}"
f"Created pending Circleback meeting document {meeting_id} in workspace {workspace_id}"
)
# =======================================================================
@ -213,11 +213,11 @@ async def add_circleback_meeting_document(
if existing_document:
logger.info(
f"Updated Circleback meeting document {meeting_id} in search space {search_space_id}"
f"Updated Circleback meeting document {meeting_id} in workspace {workspace_id}"
)
else:
logger.info(
f"Processed Circleback meeting document {meeting_id} in search space {search_space_id} - now ready"
f"Processed Circleback meeting document {meeting_id} in workspace {workspace_id} - now ready"
)
return document

12
surfsense_backend/app/tasks/document_processors/extension_processor.py
View file

@ -27,7 +27,7 @@ from .base import (
async def add_extension_received_document(
session: AsyncSession,
content: ExtensionDocumentContent,
search_space_id: int,
workspace_id: int,
user_id: str,
) -> Document | None:
"""
@ -36,13 +36,13 @@ async def add_extension_received_document(
Args:
session: Database session
content: Document content from extension
search_space_id: ID of the search space
workspace_id: ID of the workspace
user_id: ID of the user
Returns:
Document object if successful, None if failed
"""
task_logger = TaskLoggingService(session, search_space_id)
task_logger = TaskLoggingService(session, workspace_id)
# Log task start
log_entry = await task_logger.log_task_start(
@ -90,11 +90,11 @@ async def add_extension_received_document(
# Generate unique identifier hash for this extension document (using URL)
unique_identifier_hash = generate_unique_identifier_hash(
DocumentType.EXTENSION, content.metadata.VisitedWebPageURL, search_space_id
DocumentType.EXTENSION, content.metadata.VisitedWebPageURL, workspace_id
)
# Generate content hash
content_hash = generate_content_hash(combined_document_string, search_space_id)
content_hash = generate_content_hash(combined_document_string, workspace_id)
# Check if document with this unique identifier already exists
existing_document = await check_document_by_unique_identifier(
@ -146,7 +146,7 @@ async def add_extension_received_document(
else:
# Create new document
document = Document(
search_space_id=search_space_id,
workspace_id=workspace_id,
title=content.metadata.VisitedWebPageTitle,
document_type=DocumentType.EXTENSION,
document_metadata=content.metadata.model_dump(),

26
surfsense_backend/app/tasks/document_processors/file_processors.py
View file

@ -42,7 +42,7 @@ class _ProcessingContext:
session: AsyncSession
file_path: str
filename: str
search_space_id: int
workspace_id: int
user_id: str
task_logger: TaskLoggingService
log_entry: Log
@ -135,7 +135,7 @@ async def _process_non_document_upload(ctx: _ProcessingContext) -> Document | No
if ctx.use_vision_llm:
from app.services.llm_service import get_vision_llm
vision_llm = await get_vision_llm(ctx.session, ctx.search_space_id)
vision_llm = await get_vision_llm(ctx.session, ctx.workspace_id)
etl_result = await extract_with_cache(
EtlRequest(file_path=ctx.file_path, filename=ctx.filename),
@ -151,7 +151,7 @@ async def _process_non_document_upload(ctx: _ProcessingContext) -> Document | No
ctx.session,
ctx.filename,
etl_result.markdown_content,
ctx.search_space_id,
ctx.workspace_id,
ctx.user_id,
ctx.connector,
)
@ -237,7 +237,7 @@ async def _process_document_upload(ctx: _ProcessingContext) -> Document | None:
if ctx.use_vision_llm:
from app.services.llm_service import get_vision_llm
vision_llm = await get_vision_llm(ctx.session, ctx.search_space_id)
vision_llm = await get_vision_llm(ctx.session, ctx.workspace_id)
etl_result = await extract_with_cache(
EtlRequest(
@ -258,7 +258,7 @@ async def _process_document_upload(ctx: _ProcessingContext) -> Document | None:
ctx.session,
ctx.filename,
etl_result.markdown_content,
ctx.search_space_id,
ctx.workspace_id,
ctx.user_id,
etl_result.etl_service,
ctx.connector,
@ -302,7 +302,7 @@ async def _process_document_upload(ctx: _ProcessingContext) -> Document | None:
async def process_file_in_background(
file_path: str,
filename: str,
search_space_id: int,
workspace_id: int,
user_id: str,
session: AsyncSession,
task_logger: TaskLoggingService,
@ -316,7 +316,7 @@ async def process_file_in_background(
session=session,
file_path=file_path,
filename=filename,
search_space_id=search_space_id,
workspace_id=workspace_id,
user_id=user_id,
task_logger=task_logger,
log_entry=log_entry,
@ -368,7 +368,7 @@ async def process_file_in_background(
async def _extract_file_content(
file_path: str,
filename: str,
search_space_id: int,
workspace_id: int,
session: AsyncSession,
user_id: str,
task_logger: TaskLoggingService,
@ -432,7 +432,7 @@ async def _extract_file_content(
if use_vision_llm:
from app.services.llm_service import get_vision_llm
vision_llm = await get_vision_llm(session, search_space_id)
vision_llm = await get_vision_llm(session, workspace_id)
from app.etl_pipeline.cache import extract_with_cache
@ -459,7 +459,7 @@ async def process_file_in_background_with_document(
document: Document,
file_path: str,
filename: str,
search_space_id: int,
workspace_id: int,
user_id: str,
session: AsyncSession,
task_logger: TaskLoggingService,
@ -491,7 +491,7 @@ async def process_file_in_background_with_document(
markdown_content, etl_service, billable_pages = await _extract_file_content(
file_path,
filename,
search_space_id,
workspace_id,
session,
user_id,
task_logger,
@ -504,7 +504,7 @@ async def process_file_in_background_with_document(
if not markdown_content:
raise RuntimeError(f"Failed to extract content from file: {filename}")
content_hash = generate_content_hash(markdown_content, search_space_id)
content_hash = generate_content_hash(markdown_content, workspace_id)
existing_by_content = await check_duplicate_document(session, content_hash)
if existing_by_content and existing_by_content.id != doc_id:
logging.info(
@ -525,7 +525,7 @@ async def process_file_in_background_with_document(
markdown_content=markdown_content,
filename=filename,
etl_service=etl_service,
search_space_id=search_space_id,
workspace_id=workspace_id,
user_id=user_id,
)

12
surfsense_backend/app/tasks/document_processors/markdown_processor.py
View file

@ -120,7 +120,7 @@ async def add_received_markdown_file_document(
session: AsyncSession,
file_name: str,
file_in_markdown: str,
search_space_id: int,
workspace_id: int,
user_id: str,
connector: dict | None = None,
) -> Document | None:
@ -131,14 +131,14 @@ async def add_received_markdown_file_document(
session: Database session
file_name: Name of the markdown file
file_in_markdown: Content of the markdown file
search_space_id: ID of the search space
workspace_id: ID of the workspace
user_id: ID of the user
connector: Optional connector info for Google Drive files
Returns:
Document object if successful, None if failed
"""
task_logger = TaskLoggingService(session, search_space_id)
task_logger = TaskLoggingService(session, workspace_id)
# Log task start
log_entry = await task_logger.log_task_start(
@ -155,11 +155,11 @@ async def add_received_markdown_file_document(
try:
# Generate unique identifier hash (uses file_id for Google Drive, filename for others)
primary_hash, legacy_hash = get_google_drive_unique_identifier(
connector, file_name, search_space_id
connector, file_name, workspace_id
)
# Generate content hash
content_hash = generate_content_hash(file_in_markdown, search_space_id)
content_hash = generate_content_hash(file_in_markdown, workspace_id)
# Check if document exists (with migration support for Google Drive and content_hash fallback)
existing_document = await find_existing_document_with_migration(
@ -214,7 +214,7 @@ async def add_received_markdown_file_document(
doc_type = DocumentType.GOOGLE_DRIVE_FILE
document = Document(
search_space_id=search_space_id,
workspace_id=workspace_id,
title=file_name,
document_type=doc_type,
document_metadata={

12
surfsense_backend/app/tasks/document_processors/youtube_processor.py
View file

@ -63,7 +63,7 @@ def get_youtube_video_id(url: str) -> str | None:
async def add_youtube_video_document(
session: AsyncSession,
url: str,
search_space_id: int,
workspace_id: int,
user_id: str,
notification=None,
) -> Document:
@ -77,7 +77,7 @@ async def add_youtube_video_document(
Args:
session: Database session for storing the document
url: YouTube video URL (supports standard, shortened, and embed formats)
search_space_id: ID of the search space to add the document to
workspace_id: ID of the workspace to add the document to
user_id: ID of the user
notification: Optional notification object — if provided, the document_id
is stored in its metadata right after document creation so the stale
@ -91,7 +91,7 @@ async def add_youtube_video_document(
SQLAlchemyError: If there's a database error
RuntimeError: If the video processing fails
"""
task_logger = TaskLoggingService(session, search_space_id)
task_logger = TaskLoggingService(session, workspace_id)
# Log task start
log_entry = await task_logger.log_task_start(
@ -125,7 +125,7 @@ async def add_youtube_video_document(
# Generate unique identifier hash for this YouTube video
unique_identifier_hash = generate_unique_identifier_hash(
DocumentType.YOUTUBE_VIDEO, video_id, search_space_id
DocumentType.YOUTUBE_VIDEO, video_id, workspace_id
)
# Check if document with this unique identifier already exists
@ -181,7 +181,7 @@ async def add_youtube_video_document(
embedding=None,
chunks=[], # Empty at creation
status=DocumentStatus.pending(), # PENDING status - visible in UI
search_space_id=search_space_id,
workspace_id=workspace_id,
updated_at=get_current_timestamp(),
created_by_id=user_id,
)
@ -346,7 +346,7 @@ async def add_youtube_video_document(
combined_document_string = "\n".join(document_parts)
# Generate content hash
content_hash = generate_content_hash(combined_document_string, search_space_id)
content_hash = generate_content_hash(combined_document_string, workspace_id)
# For existing documents, check if content has changed
if not is_new_document and existing_document.content_hash == content_hash: