diff --git a/surfsense_backend/alembic/versions/149_add_gateway_tables.py b/surfsense_backend/alembic/versions/149_add_gateway_tables.py index 888da0691..a77e6a69b 100644 --- a/surfsense_backend/alembic/versions/149_add_gateway_tables.py +++ b/surfsense_backend/alembic/versions/149_add_gateway_tables.py @@ -54,6 +54,17 @@ USER_COLS = [ "premium_credit_micros_used", ] +AUTOMATION_RUN_COLS = [ + "id", + "automation_id", + "trigger_id", + "status", + "step_results", + "started_at", + "finished_at", + "created_at", +] + def _has_zero_version(conn, table: str) -> bool: return ( conn.execute( @@ -150,7 +161,8 @@ def _build_set_table_ddl( f"new_chat_messages, " f"chat_comments, " f"chat_session_state, " - f'"user" ({_cols(user_cols)})' + f'"user" ({_cols(user_cols)}), ' + f"automation_runs ({_cols(AUTOMATION_RUN_COLS)})" ) @@ -523,7 +535,7 @@ def downgrade() -> None: if exists: documents_has_zero_ver = _has_zero_version(conn, "documents") user_has_zero_ver = _has_zero_version(conn, "user") - # Restore the publication shape from migration 143. + # Restore the publication shape from migration 148. doc_cols = DOCUMENT_COLS + (['"_0_version"'] if documents_has_zero_ver else []) user_cols = USER_COLS + (['"_0_version"'] if user_has_zero_ver else []) ddl = ( @@ -535,7 +547,8 @@ def downgrade() -> None: f"new_chat_messages, " f"chat_comments, " f"chat_session_state, " - f'"user" ({_cols(user_cols)})' + f'"user" ({_cols(user_cols)}), ' + f"automation_runs ({_cols(AUTOMATION_RUN_COLS)})" ) tx = conn.begin_nested() if conn.in_transaction() else conn.begin() with tx: diff --git a/surfsense_backend/alembic/versions/154_remove_document_summary_llm.py b/surfsense_backend/alembic/versions/154_remove_document_summary_llm.py new file mode 100644 index 000000000..6d0eb45cf --- /dev/null +++ b/surfsense_backend/alembic/versions/154_remove_document_summary_llm.py @@ -0,0 +1,134 @@ +"""remove document summary llm settings + +Revision ID: 154 +Revises: 153 +""" + +from collections.abc import Sequence + +import sqlalchemy as sa + +from alembic import op + +revision: str = "154" +down_revision: str | None = "153" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +PUBLICATION_NAME = "zero_publication" + +DOCUMENT_COLS = [ + "id", + "title", + "document_type", + "search_space_id", + "folder_id", + "created_by_id", + "status", + "created_at", + "updated_at", +] + +USER_COLS = [ + "id", + "pages_limit", + "pages_used", + "premium_credit_micros_limit", + "premium_credit_micros_used", +] + +AUTOMATION_RUN_COLS = [ + "id", + "automation_id", + "trigger_id", + "status", + "step_results", + "started_at", + "finished_at", + "created_at", +] + + +def _column_exists(conn, table: str, column: str) -> bool: + return ( + conn.execute( + sa.text( + "SELECT 1 FROM information_schema.columns " + "WHERE table_name = :table AND column_name = :column" + ), + {"table": table, "column": column}, + ).fetchone() + is not None + ) + + +def _has_zero_version(conn, table: str) -> bool: + return _column_exists(conn, table, "_0_version") + + +def _set_table_ddl(conn) -> str: + doc_cols = DOCUMENT_COLS + (['"_0_version"'] if _has_zero_version(conn, "documents") else []) + user_cols = USER_COLS + (['"_0_version"'] if _has_zero_version(conn, "user") else []) + tables = [ + "notifications", + f"documents ({', '.join(doc_cols)})", + "folders", + "search_source_connectors", + "new_chat_messages", + "chat_comments", + "chat_session_state", + f'"user" ({", ".join(user_cols)})', + f"automation_runs ({', '.join(AUTOMATION_RUN_COLS)})", + ] + return f"ALTER PUBLICATION {PUBLICATION_NAME} SET TABLE " + ", ".join(tables) + + +def _resync_zero_publication(tag: str) -> None: + conn = op.get_bind() + exists = conn.execute( + sa.text("SELECT 1 FROM pg_publication WHERE pubname = :name"), + {"name": PUBLICATION_NAME}, + ).fetchone() + if not exists: + return + + tx = conn.begin_nested() if conn.in_transaction() else conn.begin() + with tx: + conn.execute(sa.text(f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'pre-{tag}'")) + conn.execute(sa.text(_set_table_ddl(conn))) + conn.execute(sa.text(f"COMMENT ON PUBLICATION {PUBLICATION_NAME} IS 'post-{tag}'")) + + +def upgrade() -> None: + conn = op.get_bind() + + if _column_exists(conn, "searchspaces", "document_summary_llm_id"): + op.drop_column("searchspaces", "document_summary_llm_id") + + if _column_exists(conn, "search_source_connectors", "enable_summary"): + op.drop_column("search_source_connectors", "enable_summary") + + _resync_zero_publication("154-summary-removal") + + +def downgrade() -> None: + conn = op.get_bind() + + if not _column_exists(conn, "searchspaces", "document_summary_llm_id"): + op.add_column( + "searchspaces", + sa.Column("document_summary_llm_id", sa.Integer(), nullable=True, server_default="0"), + ) + + if not _column_exists(conn, "search_source_connectors", "enable_summary"): + op.add_column( + "search_source_connectors", + sa.Column( + "enable_summary", + sa.Boolean(), + nullable=False, + server_default=sa.text("false"), + ), + ) + + _resync_zero_publication("154-summary-removal-downgrade") diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/deliverables/tools/report.py b/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/deliverables/tools/report.py index f12ca8a90..39b7c4694 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/deliverables/tools/report.py +++ b/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/deliverables/tools/report.py @@ -16,7 +16,7 @@ from app.agents.shared.receipt import make_receipt from app.agents.shared.receipt_command import with_receipt from app.db import Report, shielded_async_session from app.services.connector_service import ConnectorService -from app.services.llm_service import get_document_summary_llm +from app.services.llm_service import get_agent_llm logger = logging.getLogger(__name__) @@ -546,7 +546,7 @@ def create_generate_report_tool( Factory function to create the generate_report tool with injected dependencies. The tool generates a Markdown report inline using the search space's - document summary LLM, saves it to the database, and returns immediately. + agent LLM, saves it to the database, and returns immediately. Uses short-lived database sessions for each DB operation so no connection is held during the long LLM API call. @@ -767,7 +767,7 @@ def create_generate_report_tool( "creating standalone report" ) - llm = await get_document_summary_llm(read_session, search_space_id) + llm = await get_agent_llm(read_session, search_space_id) # read_session closed — connection returned to pool if not llm: diff --git a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/deliverables/tools/resume.py b/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/deliverables/tools/resume.py index ad16b7ba7..86332ccbe 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/deliverables/tools/resume.py +++ b/surfsense_backend/app/agents/multi_agent_chat/subagents/builtins/deliverables/tools/resume.py @@ -17,7 +17,7 @@ from langgraph.types import Command from app.agents.shared.receipt import make_receipt from app.agents.shared.receipt_command import with_receipt from app.db import Report, shielded_async_session -from app.services.llm_service import get_document_summary_llm +from app.services.llm_service import get_agent_llm logger = logging.getLogger(__name__) @@ -578,7 +578,7 @@ def create_generate_resume_tool( f"(group {report_group_id})" ) - llm = await get_document_summary_llm(read_session, search_space_id) + llm = await get_agent_llm(read_session, search_space_id) if not llm: error_msg = ( diff --git a/surfsense_backend/app/agents/new_chat/tools/report.py b/surfsense_backend/app/agents/new_chat/tools/report.py index 6bc1b7d57..8c0bd95ea 100644 --- a/surfsense_backend/app/agents/new_chat/tools/report.py +++ b/surfsense_backend/app/agents/new_chat/tools/report.py @@ -35,7 +35,7 @@ from langchain_core.tools import tool from app.db import Report, shielded_async_session from app.services.connector_service import ConnectorService -from app.services.llm_service import get_document_summary_llm +from app.services.llm_service import get_agent_llm logger = logging.getLogger(__name__) @@ -565,7 +565,7 @@ def create_generate_report_tool( Factory function to create the generate_report tool with injected dependencies. The tool generates a Markdown report inline using the search space's - document summary LLM, saves it to the database, and returns immediately. + agent LLM, saves it to the database, and returns immediately. Uses short-lived database sessions for each DB operation so no connection is held during the long LLM API call. @@ -768,7 +768,7 @@ def create_generate_report_tool( "creating standalone report" ) - llm = await get_document_summary_llm(read_session, search_space_id) + llm = await get_agent_llm(read_session, search_space_id) # read_session closed — connection returned to pool if not llm: diff --git a/surfsense_backend/app/agents/new_chat/tools/resume.py b/surfsense_backend/app/agents/new_chat/tools/resume.py index 4abe48ba6..17849bce7 100644 --- a/surfsense_backend/app/agents/new_chat/tools/resume.py +++ b/surfsense_backend/app/agents/new_chat/tools/resume.py @@ -26,7 +26,7 @@ from langchain_core.messages import HumanMessage from langchain_core.tools import tool from app.db import Report, shielded_async_session -from app.services.llm_service import get_document_summary_llm +from app.services.llm_service import get_agent_llm logger = logging.getLogger(__name__) @@ -547,7 +547,7 @@ def create_generate_resume_tool( f"(group {report_group_id})" ) - llm = await get_document_summary_llm(read_session, search_space_id) + llm = await get_agent_llm(read_session, search_space_id) if not llm: error_msg = ( diff --git a/surfsense_backend/app/agents/podcaster/nodes.py b/surfsense_backend/app/agents/podcaster/nodes.py index 517d900a3..277536211 100644 --- a/surfsense_backend/app/agents/podcaster/nodes.py +++ b/surfsense_backend/app/agents/podcaster/nodes.py @@ -31,12 +31,10 @@ async def create_podcast_transcript( search_space_id = configuration.search_space_id user_prompt = configuration.user_prompt - # Get search space's document summary LLM + # Use the search space's agent LLM for podcast transcript generation. llm = await get_agent_llm(state.db_session, search_space_id) if not llm: - error_message = ( - f"No document summary LLM configured for search space {search_space_id}" - ) + error_message = f"No agent LLM configured for search space {search_space_id}" print(error_message) raise RuntimeError(error_message) diff --git a/surfsense_backend/app/celery_app.py b/surfsense_backend/app/celery_app.py index c60b2e4df..c7ee72667 100644 --- a/surfsense_backend/app/celery_app.py +++ b/surfsense_backend/app/celery_app.py @@ -103,7 +103,7 @@ def init_worker(**kwargs): """Initialize the LLM Router and Image Gen Router when a Celery worker process starts. This ensures the Auto mode (LiteLLM Router) is available for background tasks - like document summarization and image generation. + like agent workflows and image generation. """ from app.observability.bootstrap import init_otel diff --git a/surfsense_backend/app/connectors/google_drive/content_extractor.py b/surfsense_backend/app/connectors/google_drive/content_extractor.py index 86c789b97..59392831d 100644 --- a/surfsense_backend/app/connectors/google_drive/content_extractor.py +++ b/surfsense_backend/app/connectors/google_drive/content_extractor.py @@ -141,7 +141,6 @@ async def download_and_process_file( task_logger: TaskLoggingService, log_entry: Log, connector_id: int | None = None, - enable_summary: bool = True, ) -> tuple[Any, str | None, dict[str, Any] | None]: """ Download Google Drive file and process using Surfsense file processors. @@ -215,8 +214,6 @@ async def download_and_process_file( "source_connector": "google_drive", }, } - # Include connector_id for de-indexing support - connector_info["enable_summary"] = enable_summary if connector_id is not None: connector_info["connector_id"] = connector_id diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py index 5be10427f..c6fe1ee37 100644 --- a/surfsense_backend/app/db.py +++ b/surfsense_backend/app/db.py @@ -1781,9 +1781,6 @@ class SearchSpace(BaseModel, TimestampMixin): agent_llm_id = Column( Integer, nullable=True, default=0 ) # For agent/chat operations, defaults to Auto mode - document_summary_llm_id = Column( - Integer, nullable=True, default=0 - ) # For document summarization, defaults to Auto mode image_generation_config_id = Column( Integer, nullable=True, default=0 ) # For image generation, defaults to Auto mode @@ -1951,12 +1948,6 @@ class SearchSourceConnector(BaseModel, TimestampMixin): last_indexed_at = Column(TIMESTAMP(timezone=True), nullable=True) config = Column(JSON, nullable=False) - # Summary generation (LLM-based) - disabled by default to save resources. - # When enabled, improves hybrid search quality at the cost of LLM calls. - enable_summary = Column( - Boolean, nullable=False, default=False, server_default="false" - ) - # Vision LLM for image files - disabled by default to save cost/time. # When enabled, images are described via a vision language model instead # of falling back to the document parser. @@ -2972,7 +2963,7 @@ async def shielded_async_session(): async def setup_indexes(): async with engine.begin() as conn: # Create indexes - # Document Summary Indexes + # Document embedding indexes await conn.execute( text( "CREATE INDEX IF NOT EXISTS document_vector_index ON documents USING hnsw (embedding public.vector_cosine_ops)" diff --git a/surfsense_backend/app/indexing_pipeline/adapters/file_upload_adapter.py b/surfsense_backend/app/indexing_pipeline/adapters/file_upload_adapter.py index 0bbb67105..9a9e4e4d6 100644 --- a/surfsense_backend/app/indexing_pipeline/adapters/file_upload_adapter.py +++ b/surfsense_backend/app/indexing_pipeline/adapters/file_upload_adapter.py @@ -18,8 +18,6 @@ class UploadDocumentAdapter: etl_service: str, search_space_id: int, user_id: str, - llm, - should_summarize: bool = False, ) -> None: connector_doc = ConnectorDocument( title=filename, @@ -29,9 +27,7 @@ class UploadDocumentAdapter: search_space_id=search_space_id, created_by_id=user_id, connector_id=None, - should_summarize=should_summarize, should_use_code_chunker=False, - fallback_summary=markdown_content[:4000], metadata={ "FILE_NAME": filename, "ETL_SERVICE": etl_service, @@ -43,7 +39,7 @@ class UploadDocumentAdapter: if not documents: raise RuntimeError("prepare_for_indexing returned no documents") - indexed = await self._service.index(documents[0], connector_doc, llm) + indexed = await self._service.index(documents[0], connector_doc) if not DocumentStatus.is_state(indexed.status, DocumentStatus.READY): raise RuntimeError(indexed.status.get("reason", "Indexing failed")) @@ -51,7 +47,7 @@ class UploadDocumentAdapter: indexed.content_needs_reindexing = False await self._session.commit() - async def reindex(self, document: Document, llm) -> None: + async def reindex(self, document: Document) -> None: """Re-index an existing document after its source_markdown has been updated.""" if not document.source_markdown: raise RuntimeError("Document has no source_markdown to reindex") @@ -66,15 +62,13 @@ class UploadDocumentAdapter: search_space_id=document.search_space_id, created_by_id=str(document.created_by_id), connector_id=document.connector_id, - should_summarize=True, should_use_code_chunker=False, - fallback_summary=document.source_markdown[:4000], metadata=metadata, ) document.content_hash = compute_content_hash(connector_doc) - indexed = await self._service.index(document, connector_doc, llm) + indexed = await self._service.index(document, connector_doc) if not DocumentStatus.is_state(indexed.status, DocumentStatus.READY): raise RuntimeError(indexed.status.get("reason", "Reindexing failed")) diff --git a/surfsense_backend/app/indexing_pipeline/connector_document.py b/surfsense_backend/app/indexing_pipeline/connector_document.py index 4f5d6e2e0..1297a6b46 100644 --- a/surfsense_backend/app/indexing_pipeline/connector_document.py +++ b/surfsense_backend/app/indexing_pipeline/connector_document.py @@ -11,9 +11,7 @@ class ConnectorDocument(BaseModel): unique_id: str document_type: DocumentType search_space_id: int = Field(gt=0) - should_summarize: bool = True should_use_code_chunker: bool = False - fallback_summary: str | None = None metadata: dict = {} connector_id: int | None = None created_by_id: str diff --git a/surfsense_backend/app/indexing_pipeline/document_summarizer.py b/surfsense_backend/app/indexing_pipeline/document_summarizer.py deleted file mode 100644 index 76cc77377..000000000 --- a/surfsense_backend/app/indexing_pipeline/document_summarizer.py +++ /dev/null @@ -1,30 +0,0 @@ -from app.prompts import SUMMARY_PROMPT_TEMPLATE -from app.utils.document_converters import optimize_content_for_context_window - - -async def summarize_document( - source_markdown: str, llm, metadata: dict | None = None -) -> str: - """Generate a text summary of a document using an LLM, prefixed with metadata when provided.""" - model_name = getattr(llm, "model", "gpt-3.5-turbo") - optimized_content = optimize_content_for_context_window( - source_markdown, metadata, model_name - ) - - summary_chain = SUMMARY_PROMPT_TEMPLATE | llm - content_with_metadata = ( - f"\n\n{metadata}\n\n" - f"\n\n\n\n{optimized_content}\n\n" - ) - summary_result = await summary_chain.ainvoke({"document": content_with_metadata}) - summary_content = summary_result.content - - if metadata: - metadata_parts = ["# DOCUMENT METADATA"] - for key, value in metadata.items(): - if value: - metadata_parts.append(f"**{key.replace('_', ' ').title()}:** {value}") - metadata_section = "\n".join(metadata_parts) - return f"{metadata_section}\n\n# DOCUMENT SUMMARY\n\n{summary_content}" - - return summary_content diff --git a/surfsense_backend/app/indexing_pipeline/indexing_pipeline_service.py b/surfsense_backend/app/indexing_pipeline/indexing_pipeline_service.py index 282bd6034..3d0124059 100644 --- a/surfsense_backend/app/indexing_pipeline/indexing_pipeline_service.py +++ b/surfsense_backend/app/indexing_pipeline/indexing_pipeline_service.py @@ -31,7 +31,6 @@ from app.indexing_pipeline.document_persistence import ( attach_chunks_to_document, rollback_and_persist_failure, ) -from app.indexing_pipeline.document_summarizer import summarize_document from app.indexing_pipeline.exceptions import ( EMBEDDING_ERRORS, PERMANENT_LLM_ERRORS, @@ -203,9 +202,7 @@ class IndexingPipelineService: await self.session.commit() - async def index_batch( - self, connector_docs: list[ConnectorDocument], llm - ) -> list[Document]: + async def index_batch(self, connector_docs: list[ConnectorDocument]) -> list[Document]: """Convenience method: prepare_for_indexing then index each document. Indexers that need heartbeat callbacks or custom per-document logic @@ -218,7 +215,7 @@ class IndexingPipelineService: connector_doc = doc_map.get(document.unique_identifier_hash) if connector_doc is None: continue - result = await self.index(document, connector_doc, llm) + result = await self.index(document, connector_doc) results.append(result) return results @@ -350,11 +347,9 @@ class IndexingPipelineService: await self.session.rollback() return [] - async def index( - self, document: Document, connector_doc: ConnectorDocument, llm - ) -> Document: + async def index(self, document: Document, connector_doc: ConnectorDocument) -> Document: """ - Run summarization, embedding, and chunking for a document and persist the results. + Run deterministic content storage, embedding, and chunking for a document. """ ctx = PipelineLogContext( connector_id=connector_doc.connector_id, @@ -379,20 +374,7 @@ class IndexingPipelineService: document.status = DocumentStatus.processing() await self.session.commit() - t_step = time.perf_counter() - if connector_doc.should_summarize and llm is not None: - content = await summarize_document( - connector_doc.source_markdown, llm, connector_doc.metadata - ) - perf.info( - "[indexing] summarize_document doc=%d in %.3fs", - document.id, - time.perf_counter() - t_step, - ) - elif connector_doc.should_summarize and connector_doc.fallback_summary: - content = connector_doc.fallback_summary - else: - content = connector_doc.source_markdown + content = connector_doc.source_markdown await self.session.execute( delete(Chunk).where(Chunk.document_id == document.id) @@ -523,7 +505,6 @@ class IndexingPipelineService: async def index_batch_parallel( self, connector_docs: list[ConnectorDocument], - get_llm: Callable[[AsyncSession], Awaitable], *, max_concurrency: int = 4, on_heartbeat: Callable[[int], Awaitable[None]] | None = None, @@ -532,8 +513,8 @@ class IndexingPipelineService: """Index documents in parallel with bounded concurrency. Phase 1 (serial): prepare_for_indexing using self.session. - Phase 2 (parallel): index each document in an isolated session, - bounded by a semaphore to avoid overwhelming APIs/DB. + Phase 2 (parallel): index each document in an isolated session, bounded + by a semaphore to avoid overwhelming embedding APIs/DB. """ logger = logging.getLogger(__name__) perf = get_perf_logger() @@ -577,9 +558,8 @@ class IndexingPipelineService: failed_count += 1 return document - llm = await get_llm(isolated_session) iso_pipeline = IndexingPipelineService(isolated_session) - result = await iso_pipeline.index(refetched, connector_doc, llm) + result = await iso_pipeline.index(refetched, connector_doc) async with lock: if DocumentStatus.is_state( diff --git a/surfsense_backend/app/routes/documents_routes.py b/surfsense_backend/app/routes/documents_routes.py index 4501f2111..cafd34ef7 100644 --- a/surfsense_backend/app/routes/documents_routes.py +++ b/surfsense_backend/app/routes/documents_routes.py @@ -125,7 +125,6 @@ async def create_documents( async def create_documents_file_upload( files: list[UploadFile], search_space_id: int = Form(...), - should_summarize: bool = Form(False), use_vision_llm: bool = Form(False), processing_mode: str = Form("basic"), session: AsyncSession = Depends(get_async_session), @@ -309,7 +308,6 @@ async def create_documents_file_upload( filename=filename, search_space_id=search_space_id, user_id=str(user.id), - should_summarize=should_summarize, use_vision_llm=use_vision_llm, processing_mode=validated_mode.value, ) @@ -1586,7 +1584,6 @@ async def folder_upload( search_space_id: int = Form(...), relative_paths: str = Form(...), root_folder_id: int | None = Form(None), - enable_summary: bool = Form(False), use_vision_llm: bool = Form(False), processing_mode: str = Form("basic"), session: AsyncSession = Depends(get_async_session), @@ -1719,7 +1716,6 @@ async def folder_upload( user_id=str(user.id), folder_name=folder_name, root_folder_id=root_folder_id, - enable_summary=enable_summary, use_vision_llm=use_vision_llm, file_mappings=list(file_mappings), processing_mode=validated_mode.value, diff --git a/surfsense_backend/app/routes/search_spaces_routes.py b/surfsense_backend/app/routes/search_spaces_routes.py index db230b0f5..898077b7a 100644 --- a/surfsense_backend/app/routes/search_spaces_routes.py +++ b/surfsense_backend/app/routes/search_spaces_routes.py @@ -617,9 +617,6 @@ async def get_llm_preferences( # Get full config objects for each role agent_llm = await _get_llm_config_by_id(session, search_space.agent_llm_id) - document_summary_llm = await _get_llm_config_by_id( - session, search_space.document_summary_llm_id - ) image_generation_config = await _get_image_gen_config_by_id( session, search_space.image_generation_config_id ) @@ -629,11 +626,9 @@ async def get_llm_preferences( return LLMPreferencesRead( agent_llm_id=search_space.agent_llm_id, - document_summary_llm_id=search_space.document_summary_llm_id, image_generation_config_id=search_space.image_generation_config_id, vision_llm_config_id=search_space.vision_llm_config_id, agent_llm=agent_llm, - document_summary_llm=document_summary_llm, image_generation_config=image_generation_config, vision_llm_config=vision_llm_config, ) @@ -707,9 +702,6 @@ async def update_llm_preferences( # Get full config objects for response agent_llm = await _get_llm_config_by_id(session, search_space.agent_llm_id) - document_summary_llm = await _get_llm_config_by_id( - session, search_space.document_summary_llm_id - ) image_generation_config = await _get_image_gen_config_by_id( session, search_space.image_generation_config_id ) @@ -719,11 +711,9 @@ async def update_llm_preferences( return LLMPreferencesRead( agent_llm_id=search_space.agent_llm_id, - document_summary_llm_id=search_space.document_summary_llm_id, image_generation_config_id=search_space.image_generation_config_id, vision_llm_config_id=search_space.vision_llm_config_id, agent_llm=agent_llm, - document_summary_llm=document_summary_llm, image_generation_config=image_generation_config, vision_llm_config=vision_llm_config, ) diff --git a/surfsense_backend/app/schemas/new_llm_config.py b/surfsense_backend/app/schemas/new_llm_config.py index e64478d38..716aa0457 100644 --- a/surfsense_backend/app/schemas/new_llm_config.py +++ b/surfsense_backend/app/schemas/new_llm_config.py @@ -221,9 +221,6 @@ class LLMPreferencesRead(BaseModel): agent_llm_id: int | None = Field( None, description="ID of the LLM config to use for agent/chat tasks" ) - document_summary_llm_id: int | None = Field( - None, description="ID of the LLM config to use for document summarization" - ) image_generation_config_id: int | None = Field( None, description="ID of the image generation config to use" ) @@ -234,9 +231,6 @@ class LLMPreferencesRead(BaseModel): agent_llm: dict[str, Any] | None = Field( None, description="Full config for agent LLM" ) - document_summary_llm: dict[str, Any] | None = Field( - None, description="Full config for document summary LLM" - ) image_generation_config: dict[str, Any] | None = Field( None, description="Full config for image generation" ) @@ -253,9 +247,6 @@ class LLMPreferencesUpdate(BaseModel): agent_llm_id: int | None = Field( None, description="ID of the LLM config to use for agent/chat tasks" ) - document_summary_llm_id: int | None = Field( - None, description="ID of the LLM config to use for document summarization" - ) image_generation_config_id: int | None = Field( None, description="ID of the image generation config to use" ) diff --git a/surfsense_backend/app/schemas/search_source_connector.py b/surfsense_backend/app/schemas/search_source_connector.py index aac7b92d5..982931859 100644 --- a/surfsense_backend/app/schemas/search_source_connector.py +++ b/surfsense_backend/app/schemas/search_source_connector.py @@ -16,7 +16,6 @@ class SearchSourceConnectorBase(BaseModel): is_indexable: bool last_indexed_at: datetime | None = None config: dict[str, Any] - enable_summary: bool = False enable_vision_llm: bool = False periodic_indexing_enabled: bool = False indexing_frequency_minutes: int | None = None @@ -67,7 +66,6 @@ class SearchSourceConnectorUpdate(BaseModel): is_indexable: bool | None = None last_indexed_at: datetime | None = None config: dict[str, Any] | None = None - enable_summary: bool | None = None enable_vision_llm: bool | None = None periodic_indexing_enabled: bool | None = None indexing_frequency_minutes: int | None = None diff --git a/surfsense_backend/app/services/confluence/kb_sync_service.py b/surfsense_backend/app/services/confluence/kb_sync_service.py index cae2bef88..df07c3e81 100644 --- a/surfsense_backend/app/services/confluence/kb_sync_service.py +++ b/surfsense_backend/app/services/confluence/kb_sync_service.py @@ -9,7 +9,6 @@ from app.utils.document_converters import ( create_document_chunks, embed_text, generate_content_hash, - generate_document_summary, generate_unique_identifier_hash, ) @@ -65,29 +64,11 @@ class ConfluenceKBSyncService: if dup: content_hash = unique_hash - from app.services.llm_service import get_user_long_context_llm - user_llm = await get_user_long_context_llm( - self.db_session, - user_id, - search_space_id, - disable_streaming=True, - ) - doc_metadata_for_summary = { - "page_title": page_title, - "space_id": space_id, - "document_type": "Confluence Page", - "connector_type": "Confluence", - } - if user_llm: - summary_content, summary_embedding = await generate_document_summary( - page_content, user_llm, doc_metadata_for_summary - ) - else: - summary_content = f"Confluence Page: {page_title}\n\n{page_content}" - summary_embedding = embed_text(summary_content) + summary_content = f"Confluence Page: {page_title}\n\n{page_content}" + summary_embedding = embed_text(summary_content) chunks = await create_document_chunks(page_content) now_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S") @@ -185,25 +166,10 @@ class ConfluenceKBSyncService: space_id = (document.document_metadata or {}).get("space_id", "") - from app.services.llm_service import get_user_long_context_llm - user_llm = await get_user_long_context_llm( - self.db_session, user_id, search_space_id, disable_streaming=True - ) - if user_llm: - doc_meta = { - "page_title": page_title, - "space_id": space_id, - "document_type": "Confluence Page", - "connector_type": "Confluence", - } - summary_content, summary_embedding = await generate_document_summary( - page_content, user_llm, doc_meta - ) - else: - summary_content = f"Confluence Page: {page_title}\n\n{page_content}" - summary_embedding = embed_text(summary_content) + summary_content = f"Confluence Page: {page_title}\n\n{page_content}" + summary_embedding = embed_text(summary_content) chunks = await create_document_chunks(page_content) diff --git a/surfsense_backend/app/services/docling_service.py b/surfsense_backend/app/services/docling_service.py index cf51efb4a..dc87e75f0 100644 --- a/surfsense_backend/app/services/docling_service.py +++ b/surfsense_backend/app/services/docling_service.py @@ -191,149 +191,6 @@ class DoclingService: logger.error(f"Full traceback: {traceback.format_exc()}") raise RuntimeError(f"Docling processing failed: {e}") from e - async def process_large_document_summary( - self, content: str, llm, document_title: str = "Document" - ) -> str: - """ - Process large documents using chunked LLM summarization. - - Args: - content: The full document content - llm: The language model to use for summarization - document_title: Title of the document for context - - Returns: - Final summary of the document - """ - # Large document threshold (100K characters ≈ 25K tokens) - large_document_threshold = 100_000 - - if len(content) <= large_document_threshold: - # For smaller documents, use direct processing - logger.info( - f"📄 Document size: {len(content)} chars - using direct processing" - ) - from app.prompts import SUMMARY_PROMPT_TEMPLATE - - summary_chain = SUMMARY_PROMPT_TEMPLATE | llm - result = await summary_chain.ainvoke({"document": content}) - return result.content - - logger.info( - f"📚 Large document detected: {len(content)} chars - using chunked processing" - ) - - # Import chunker from config - # Create LLM-optimized chunks (8K tokens max for safety) - from chonkie import OverlapRefinery, RecursiveChunker - from langchain_core.prompts import PromptTemplate - - llm_chunker = RecursiveChunker( - chunk_size=8000 # Conservative for most LLMs - ) - - # Apply overlap refinery for context preservation (10% overlap = 800 tokens) - overlap_refinery = OverlapRefinery( - context_size=0.1, # 10% overlap for context preservation - method="suffix", # Add next chunk context to current chunk - ) - - # First chunk the content, then apply overlap refinery - initial_chunks = llm_chunker.chunk(content) - chunks = overlap_refinery.refine(initial_chunks) - total_chunks = len(chunks) - - logger.info(f"📄 Split into {total_chunks} chunks for LLM processing") - - # Template for chunk processing - chunk_template = PromptTemplate( - input_variables=["chunk", "chunk_number", "total_chunks"], - template=""" -You are summarizing chunk {chunk_number} of {total_chunks} from a large document. - -Create a comprehensive summary of this document chunk. Focus on: -- Key concepts, facts, and information -- Important details and context -- Main topics and themes - -Provide a clear, structured summary that captures the essential content. - -Chunk {chunk_number}/{total_chunks}: - -{chunk} - -""", - ) - - # Process each chunk individually - chunk_summaries = [] - for i, chunk in enumerate(chunks, 1): - try: - logger.info( - f"🔄 Processing chunk {i}/{total_chunks} ({len(chunk.text)} chars)" - ) - - chunk_chain = chunk_template | llm - chunk_result = await chunk_chain.ainvoke( - { - "chunk": chunk.text, - "chunk_number": i, - "total_chunks": total_chunks, - } - ) - - chunk_summary = chunk_result.content - chunk_summaries.append(f"=== Section {i} ===\n{chunk_summary}") - - logger.info(f"✅ Completed chunk {i}/{total_chunks}") - - except Exception as e: - logger.error(f"❌ Failed to process chunk {i}/{total_chunks}: {e}") - chunk_summaries.append(f"=== Section {i} ===\n[Processing failed]") - - # Combine summaries into final document summary - logger.info(f"🔄 Combining {len(chunk_summaries)} chunk summaries") - - try: - combine_template = PromptTemplate( - input_variables=["summaries", "document_title"], - template=""" -You are combining multiple section summaries into a final comprehensive document summary. - -Create a unified, coherent summary from the following section summaries of "{document_title}". -Ensure: -- Logical flow and organization -- No redundancy or repetition -- Comprehensive coverage of all key points -- Professional, objective tone - - -{summaries} - -""", - ) - - combined_summaries = "\n\n".join(chunk_summaries) - combine_chain = combine_template | llm - - final_result = await combine_chain.ainvoke( - {"summaries": combined_summaries, "document_title": document_title} - ) - - final_summary = final_result.content - logger.info( - f"✅ Large document processing complete: {len(final_summary)} chars summary" - ) - - return final_summary - - except Exception as e: - logger.error(f"❌ Failed to combine summaries: {e}") - # Fallback: return concatenated chunk summaries - fallback_summary = "\n\n".join(chunk_summaries) - logger.warning("⚠️ Using fallback combined summary") - return fallback_summary - def create_docling_service() -> DoclingService: """Create a Docling service instance.""" diff --git a/surfsense_backend/app/services/dropbox/kb_sync_service.py b/surfsense_backend/app/services/dropbox/kb_sync_service.py index 9d1951013..b455e4fdd 100644 --- a/surfsense_backend/app/services/dropbox/kb_sync_service.py +++ b/surfsense_backend/app/services/dropbox/kb_sync_service.py @@ -9,7 +9,6 @@ from app.utils.document_converters import ( create_document_chunks, embed_text, generate_content_hash, - generate_document_summary, ) logger = logging.getLogger(__name__) @@ -72,29 +71,11 @@ class DropboxKBSyncService: ) content_hash = unique_hash - from app.services.llm_service import get_user_long_context_llm - user_llm = await get_user_long_context_llm( - self.db_session, - user_id, - search_space_id, - disable_streaming=True, - ) - doc_metadata_for_summary = { - "file_name": file_name, - "document_type": "Dropbox File", - "connector_type": "Dropbox", - } - if user_llm: - summary_content, summary_embedding = await generate_document_summary( - indexable_content, user_llm, doc_metadata_for_summary - ) - else: - logger.warning("No LLM configured — using fallback summary") - summary_content = f"Dropbox File: {file_name}\n\n{indexable_content}" - summary_embedding = embed_text(summary_content) + summary_content = f"Dropbox File: {file_name}\n\n{indexable_content}" + summary_embedding = embed_text(summary_content) chunks = await create_document_chunks(indexable_content) now_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S") diff --git a/surfsense_backend/app/services/gmail/kb_sync_service.py b/surfsense_backend/app/services/gmail/kb_sync_service.py index 85e25fcb6..f2a8bed30 100644 --- a/surfsense_backend/app/services/gmail/kb_sync_service.py +++ b/surfsense_backend/app/services/gmail/kb_sync_service.py @@ -9,7 +9,6 @@ from app.utils.document_converters import ( create_document_chunks, embed_text, generate_content_hash, - generate_document_summary, generate_unique_identifier_hash, ) @@ -78,30 +77,11 @@ class GmailKBSyncService: ) content_hash = unique_hash - from app.services.llm_service import get_user_long_context_llm - user_llm = await get_user_long_context_llm( - self.db_session, - user_id, - search_space_id, - disable_streaming=True, - ) - doc_metadata_for_summary = { - "subject": subject, - "sender": sender, - "document_type": "Gmail Message", - "connector_type": "Gmail", - } - if user_llm: - summary_content, summary_embedding = await generate_document_summary( - indexable_content, user_llm, doc_metadata_for_summary - ) - else: - logger.warning("No LLM configured -- using fallback summary") - summary_content = f"Gmail Message: {subject}\n\n{indexable_content}" - summary_embedding = await asyncio.to_thread(embed_text, summary_content) + summary_content = f"Gmail Message: {subject}\n\n{indexable_content}" + summary_embedding = await asyncio.to_thread(embed_text, summary_content) chunks = await create_document_chunks(indexable_content) now_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S") diff --git a/surfsense_backend/app/services/google_calendar/kb_sync_service.py b/surfsense_backend/app/services/google_calendar/kb_sync_service.py index e59868aff..36fddc82a 100644 --- a/surfsense_backend/app/services/google_calendar/kb_sync_service.py +++ b/surfsense_backend/app/services/google_calendar/kb_sync_service.py @@ -19,7 +19,6 @@ from app.utils.document_converters import ( create_document_chunks, embed_text, generate_content_hash, - generate_document_summary, generate_unique_identifier_hash, ) @@ -90,33 +89,13 @@ class GoogleCalendarKBSyncService: ) content_hash = unique_hash - from app.services.llm_service import get_user_long_context_llm - user_llm = await get_user_long_context_llm( - self.db_session, - user_id, - search_space_id, - disable_streaming=True, + + + summary_content = ( + f"Google Calendar Event: {event_summary}\n\n{indexable_content}" ) - - doc_metadata_for_summary = { - "event_summary": event_summary, - "start_time": start_time, - "end_time": end_time, - "document_type": "Google Calendar Event", - "connector_type": "Google Calendar", - } - - if user_llm: - summary_content, summary_embedding = await generate_document_summary( - indexable_content, user_llm, doc_metadata_for_summary - ) - else: - logger.warning("No LLM configured -- using fallback summary") - summary_content = ( - f"Google Calendar Event: {event_summary}\n\n{indexable_content}" - ) - summary_embedding = await asyncio.to_thread(embed_text, summary_content) + summary_embedding = await asyncio.to_thread(embed_text, summary_content) chunks = await create_document_chunks(indexable_content) now_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S") @@ -273,29 +252,13 @@ class GoogleCalendarKBSyncService: if not indexable_content: return {"status": "error", "message": "Event produced empty content"} - from app.services.llm_service import get_user_long_context_llm - user_llm = await get_user_long_context_llm( - self.db_session, user_id, search_space_id, disable_streaming=True + + + summary_content = ( + f"Google Calendar Event: {event_summary}\n\n{indexable_content}" ) - - doc_metadata_for_summary = { - "event_summary": event_summary, - "start_time": start_time, - "end_time": end_time, - "document_type": "Google Calendar Event", - "connector_type": "Google Calendar", - } - - if user_llm: - summary_content, summary_embedding = await generate_document_summary( - indexable_content, user_llm, doc_metadata_for_summary - ) - else: - summary_content = ( - f"Google Calendar Event: {event_summary}\n\n{indexable_content}" - ) - summary_embedding = await asyncio.to_thread(embed_text, summary_content) + summary_embedding = await asyncio.to_thread(embed_text, summary_content) chunks = await create_document_chunks(indexable_content) now_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S") diff --git a/surfsense_backend/app/services/google_drive/kb_sync_service.py b/surfsense_backend/app/services/google_drive/kb_sync_service.py index 0a8eb47a6..78c0e2491 100644 --- a/surfsense_backend/app/services/google_drive/kb_sync_service.py +++ b/surfsense_backend/app/services/google_drive/kb_sync_service.py @@ -8,7 +8,6 @@ from app.utils.document_converters import ( create_document_chunks, embed_text, generate_content_hash, - generate_document_summary, generate_unique_identifier_hash, ) @@ -74,32 +73,13 @@ class GoogleDriveKBSyncService: ) content_hash = unique_hash - from app.services.llm_service import get_user_long_context_llm - user_llm = await get_user_long_context_llm( - self.db_session, - user_id, - search_space_id, - disable_streaming=True, + + + summary_content = ( + f"Google Drive File: {file_name}\n\n{indexable_content}" ) - - doc_metadata_for_summary = { - "file_name": file_name, - "mime_type": mime_type, - "document_type": "Google Drive File", - "connector_type": "Google Drive", - } - - if user_llm: - summary_content, summary_embedding = await generate_document_summary( - indexable_content, user_llm, doc_metadata_for_summary - ) - else: - logger.warning("No LLM configured — using fallback summary") - summary_content = ( - f"Google Drive File: {file_name}\n\n{indexable_content}" - ) - summary_embedding = embed_text(summary_content) + summary_embedding = embed_text(summary_content) chunks = await create_document_chunks(indexable_content) now_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S") diff --git a/surfsense_backend/app/services/linear/kb_sync_service.py b/surfsense_backend/app/services/linear/kb_sync_service.py index 471227602..9ca7c99e5 100644 --- a/surfsense_backend/app/services/linear/kb_sync_service.py +++ b/surfsense_backend/app/services/linear/kb_sync_service.py @@ -9,7 +9,6 @@ from app.utils.document_converters import ( create_document_chunks, embed_text, generate_content_hash, - generate_document_summary, generate_unique_identifier_hash, ) @@ -84,32 +83,13 @@ class LinearKBSyncService: ) content_hash = unique_hash - from app.services.llm_service import get_user_long_context_llm - user_llm = await get_user_long_context_llm( - self.db_session, - user_id, - search_space_id, - disable_streaming=True, + + + summary_content = ( + f"Linear Issue {issue_identifier}: {issue_title}\n\n{issue_content}" ) - - doc_metadata_for_summary = { - "issue_id": issue_identifier, - "issue_title": issue_title, - "document_type": "Linear Issue", - "connector_type": "Linear", - } - - if user_llm: - summary_content, summary_embedding = await generate_document_summary( - issue_content, user_llm, doc_metadata_for_summary - ) - else: - logger.warning("No LLM configured — using fallback summary") - summary_content = ( - f"Linear Issue {issue_identifier}: {issue_title}\n\n{issue_content}" - ) - summary_embedding = embed_text(summary_content) + summary_embedding = embed_text(summary_content) chunks = await create_document_chunks(issue_content) now_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S") @@ -227,30 +207,12 @@ class LinearKBSyncService: comment_count = len(formatted_issue.get("comments", [])) formatted_issue.get("description", "") - from app.services.llm_service import get_user_long_context_llm - user_llm = await get_user_long_context_llm( - self.db_session, user_id, search_space_id, disable_streaming=True + + summary_content = ( + f"Linear Issue {issue_identifier}: {issue_title}\n\n{issue_content}" ) - - if user_llm: - document_metadata_for_summary = { - "issue_id": issue_identifier, - "issue_title": issue_title, - "state": state, - "priority": priority, - "comment_count": comment_count, - "document_type": "Linear Issue", - "connector_type": "Linear", - } - summary_content, summary_embedding = await generate_document_summary( - issue_content, user_llm, document_metadata_for_summary - ) - else: - summary_content = ( - f"Linear Issue {issue_identifier}: {issue_title}\n\n{issue_content}" - ) - summary_embedding = embed_text(summary_content) + summary_embedding = embed_text(summary_content) chunks = await create_document_chunks(issue_content) diff --git a/surfsense_backend/app/services/llm_service.py b/surfsense_backend/app/services/llm_service.py index aadb60cde..099e7c573 100644 --- a/surfsense_backend/app/services/llm_service.py +++ b/surfsense_backend/app/services/llm_service.py @@ -68,7 +68,6 @@ def _is_interactive_auth_provider( class LLMRole: AGENT = "agent" # For agent/chat operations - DOCUMENT_SUMMARY = "document_summary" # For document summarization def get_global_llm_config(llm_config_id: int) -> dict | None: @@ -266,7 +265,7 @@ async def get_search_space_llm_instance( Args: session: Database session search_space_id: Search Space ID - role: LLM role ('agent' or 'document_summary') + role: LLM role ('agent') Returns: ChatLiteLLM or ChatLiteLLMRouter instance, or None if not found @@ -283,11 +282,8 @@ async def get_search_space_llm_instance( return None # Get the appropriate LLM config ID based on role - llm_config_id = None if role == LLMRole.AGENT: llm_config_id = search_space.agent_llm_id - elif role == LLMRole.DOCUMENT_SUMMARY: - llm_config_id = search_space.document_summary_llm_id else: logger.error(f"Invalid LLM role: {role}") return None @@ -470,20 +466,13 @@ async def get_search_space_llm_instance( async def get_agent_llm( - session: AsyncSession, search_space_id: int -) -> ChatLiteLLM | ChatLiteLLMRouter | None: - """Get the search space's agent LLM instance for chat operations.""" - return await get_search_space_llm_instance(session, search_space_id, LLMRole.AGENT) - - -async def get_document_summary_llm( session: AsyncSession, search_space_id: int, disable_streaming: bool = False ) -> ChatLiteLLM | ChatLiteLLMRouter | None: - """Get the search space's document summary LLM instance.""" + """Get the search space's agent LLM instance for chat operations.""" return await get_search_space_llm_instance( session, search_space_id, - LLMRole.DOCUMENT_SUMMARY, + LLMRole.AGENT, disable_streaming=disable_streaming, ) @@ -645,22 +634,6 @@ async def get_vision_llm( return None -# Backward-compatible alias (LLM preferences are now per-search-space, not per-user) -async def get_user_long_context_llm( - session: AsyncSession, - user_id: str, - search_space_id: int, - disable_streaming: bool = False, -) -> ChatLiteLLM | ChatLiteLLMRouter | None: - """ - Deprecated: Use get_document_summary_llm instead. - The user_id parameter is ignored as LLM preferences are now per-search-space. - """ - return await get_document_summary_llm( - session, search_space_id, disable_streaming=disable_streaming - ) - - def get_planner_llm() -> ChatLiteLLM | None: """Return a planner LLM instance from the first global config marked ``is_planner: true``, or ``None`` if no planner config is defined. diff --git a/surfsense_backend/app/services/notion/kb_sync_service.py b/surfsense_backend/app/services/notion/kb_sync_service.py index b10d1b157..826d01a15 100644 --- a/surfsense_backend/app/services/notion/kb_sync_service.py +++ b/surfsense_backend/app/services/notion/kb_sync_service.py @@ -8,7 +8,6 @@ from app.utils.document_converters import ( create_document_chunks, embed_text, generate_content_hash, - generate_document_summary, generate_unique_identifier_hash, ) @@ -73,30 +72,11 @@ class NotionKBSyncService: ) content_hash = unique_hash - from app.services.llm_service import get_user_long_context_llm - user_llm = await get_user_long_context_llm( - self.db_session, - user_id, - search_space_id, - disable_streaming=True, - ) - doc_metadata_for_summary = { - "page_title": page_title, - "page_id": page_id, - "document_type": "Notion Page", - "connector_type": "Notion", - } - if user_llm: - summary_content, summary_embedding = await generate_document_summary( - markdown_content, user_llm, doc_metadata_for_summary - ) - else: - logger.warning("No LLM configured — using fallback summary") - summary_content = f"Notion Page: {page_title}\n\n{markdown_content}" - summary_embedding = embed_text(summary_content) + summary_content = f"Notion Page: {page_title}\n\n{markdown_content}" + summary_embedding = embed_text(summary_content) chunks = await create_document_chunks(markdown_content) now_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S") @@ -245,31 +225,11 @@ class NotionKBSyncService: f"Final content length: {len(full_content)} chars, verified={content_verified}" ) - from app.services.llm_service import get_user_long_context_llm logger.debug("Generating summary and embeddings") - user_llm = await get_user_long_context_llm( - self.db_session, - user_id, - search_space_id, - disable_streaming=True, # disable streaming to avoid leaking into the chat - ) - if user_llm: - document_metadata_for_summary = { - "page_title": document.document_metadata.get("page_title"), - "page_id": document.document_metadata.get("page_id"), - "document_type": "Notion Page", - "connector_type": "Notion", - } - summary_content, summary_embedding = await generate_document_summary( - full_content, user_llm, document_metadata_for_summary - ) - logger.debug(f"Generated summary length: {len(summary_content)} chars") - else: - logger.warning("No LLM configured - using fallback summary") - summary_content = f"Notion Page: {document.document_metadata.get('page_title')}\n\n{full_content}" - summary_embedding = embed_text(summary_content) + summary_content = f"Notion Page: {document.document_metadata.get('page_title')}\n\n{full_content}" + summary_embedding = embed_text(summary_content) logger.debug("Creating new chunks") chunks = await create_document_chunks(full_content) diff --git a/surfsense_backend/app/services/obsidian_plugin_indexer.py b/surfsense_backend/app/services/obsidian_plugin_indexer.py index 0fc4f30f4..13f43d1ee 100644 --- a/surfsense_backend/app/services/obsidian_plugin_indexer.py +++ b/surfsense_backend/app/services/obsidian_plugin_indexer.py @@ -233,18 +233,6 @@ async def _resolve_attachment_vision_llm( return await get_vision_llm(session, search_space_id) -async def _resolve_summary_llm( - session: AsyncSession, *, user_id: str, search_space_id: int, should_summarize: bool -): - """Fetch summary LLM only when indexing summary is enabled.""" - if not should_summarize: - return None - - from app.services.llm_service import get_user_long_context_llm - - return await get_user_long_context_llm(session, user_id, search_space_id) - - def _require_extracted_attachment_content( *, content: str, etl_meta: dict[str, Any], path: str ) -> str: @@ -349,13 +337,6 @@ async def upsert_note( path=payload.path, ) - llm = await _resolve_summary_llm( - session, - user_id=str(user_id), - search_space_id=search_space_id, - should_summarize=connector.enable_summary, - ) - document_string = _build_document_string( payload, vault_name, content_override=content_for_index ) @@ -374,8 +355,6 @@ async def upsert_note( search_space_id=search_space_id, connector_id=connector.id, created_by_id=str(user_id), - should_summarize=connector.enable_summary, - fallback_summary=f"Obsidian Note: {payload.name}\n\n{content_for_index}", metadata=metadata, ) @@ -388,7 +367,7 @@ async def upsert_note( document = prepared[0] - return await pipeline.index(document, connector_doc, llm) + return await pipeline.index(document, connector_doc) async def rename_note( diff --git a/surfsense_backend/app/services/onedrive/kb_sync_service.py b/surfsense_backend/app/services/onedrive/kb_sync_service.py index 731f081dd..66a885b1c 100644 --- a/surfsense_backend/app/services/onedrive/kb_sync_service.py +++ b/surfsense_backend/app/services/onedrive/kb_sync_service.py @@ -10,7 +10,6 @@ from app.utils.document_converters import ( create_document_chunks, embed_text, generate_content_hash, - generate_document_summary, ) logger = logging.getLogger(__name__) @@ -73,30 +72,11 @@ class OneDriveKBSyncService: ) content_hash = unique_hash - from app.services.llm_service import get_user_long_context_llm - user_llm = await get_user_long_context_llm( - self.db_session, - user_id, - search_space_id, - disable_streaming=True, - ) - doc_metadata_for_summary = { - "file_name": file_name, - "mime_type": mime_type, - "document_type": "OneDrive File", - "connector_type": "OneDrive", - } - if user_llm: - summary_content, summary_embedding = await generate_document_summary( - indexable_content, user_llm, doc_metadata_for_summary - ) - else: - logger.warning("No LLM configured — using fallback summary") - summary_content = f"OneDrive File: {file_name}\n\n{indexable_content}" - summary_embedding = await asyncio.to_thread(embed_text, summary_content) + summary_content = f"OneDrive File: {file_name}\n\n{indexable_content}" + summary_embedding = await asyncio.to_thread(embed_text, summary_content) chunks = await create_document_chunks(indexable_content) now_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S") diff --git a/surfsense_backend/app/services/task_dispatcher.py b/surfsense_backend/app/services/task_dispatcher.py index 210084102..43957be03 100644 --- a/surfsense_backend/app/services/task_dispatcher.py +++ b/surfsense_backend/app/services/task_dispatcher.py @@ -18,7 +18,6 @@ class TaskDispatcher(Protocol): filename: str, search_space_id: int, user_id: str, - should_summarize: bool = False, use_vision_llm: bool = False, processing_mode: str = "basic", ) -> None: ... @@ -35,7 +34,6 @@ class CeleryTaskDispatcher: filename: str, search_space_id: int, user_id: str, - should_summarize: bool = False, use_vision_llm: bool = False, processing_mode: str = "basic", ) -> None: @@ -49,7 +47,6 @@ class CeleryTaskDispatcher: filename=filename, search_space_id=search_space_id, user_id=user_id, - should_summarize=should_summarize, use_vision_llm=use_vision_llm, processing_mode=processing_mode, ) diff --git a/surfsense_backend/app/tasks/celery_tasks/document_reindex_tasks.py b/surfsense_backend/app/tasks/celery_tasks/document_reindex_tasks.py index 5d6bde6c1..d36a7c05f 100644 --- a/surfsense_backend/app/tasks/celery_tasks/document_reindex_tasks.py +++ b/surfsense_backend/app/tasks/celery_tasks/document_reindex_tasks.py @@ -9,7 +9,6 @@ from sqlalchemy.orm import selectinload from app.celery_app import celery_app from app.db import Document from app.indexing_pipeline.adapters.file_upload_adapter import UploadDocumentAdapter -from app.services.llm_service import get_user_long_context_llm from app.services.task_logging_service import TaskLoggingService from app.tasks.celery_tasks import get_celery_session_maker, run_async_celery_task @@ -68,12 +67,8 @@ async def _reindex_document(document_id: int, user_id: str): logger.info(f"Reindexing document {document_id} ({document.title})") - user_llm = await get_user_long_context_llm( - session, user_id, document.search_space_id - ) - adapter = UploadDocumentAdapter(session) - await adapter.reindex(document=document, llm=user_llm) + await adapter.reindex(document=document) await task_logger.log_task_success( log_entry, diff --git a/surfsense_backend/app/tasks/celery_tasks/document_tasks.py b/surfsense_backend/app/tasks/celery_tasks/document_tasks.py index 4781ca6a5..5b7b62f35 100644 --- a/surfsense_backend/app/tasks/celery_tasks/document_tasks.py +++ b/surfsense_backend/app/tasks/celery_tasks/document_tasks.py @@ -765,7 +765,6 @@ def process_file_upload_with_document_task( filename: str, search_space_id: int, user_id: str, - should_summarize: bool = False, use_vision_llm: bool = False, processing_mode: str = "basic", ): @@ -782,7 +781,6 @@ def process_file_upload_with_document_task( filename: Original filename search_space_id: ID of the search space user_id: ID of the user - should_summarize: Whether to generate an LLM summary """ import traceback @@ -814,7 +812,6 @@ def process_file_upload_with_document_task( filename, search_space_id, user_id, - should_summarize=should_summarize, use_vision_llm=use_vision_llm, processing_mode=processing_mode, ) @@ -850,7 +847,6 @@ async def _process_file_with_document( filename: str, search_space_id: int, user_id: str, - should_summarize: bool = False, use_vision_llm: bool = False, processing_mode: str = "basic", ): @@ -954,7 +950,6 @@ async def _process_file_with_document( task_logger=task_logger, log_entry=log_entry, notification=notification, - should_summarize=should_summarize, use_vision_llm=use_vision_llm, processing_mode=processing_mode, ) @@ -1258,7 +1253,6 @@ def index_local_folder_task( exclude_patterns: list[str] | None = None, file_extensions: list[str] | None = None, root_folder_id: int | None = None, - enable_summary: bool = False, target_file_paths: list[str] | None = None, ): """Celery task to index a local folder. Config is passed directly — no connector row.""" @@ -1271,7 +1265,6 @@ def index_local_folder_task( exclude_patterns=exclude_patterns, file_extensions=file_extensions, root_folder_id=root_folder_id, - enable_summary=enable_summary, target_file_paths=target_file_paths, ) ) @@ -1285,7 +1278,6 @@ async def _index_local_folder_async( exclude_patterns: list[str] | None = None, file_extensions: list[str] | None = None, root_folder_id: int | None = None, - enable_summary: bool = False, target_file_paths: list[str] | None = None, ): """Run local folder indexing with notification + heartbeat.""" @@ -1343,8 +1335,7 @@ async def _index_local_folder_async( exclude_patterns=exclude_patterns, file_extensions=file_extensions, root_folder_id=root_folder_id, - enable_summary=enable_summary, - target_file_paths=target_file_paths, + target_file_paths=target_file_paths, on_heartbeat_callback=_heartbeat_progress if (is_batch or is_full_scan) else None, @@ -1400,7 +1391,6 @@ def index_uploaded_folder_files_task( user_id: str, folder_name: str, root_folder_id: int, - enable_summary: bool, file_mappings: list[dict], use_vision_llm: bool = False, processing_mode: str = "basic", @@ -1412,7 +1402,6 @@ def index_uploaded_folder_files_task( user_id=user_id, folder_name=folder_name, root_folder_id=root_folder_id, - enable_summary=enable_summary, file_mappings=file_mappings, use_vision_llm=use_vision_llm, processing_mode=processing_mode, @@ -1425,7 +1414,6 @@ async def _index_uploaded_folder_files_async( user_id: str, folder_name: str, root_folder_id: int, - enable_summary: bool, file_mappings: list[dict], use_vision_llm: bool = False, processing_mode: str = "basic", @@ -1475,8 +1463,7 @@ async def _index_uploaded_folder_files_async( user_id=user_id, folder_name=folder_name, root_folder_id=root_folder_id, - enable_summary=enable_summary, - file_mappings=file_mappings, + file_mappings=file_mappings, on_heartbeat_callback=_heartbeat_progress, use_vision_llm=use_vision_llm, processing_mode=processing_mode, @@ -1563,12 +1550,10 @@ async def _ai_sort_search_space_async(search_space_id: int, user_id: str): t_start = time.perf_counter() try: from app.services.ai_file_sort_service import ai_sort_all_documents - from app.services.llm_service import get_document_summary_llm + from app.services.llm_service import get_agent_llm async with get_celery_session_maker()() as session: - llm = await get_document_summary_llm( - session, search_space_id, disable_streaming=True - ) + llm = await get_agent_llm(session, search_space_id, disable_streaming=True) if llm is None: logger.warning( "No LLM configured for search_space=%d, skipping AI sort", @@ -1604,7 +1589,7 @@ def ai_sort_document_task(self, search_space_id: int, user_id: str, document_id: async def _ai_sort_document_async(search_space_id: int, user_id: str, document_id: int): from app.db import Document from app.services.ai_file_sort_service import ai_sort_document - from app.services.llm_service import get_document_summary_llm + from app.services.llm_service import get_agent_llm async with get_celery_session_maker()() as session: document = await session.get(Document, document_id) @@ -1612,9 +1597,7 @@ async def _ai_sort_document_async(search_space_id: int, user_id: str, document_i logger.warning("Document %d not found, skipping AI sort", document_id) return - llm = await get_document_summary_llm( - session, search_space_id, disable_streaming=True - ) + llm = await get_agent_llm(session, search_space_id, disable_streaming=True) if llm is None: logger.warning( "No LLM for search_space=%d, skipping AI sort of doc=%d", diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/input_state.py b/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/input_state.py index 0c6704bd1..c9ef6edd6 100644 --- a/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/input_state.py +++ b/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/input_state.py @@ -62,6 +62,7 @@ async def build_new_chat_input_state( user_image_data_urls: list[str] | None, mentioned_document_ids: list[int] | None, mentioned_folder_ids: list[int] | None, + mentioned_connectors: list[dict[str, Any]] | None, mentioned_documents: list[dict[str, Any]] | None, needs_history_bootstrap: bool, thread_visibility: ChatVisibility, @@ -110,6 +111,7 @@ async def build_new_chat_input_state( final_query = _render_query_with_context( agent_user_query=agent_user_query, + mentioned_connectors=mentioned_connectors, recent_reports=recent_reports, ) @@ -196,11 +198,16 @@ async def _resolve_mentions_for_query( def _render_query_with_context( *, agent_user_query: str, + mentioned_connectors: list[dict[str, Any]] | None, recent_reports: list[Report], ) -> str: - """Prepend recent-reports XML block to the user query.""" + """Prepend connector/report XML context blocks to the user query.""" context_parts: list[str] = [] + connector_context = _render_mentioned_connectors(mentioned_connectors) + if connector_context: + context_parts.append(connector_context) + if recent_reports: report_lines: list[str] = [] for r in recent_reports: @@ -225,3 +232,40 @@ def _render_query_with_context( return f"{context}\n\n{agent_user_query}" return agent_user_query + + +def _render_mentioned_connectors( + mentioned_connectors: list[dict[str, Any]] | None, +) -> str | None: + """Render selected connector account metadata for connector-backed tools.""" + if not mentioned_connectors: + return None + + connector_lines: list[str] = [] + for connector in mentioned_connectors: + if not isinstance(connector, dict): + continue + connector_id = connector.get("id") + connector_type = connector.get("connector_type") or connector.get( + "document_type" + ) + account_name = connector.get("account_name") or connector.get("title") + if connector_id is None or connector_type is None: + continue + connector_lines.append( + f' - connector_id={connector_id}, connector_type="{connector_type}", ' + f'account_name="{account_name or ""}"' + ) + + if not connector_lines: + return None + + return ( + "\n" + "The user selected these exact connector accounts with @. " + "These entries are selection metadata, not retrieved connector content. " + "When a connector-backed tool needs an account, use the matching " + "connector_id from this list if the tool supports connector_id:\n" + + "\n".join(connector_lines) + + "\n" + ) diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/orchestrator.py b/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/orchestrator.py index 1892320d3..9c25218bf 100644 --- a/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/orchestrator.py +++ b/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/orchestrator.py @@ -124,6 +124,8 @@ async def stream_new_chat( llm_config_id: int = -1, mentioned_document_ids: list[int] | None = None, mentioned_folder_ids: list[int] | None = None, + mentioned_connector_ids: list[int] | None = None, + mentioned_connectors: list[dict[str, Any]] | None = None, mentioned_documents: list[dict[str, Any]] | None = None, checkpoint_id: str | None = None, needs_history_bootstrap: bool = False, @@ -435,6 +437,7 @@ async def stream_new_chat( user_image_data_urls=user_image_data_urls, mentioned_document_ids=mentioned_document_ids, mentioned_folder_ids=mentioned_folder_ids, + mentioned_connectors=mentioned_connectors, mentioned_documents=mentioned_documents, needs_history_bootstrap=needs_history_bootstrap, thread_visibility=visibility, @@ -588,6 +591,8 @@ async def stream_new_chat( mentioned_document_ids=mentioned_document_ids, accepted_folder_ids=accepted_folder_ids, mentioned_folder_ids=mentioned_folder_ids, + mentioned_connector_ids=mentioned_connector_ids, + mentioned_connectors=mentioned_connectors, request_id=request_id, turn_id=stream_result.turn_id, ) diff --git a/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/runtime_context.py b/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/runtime_context.py index cf1e8c3fb..66233dec8 100644 --- a/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/runtime_context.py +++ b/surfsense_backend/app/tasks/chat/streaming/flows/new_chat/runtime_context.py @@ -8,6 +8,8 @@ mention lists / request ids / turn ids without rebuilding the graph. from __future__ import annotations +from typing import Any + from app.agents.new_chat.context import SurfSenseContextSchema @@ -17,6 +19,8 @@ def build_new_chat_runtime_context( mentioned_document_ids: list[int] | None, accepted_folder_ids: list[int], mentioned_folder_ids: list[int] | None, + mentioned_connector_ids: list[int] | None, + mentioned_connectors: list[dict[str, Any]] | None, request_id: str | None, turn_id: str, ) -> SurfSenseContextSchema: @@ -31,6 +35,8 @@ def build_new_chat_runtime_context( search_space_id=search_space_id, mentioned_document_ids=list(mentioned_document_ids or []), mentioned_folder_ids=list(accepted_folder_ids or mentioned_folder_ids or []), + mentioned_connector_ids=list(mentioned_connector_ids or []), + mentioned_connectors=list(mentioned_connectors or []), request_id=request_id, turn_id=turn_id, ) diff --git a/surfsense_backend/app/tasks/connector_indexers/airtable_indexer.py b/surfsense_backend/app/tasks/connector_indexers/airtable_indexer.py index f77a0632a..ac38b7bf7 100644 --- a/surfsense_backend/app/tasks/connector_indexers/airtable_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/airtable_indexer.py @@ -14,13 +14,11 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.connectors.airtable_history import AirtableHistoryConnector from app.db import Document, DocumentStatus, DocumentType, SearchSourceConnectorType -from app.services.llm_service import get_user_long_context_llm from app.services.task_logging_service import TaskLoggingService from app.utils.document_converters import ( create_document_chunks, embed_text, generate_content_hash, - generate_document_summary, generate_unique_identifier_hash, ) @@ -394,29 +392,10 @@ async def index_airtable_records( document.status = DocumentStatus.processing() await session.commit() - # Heavy processing (LLM, embeddings, chunks) - user_llm = await get_user_long_context_llm( - session, user_id, search_space_id - ) + # Heavy processing (embeddings, chunks) - if user_llm and connector.enable_summary: - document_metadata_for_summary = { - "record_id": item["record_id"], - "created_time": item["record"].get("CREATED_TIME()", ""), - "document_type": "Airtable Record", - "connector_type": "Airtable", - } - ( - summary_content, - summary_embedding, - ) = await generate_document_summary( - item["markdown_content"], - user_llm, - document_metadata_for_summary, - ) - else: - summary_content = f"Airtable Record: {item['record_id']}\n\n{item['markdown_content']}" - summary_embedding = embed_text(summary_content) + summary_content = f"Airtable Record: {item['record_id']}\n\n{item['markdown_content']}" + summary_embedding = embed_text(summary_content) chunks = await create_document_chunks(item["markdown_content"]) diff --git a/surfsense_backend/app/tasks/connector_indexers/bookstack_indexer.py b/surfsense_backend/app/tasks/connector_indexers/bookstack_indexer.py index 8e64e56ba..74234a3b9 100644 --- a/surfsense_backend/app/tasks/connector_indexers/bookstack_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/bookstack_indexer.py @@ -15,13 +15,11 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.connectors.bookstack_connector import BookStackConnector from app.db import Document, DocumentStatus, DocumentType, SearchSourceConnectorType -from app.services.llm_service import get_user_long_context_llm from app.services.task_logging_service import TaskLoggingService from app.utils.document_converters import ( create_document_chunks, embed_text, generate_content_hash, - generate_document_summary, generate_unique_identifier_hash, ) @@ -384,10 +382,7 @@ async def index_bookstack_pages( document.status = DocumentStatus.processing() await session.commit() - # Heavy processing (LLM, embeddings, chunks) - user_llm = await get_user_long_context_llm( - session, user_id, search_space_id - ) + # Heavy processing (embeddings, chunks) # Build document metadata doc_metadata = { @@ -403,23 +398,8 @@ async def index_bookstack_pages( "connector_id": connector_id, } - if user_llm and connector.enable_summary: - summary_metadata = { - "page_name": item["page_name"], - "page_id": item["page_id"], - "book_id": item["book_id"], - "document_type": "BookStack Page", - "connector_type": "BookStack", - } - ( - summary_content, - summary_embedding, - ) = await generate_document_summary( - item["full_content"], user_llm, summary_metadata - ) - else: - summary_content = f"BookStack Page: {item['page_name']}\n\nBook ID: {item['book_id']}\n\n{item['full_content']}" - summary_embedding = embed_text(summary_content) + summary_content = f"BookStack Page: {item['page_name']}\n\nBook ID: {item['book_id']}\n\n{item['full_content']}" + summary_embedding = embed_text(summary_content) # Process chunks - using the full page content chunks = await create_document_chunks(item["full_content"]) diff --git a/surfsense_backend/app/tasks/connector_indexers/clickup_indexer.py b/surfsense_backend/app/tasks/connector_indexers/clickup_indexer.py index 5a6cc3485..7b40a4b22 100644 --- a/surfsense_backend/app/tasks/connector_indexers/clickup_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/clickup_indexer.py @@ -16,13 +16,11 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.connectors.clickup_history import ClickUpHistoryConnector from app.db import Document, DocumentStatus, DocumentType, SearchSourceConnectorType -from app.services.llm_service import get_user_long_context_llm from app.services.task_logging_service import TaskLoggingService from app.utils.document_converters import ( create_document_chunks, embed_text, generate_content_hash, - generate_document_summary, generate_unique_identifier_hash, ) @@ -393,32 +391,10 @@ async def index_clickup_tasks( document.status = DocumentStatus.processing() await session.commit() - # Heavy processing (LLM, embeddings, chunks) - user_llm = await get_user_long_context_llm( - session, user_id, search_space_id - ) + # Heavy processing (embeddings, chunks) - if user_llm and connector.enable_summary: - document_metadata_for_summary = { - "task_id": item["task_id"], - "task_name": item["task_name"], - "task_status": item["task_status"], - "task_priority": item["task_priority"], - "task_list": item["task_list_name"], - "task_space": item["task_space_name"], - "assignees": len(item["task_assignees"]), - "document_type": "ClickUp Task", - "connector_type": "ClickUp", - } - ( - summary_content, - summary_embedding, - ) = await generate_document_summary( - item["task_content"], user_llm, document_metadata_for_summary - ) - else: - summary_content = item["task_content"] - summary_embedding = embed_text(item["task_content"]) + summary_content = item["task_content"] + summary_embedding = embed_text(item["task_content"]) chunks = await create_document_chunks(item["task_content"]) diff --git a/surfsense_backend/app/tasks/connector_indexers/confluence_indexer.py b/surfsense_backend/app/tasks/connector_indexers/confluence_indexer.py index a8c2e3c18..5dbe4caec 100644 --- a/surfsense_backend/app/tasks/connector_indexers/confluence_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/confluence_indexer.py @@ -14,7 +14,6 @@ from app.indexing_pipeline.indexing_pipeline_service import ( IndexingPipelineService, PlaceholderInfo, ) -from app.services.llm_service import get_user_long_context_llm from app.services.task_logging_service import TaskLoggingService from .base import ( @@ -36,7 +35,6 @@ def _build_connector_doc( connector_id: int, search_space_id: int, user_id: str, - enable_summary: bool, ) -> ConnectorDocument: """Map a raw Confluence page dict to a ConnectorDocument.""" page_id = page.get("id", "") @@ -54,10 +52,6 @@ def _build_connector_doc( "connector_type": "Confluence", } - fallback_summary = ( - f"Confluence Page: {page_title}\n\nSpace ID: {space_id}\n\n{full_content}" - ) - return ConnectorDocument( title=page_title, source_markdown=full_content, @@ -66,8 +60,6 @@ def _build_connector_doc( search_space_id=search_space_id, connector_id=connector_id, created_by_id=user_id, - should_summarize=enable_summary, - fallback_summary=fallback_summary, metadata=metadata, ) @@ -268,8 +260,7 @@ async def index_confluence_pages( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=connector.enable_summary, - ) + ) with session.no_autoflush: duplicate_by_content = await check_duplicate_document_by_hash( @@ -297,12 +288,8 @@ async def index_confluence_pages( await pipeline.migrate_legacy_docs(connector_docs) - async def _get_llm(s: AsyncSession): - return await get_user_long_context_llm(s, user_id, search_space_id) - _, documents_indexed, documents_failed = await pipeline.index_batch_parallel( connector_docs, - _get_llm, max_concurrency=3, on_heartbeat=on_heartbeat_callback, heartbeat_interval=HEARTBEAT_INTERVAL_SECONDS, diff --git a/surfsense_backend/app/tasks/connector_indexers/dropbox_indexer.py b/surfsense_backend/app/tasks/connector_indexers/dropbox_indexer.py index 9f8c1a33a..6e61bce18 100644 --- a/surfsense_backend/app/tasks/connector_indexers/dropbox_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/dropbox_indexer.py @@ -27,7 +27,6 @@ from app.db import Document, DocumentStatus, DocumentType, SearchSourceConnector from app.indexing_pipeline.connector_document import ConnectorDocument from app.indexing_pipeline.document_hashing import compute_identifier_hash from app.indexing_pipeline.indexing_pipeline_service import IndexingPipelineService -from app.services.llm_service import get_user_long_context_llm from app.services.page_limit_service import PageLimitService from app.services.task_logging_service import TaskLoggingService from app.tasks.connector_indexers.base import ( @@ -126,7 +125,6 @@ def _build_connector_doc( connector_id: int, search_space_id: int, user_id: str, - enable_summary: bool, ) -> ConnectorDocument: file_id = file.get("id", "") file_name = file.get("name", "Unknown") @@ -138,8 +136,6 @@ def _build_connector_doc( "connector_type": "Dropbox", } - fallback_summary = f"File: {file_name}\n\n{markdown[:4000]}" - return ConnectorDocument( title=file_name, source_markdown=markdown, @@ -148,8 +144,6 @@ def _build_connector_doc( search_space_id=search_space_id, connector_id=connector_id, created_by_id=user_id, - should_summarize=enable_summary, - fallback_summary=fallback_summary, metadata=metadata, ) @@ -161,7 +155,6 @@ async def _download_files_parallel( connector_id: int, search_space_id: int, user_id: str, - enable_summary: bool, max_concurrency: int = 3, on_heartbeat: HeartbeatCallbackType | None = None, vision_llm=None, @@ -191,7 +184,6 @@ async def _download_files_parallel( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=enable_summary, ) async with hb_lock: completed_count += 1 @@ -223,7 +215,6 @@ async def _download_and_index( connector_id: int, search_space_id: int, user_id: str, - enable_summary: bool, on_heartbeat: HeartbeatCallbackType | None = None, vision_llm=None, ) -> tuple[int, int]: @@ -234,7 +225,6 @@ async def _download_and_index( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=enable_summary, on_heartbeat=on_heartbeat, vision_llm=vision_llm, ) @@ -243,13 +233,8 @@ async def _download_and_index( batch_failed = 0 if connector_docs: pipeline = IndexingPipelineService(session) - - async def _get_llm(s): - return await get_user_long_context_llm(s, user_id, search_space_id) - _, batch_indexed, batch_failed = await pipeline.index_batch_parallel( connector_docs, - _get_llm, max_concurrency=3, on_heartbeat=on_heartbeat, ) @@ -289,7 +274,6 @@ async def _index_with_delta_sync( log_entry: object, max_files: int, on_heartbeat_callback: HeartbeatCallbackType | None = None, - enable_summary: bool = True, vision_llm=None, ) -> tuple[int, int, int, str]: """Delta sync using Dropbox cursor-based change tracking. @@ -361,7 +345,6 @@ async def _index_with_delta_sync( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=enable_summary, on_heartbeat=on_heartbeat_callback, vision_llm=vision_llm, ) @@ -388,7 +371,6 @@ async def _index_full_scan( include_subfolders: bool = True, incremental_sync: bool = True, on_heartbeat_callback: HeartbeatCallbackType | None = None, - enable_summary: bool = True, vision_llm=None, ) -> tuple[int, int, int]: """Full scan indexing of a folder. @@ -473,7 +455,6 @@ async def _index_full_scan( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=enable_summary, on_heartbeat=on_heartbeat_callback, vision_llm=vision_llm, ) @@ -502,7 +483,6 @@ async def _index_selected_files( connector_id: int, search_space_id: int, user_id: str, - enable_summary: bool, incremental_sync: bool = True, on_heartbeat: HeartbeatCallbackType | None = None, vision_llm=None, @@ -563,7 +543,6 @@ async def _index_selected_files( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=enable_summary, on_heartbeat=on_heartbeat, vision_llm=vision_llm, ) @@ -629,7 +608,6 @@ async def index_dropbox_files( ) return 0, 0, error_msg, 0 - connector_enable_summary = getattr(connector, "enable_summary", True) connector_enable_vision_llm = getattr(connector, "enable_vision_llm", False) vision_llm = None if connector_enable_vision_llm: @@ -664,7 +642,6 @@ async def index_dropbox_files( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=connector_enable_summary, incremental_sync=incremental_sync, vision_llm=vision_llm, ) @@ -700,7 +677,6 @@ async def index_dropbox_files( task_logger, log_entry, max_files, - enable_summary=connector_enable_summary, vision_llm=vision_llm, ) folder_cursors[folder_path] = new_cursor @@ -720,7 +696,6 @@ async def index_dropbox_files( max_files, include_subfolders, incremental_sync=incremental_sync, - enable_summary=connector_enable_summary, vision_llm=vision_llm, ) total_unsupported += unsup diff --git a/surfsense_backend/app/tasks/connector_indexers/github_indexer.py b/surfsense_backend/app/tasks/connector_indexers/github_indexer.py index ae24d750b..1d0b004d8 100644 --- a/surfsense_backend/app/tasks/connector_indexers/github_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/github_indexer.py @@ -18,13 +18,11 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.connectors.github_connector import GitHubConnector from app.db import Document, DocumentStatus, DocumentType, SearchSourceConnectorType -from app.services.llm_service import get_user_long_context_llm from app.services.task_logging_service import TaskLoggingService from app.utils.document_converters import ( create_document_chunks, embed_text, generate_content_hash, - generate_document_summary, generate_unique_identifier_hash, ) @@ -351,42 +349,14 @@ async def index_github_repos( document.status = DocumentStatus.processing() await session.commit() - # Heavy processing (LLM, embeddings, chunks) - user_llm = await get_user_long_context_llm( - session, user_id, search_space_id + # Heavy processing (embeddings, chunks) + + summary_text = ( + f"# GitHub Repository: {repo_full_name}\n\n" + f"## Summary\n{digest.summary}\n\n" + f"## File Structure\n{digest.tree}" ) - - document_metadata_for_summary = { - "repository": repo_full_name, - "document_type": "GitHub Repository", - "connector_type": "GitHub", - "ingestion_method": "gitingest", - "file_tree": digest.tree[:2000] - if len(digest.tree) > 2000 - else digest.tree, - "estimated_tokens": digest.estimated_tokens, - } - - if user_llm and connector.enable_summary: - # Prepare content for summarization - summary_content = digest.full_digest - if len(summary_content) > MAX_DIGEST_CHARS: - summary_content = ( - f"# Repository: {repo_full_name}\n\n" - f"## File Structure\n\n{digest.tree}\n\n" - f"## File Contents (truncated)\n\n{digest.content[: MAX_DIGEST_CHARS - len(digest.tree) - 200]}..." - ) - - summary_text, summary_embedding = await generate_document_summary( - summary_content, user_llm, document_metadata_for_summary - ) - else: - summary_text = ( - f"# GitHub Repository: {repo_full_name}\n\n" - f"## Summary\n{digest.summary}\n\n" - f"## File Structure\n{digest.tree}" - ) - summary_embedding = embed_text(summary_text) + summary_embedding = embed_text(summary_text) # Chunk the full digest content for granular search try: diff --git a/surfsense_backend/app/tasks/connector_indexers/google_calendar_indexer.py b/surfsense_backend/app/tasks/connector_indexers/google_calendar_indexer.py index 3c9f27303..e0053f614 100644 --- a/surfsense_backend/app/tasks/connector_indexers/google_calendar_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/google_calendar_indexer.py @@ -2,7 +2,7 @@ Google Calendar connector indexer. Uses the shared IndexingPipelineService for document deduplication, -summarization, chunking, and embedding. +chunking, and embedding. """ from collections.abc import Awaitable, Callable @@ -21,7 +21,6 @@ from app.indexing_pipeline.indexing_pipeline_service import ( PlaceholderInfo, ) from app.services.composio_service import ComposioService -from app.services.llm_service import get_user_long_context_llm from app.services.task_logging_service import TaskLoggingService from app.utils.google_credentials import COMPOSIO_GOOGLE_CONNECTOR_TYPES @@ -53,7 +52,6 @@ def _build_connector_doc( connector_id: int, search_space_id: int, user_id: str, - enable_summary: bool, ) -> ConnectorDocument: """Map a raw Google Calendar API event dict to a ConnectorDocument.""" event_id = event.get("id", "") @@ -78,8 +76,6 @@ def _build_connector_doc( "connector_type": "Google Calendar", } - fallback_summary = f"Google Calendar Event: {event_summary}\n\n{event_markdown}" - return ConnectorDocument( title=event_summary, source_markdown=event_markdown, @@ -88,8 +84,6 @@ def _build_connector_doc( search_space_id=search_space_id, connector_id=connector_id, created_by_id=user_id, - should_summarize=enable_summary, - fallback_summary=fallback_summary, metadata=metadata, ) @@ -420,8 +414,7 @@ async def index_google_calendar_events( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=connector.enable_summary, - ) + ) with session.no_autoflush: duplicate = await check_duplicate_document_by_hash( @@ -448,13 +441,8 @@ async def index_google_calendar_events( # ── Pipeline: migrate legacy docs + parallel index ───────────── await pipeline.migrate_legacy_docs(connector_docs) - - async def _get_llm(s): - return await get_user_long_context_llm(s, user_id, search_space_id) - _, documents_indexed, documents_failed = await pipeline.index_batch_parallel( connector_docs, - _get_llm, max_concurrency=3, on_heartbeat=on_heartbeat_callback, heartbeat_interval=HEARTBEAT_INTERVAL_SECONDS, diff --git a/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py b/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py index 686f13d9e..e20518ab0 100644 --- a/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/google_drive_indexer.py @@ -40,7 +40,6 @@ from app.indexing_pipeline.indexing_pipeline_service import ( PlaceholderInfo, ) from app.services.composio_service import ComposioService -from app.services.llm_service import get_user_long_context_llm from app.services.page_limit_service import PageLimitService from app.services.task_logging_service import TaskLoggingService from app.tasks.connector_indexers.base import ( @@ -381,7 +380,6 @@ def _build_connector_doc( connector_id: int, search_space_id: int, user_id: str, - enable_summary: bool, ) -> ConnectorDocument: """Build a ConnectorDocument from Drive file metadata + extracted markdown.""" file_id = file.get("id", "") @@ -394,8 +392,6 @@ def _build_connector_doc( "connector_type": "Google Drive", } - fallback_summary = f"File: {file_name}\n\n{markdown[:4000]}" - return ConnectorDocument( title=file_name, source_markdown=markdown, @@ -404,8 +400,6 @@ def _build_connector_doc( search_space_id=search_space_id, connector_id=connector_id, created_by_id=user_id, - should_summarize=enable_summary, - fallback_summary=fallback_summary, metadata=metadata, ) @@ -461,7 +455,6 @@ async def _download_files_parallel( connector_id: int, search_space_id: int, user_id: str, - enable_summary: bool, max_concurrency: int = 3, on_heartbeat: HeartbeatCallbackType | None = None, vision_llm=None, @@ -494,7 +487,6 @@ async def _download_files_parallel( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=enable_summary, ) async with hb_lock: completed_count += 1 @@ -525,7 +517,6 @@ async def _process_single_file( connector_id: int, search_space_id: int, user_id: str, - enable_summary: bool = True, vision_llm=None, ) -> tuple[int, int, int]: """Download, extract, and index a single Drive file via the pipeline. @@ -561,8 +552,7 @@ async def _process_single_file( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=enable_summary, - ) + ) pipeline = IndexingPipelineService(session) documents = await pipeline.prepare_for_indexing([doc]) @@ -578,10 +568,7 @@ async def _process_single_file( connector_doc = doc_map.get(document.unique_identifier_hash) if not connector_doc: continue - user_llm = await get_user_long_context_llm( - session, user_id, search_space_id - ) - await pipeline.index(document, connector_doc, user_llm) + await pipeline.index(document, connector_doc) await page_limit_service.update_page_usage( user_id, estimated_pages, allow_exceed=True @@ -636,7 +623,6 @@ async def _download_and_index( connector_id: int, search_space_id: int, user_id: str, - enable_summary: bool, on_heartbeat: HeartbeatCallbackType | None = None, vision_llm=None, ) -> tuple[int, int]: @@ -650,7 +636,6 @@ async def _download_and_index( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=enable_summary, on_heartbeat=on_heartbeat, vision_llm=vision_llm, ) @@ -659,13 +644,8 @@ async def _download_and_index( batch_failed = 0 if connector_docs: pipeline = IndexingPipelineService(session) - - async def _get_llm(s): - return await get_user_long_context_llm(s, user_id, search_space_id) - _, batch_indexed, batch_failed = await pipeline.index_batch_parallel( connector_docs, - _get_llm, max_concurrency=3, on_heartbeat=on_heartbeat, ) @@ -681,7 +661,6 @@ async def _index_selected_files( connector_id: int, search_space_id: int, user_id: str, - enable_summary: bool, on_heartbeat: HeartbeatCallbackType | None = None, vision_llm=None, ) -> tuple[int, int, int, list[str]]: @@ -746,7 +725,6 @@ async def _index_selected_files( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=enable_summary, on_heartbeat=on_heartbeat, vision_llm=vision_llm, ) @@ -781,7 +759,6 @@ async def _index_full_scan( max_files: int, include_subfolders: bool = False, on_heartbeat_callback: HeartbeatCallbackType | None = None, - enable_summary: bool = True, vision_llm=None, ) -> tuple[int, int, int]: """Full scan indexing of a folder. @@ -911,7 +888,6 @@ async def _index_full_scan( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=enable_summary, on_heartbeat=on_heartbeat_callback, vision_llm=vision_llm, ) @@ -946,7 +922,6 @@ async def _index_with_delta_sync( max_files: int, include_subfolders: bool = False, on_heartbeat_callback: HeartbeatCallbackType | None = None, - enable_summary: bool = True, vision_llm=None, ) -> tuple[int, int, int]: """Delta sync using change tracking. @@ -1054,7 +1029,6 @@ async def _index_with_delta_sync( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=enable_summary, on_heartbeat=on_heartbeat_callback, vision_llm=vision_llm, ) @@ -1142,7 +1116,6 @@ async def index_google_drive_files( ) return 0, 0, client_error, 0 - connector_enable_summary = getattr(connector, "enable_summary", True) connector_enable_vision_llm = getattr(connector, "enable_vision_llm", False) vision_llm = None if connector_enable_vision_llm: @@ -1189,7 +1162,6 @@ async def index_google_drive_files( max_files, include_subfolders, on_heartbeat_callback, - connector_enable_summary, vision_llm=vision_llm, ) documents_unsupported += du @@ -1208,7 +1180,6 @@ async def index_google_drive_files( max_files, include_subfolders, on_heartbeat_callback, - connector_enable_summary, vision_llm=vision_llm, ) documents_indexed += ri @@ -1234,7 +1205,6 @@ async def index_google_drive_files( max_files, include_subfolders, on_heartbeat_callback, - connector_enable_summary, vision_llm=vision_llm, ) @@ -1346,7 +1316,6 @@ async def index_google_drive_single_file( ) return 0, client_error - connector_enable_summary = getattr(connector, "enable_summary", True) connector_enable_vision_llm = getattr(connector, "enable_vision_llm", False) vision_llm = None if connector_enable_vision_llm: @@ -1370,7 +1339,6 @@ async def index_google_drive_single_file( connector_id, search_space_id, user_id, - connector_enable_summary, vision_llm=vision_llm, ) await session.commit() @@ -1467,7 +1435,6 @@ async def index_google_drive_selected_files( ) return 0, 0, [error_msg] - connector_enable_summary = getattr(connector, "enable_summary", True) connector_enable_vision_llm = getattr(connector, "enable_vision_llm", False) vision_llm = None if connector_enable_vision_llm: @@ -1481,7 +1448,6 @@ async def index_google_drive_selected_files( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=connector_enable_summary, on_heartbeat=on_heartbeat_callback, vision_llm=vision_llm, ) diff --git a/surfsense_backend/app/tasks/connector_indexers/google_gmail_indexer.py b/surfsense_backend/app/tasks/connector_indexers/google_gmail_indexer.py index 6697c0eb1..29b94a873 100644 --- a/surfsense_backend/app/tasks/connector_indexers/google_gmail_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/google_gmail_indexer.py @@ -2,7 +2,7 @@ Google Gmail connector indexer. Uses the shared IndexingPipelineService for document deduplication, -summarization, chunking, and embedding. +chunking, and embedding. """ from collections.abc import Awaitable, Callable @@ -21,7 +21,6 @@ from app.indexing_pipeline.indexing_pipeline_service import ( PlaceholderInfo, ) from app.services.composio_service import ComposioService -from app.services.llm_service import get_user_long_context_llm from app.services.task_logging_service import TaskLoggingService from app.utils.google_credentials import COMPOSIO_GOOGLE_CONNECTOR_TYPES @@ -105,7 +104,6 @@ def _build_connector_doc( connector_id: int, search_space_id: int, user_id: str, - enable_summary: bool, ) -> ConnectorDocument: """Map a raw Gmail API message dict to a ConnectorDocument.""" message_id = message.get("id", "") @@ -138,12 +136,6 @@ def _build_connector_doc( "connector_type": "Google Gmail", } - fallback_summary = ( - f"Google Gmail Message: {subject}\n\n" - f"From: {sender}\nDate: {date_str}\n\n" - f"{markdown_content}" - ) - return ConnectorDocument( title=subject, source_markdown=markdown_content, @@ -152,8 +144,6 @@ def _build_connector_doc( search_space_id=search_space_id, connector_id=connector_id, created_by_id=user_id, - should_summarize=enable_summary, - fallback_summary=fallback_summary, metadata=metadata, ) @@ -454,8 +444,7 @@ async def index_google_gmail_messages( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=connector.enable_summary, - ) + ) with session.no_autoflush: duplicate = await check_duplicate_document_by_hash( @@ -483,13 +472,8 @@ async def index_google_gmail_messages( # ── Pipeline: migrate legacy docs + parallel index ───────────── await pipeline.migrate_legacy_docs(connector_docs) - - async def _get_llm(s): - return await get_user_long_context_llm(s, user_id, search_space_id) - _, documents_indexed, documents_failed = await pipeline.index_batch_parallel( connector_docs, - _get_llm, max_concurrency=3, on_heartbeat=on_heartbeat_callback, heartbeat_interval=HEARTBEAT_INTERVAL_SECONDS, diff --git a/surfsense_backend/app/tasks/connector_indexers/linear_indexer.py b/surfsense_backend/app/tasks/connector_indexers/linear_indexer.py index 8500b700a..4ea781e6f 100644 --- a/surfsense_backend/app/tasks/connector_indexers/linear_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/linear_indexer.py @@ -2,7 +2,7 @@ Linear connector indexer. Uses the shared IndexingPipelineService for document deduplication, -summarization, chunking, and embedding with bounded parallel indexing. +chunking, and embedding with bounded parallel indexing. """ from collections.abc import Awaitable, Callable @@ -18,7 +18,6 @@ from app.indexing_pipeline.indexing_pipeline_service import ( IndexingPipelineService, PlaceholderInfo, ) -from app.services.llm_service import get_user_long_context_llm from app.services.task_logging_service import TaskLoggingService from .base import ( @@ -41,7 +40,6 @@ def _build_connector_doc( connector_id: int, search_space_id: int, user_id: str, - enable_summary: bool, ) -> ConnectorDocument: """Map a raw Linear issue dict to a ConnectorDocument.""" issue_id = issue.get("id", "") @@ -63,11 +61,6 @@ def _build_connector_doc( "connector_type": "Linear", } - fallback_summary = ( - f"Linear Issue {issue_identifier}: {issue_title}\n\n" - f"Status: {state}\n\n{issue_content}" - ) - return ConnectorDocument( title=f"{issue_identifier}: {issue_title}", source_markdown=issue_content, @@ -76,8 +69,6 @@ def _build_connector_doc( search_space_id=search_space_id, connector_id=connector_id, created_by_id=user_id, - should_summarize=enable_summary, - fallback_summary=fallback_summary, metadata=metadata, ) @@ -277,8 +268,7 @@ async def index_linear_issues( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=connector.enable_summary, - ) + ) with session.no_autoflush: duplicate = await check_duplicate_document_by_hash( @@ -306,13 +296,8 @@ async def index_linear_issues( # ── Pipeline: migrate legacy docs + parallel index ──────────── await pipeline.migrate_legacy_docs(connector_docs) - - async def _get_llm(s): - return await get_user_long_context_llm(s, user_id, search_space_id) - _, documents_indexed, documents_failed = await pipeline.index_batch_parallel( connector_docs, - _get_llm, max_concurrency=3, on_heartbeat=on_heartbeat_callback, heartbeat_interval=HEARTBEAT_INTERVAL_SECONDS, diff --git a/surfsense_backend/app/tasks/connector_indexers/local_folder_indexer.py b/surfsense_backend/app/tasks/connector_indexers/local_folder_indexer.py index 9352b60e0..0354fce2e 100644 --- a/surfsense_backend/app/tasks/connector_indexers/local_folder_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/local_folder_indexer.py @@ -33,7 +33,6 @@ from app.db import ( from app.indexing_pipeline.connector_document import ConnectorDocument from app.indexing_pipeline.document_hashing import compute_identifier_hash from app.indexing_pipeline.indexing_pipeline_service import IndexingPipelineService -from app.services.llm_service import get_user_long_context_llm from app.services.page_limit_service import PageLimitExceededError, PageLimitService from app.services.task_logging_service import TaskLoggingService from app.tasks.celery_tasks import get_celery_session_maker @@ -478,7 +477,6 @@ def _build_connector_doc( *, search_space_id: int, user_id: str, - enable_summary: bool, ) -> ConnectorDocument: """Build a ConnectorDocument from a local file's extracted content.""" unique_id = f"{folder_name}:{relative_path}" @@ -488,7 +486,6 @@ def _build_connector_doc( "document_type": "Local Folder File", "connector_type": "Local Folder", } - fallback_summary = f"File: {title}\n\n{content[:4000]}" return ConnectorDocument( title=title, @@ -498,8 +495,6 @@ def _build_connector_doc( search_space_id=search_space_id, connector_id=None, created_by_id=user_id, - should_summarize=enable_summary, - fallback_summary=fallback_summary, metadata=metadata, ) @@ -513,7 +508,6 @@ async def index_local_folder( exclude_patterns: list[str] | None = None, file_extensions: list[str] | None = None, root_folder_id: int | None = None, - enable_summary: bool = False, target_file_paths: list[str] | None = None, on_heartbeat_callback: HeartbeatCallbackType | None = None, ) -> tuple[int, int, int | None, str | None]: @@ -574,8 +568,7 @@ async def index_local_folder( folder_path=folder_path, folder_name=folder_name, target_file_path=target_file_paths[0], - enable_summary=enable_summary, - root_folder_id=root_folder_id, + root_folder_id=root_folder_id, task_logger=task_logger, log_entry=log_entry, ) @@ -587,8 +580,7 @@ async def index_local_folder( folder_path=folder_path, folder_name=folder_name, target_file_paths=target_file_paths, - enable_summary=enable_summary, - root_folder_id=root_folder_id, + root_folder_id=root_folder_id, on_progress_callback=on_heartbeat_callback, ) if err: @@ -774,8 +766,7 @@ async def index_local_folder( folder_name=folder_name, search_space_id=search_space_id, user_id=user_id, - enable_summary=enable_summary, - ) + ) connector_docs.append(doc) file_meta_map[unique_identifier] = { "relative_path": relative_path, @@ -845,15 +836,13 @@ async def index_local_folder( doc_map = {compute_unique_identifier_hash(cd): cd for cd in connector_docs} documents = await pipeline.prepare_for_indexing(connector_docs) - llm = await get_user_long_context_llm(session, user_id, search_space_id) - for document in documents: connector_doc = doc_map.get(document.unique_identifier_hash) if connector_doc is None: failed_count += 1 continue - result = await pipeline.index(document, connector_doc, llm) + result = await pipeline.index(document, connector_doc) if DocumentStatus.is_state(result.status, DocumentStatus.READY): indexed_count += 1 @@ -960,7 +949,6 @@ async def _index_batch_files( folder_path: str, folder_name: str, target_file_paths: list[str], - enable_summary: bool, root_folder_id: int | None, on_progress_callback: HeartbeatCallbackType | None = None, ) -> tuple[int, int, str | None]: @@ -995,8 +983,7 @@ async def _index_batch_files( folder_path=folder_path, folder_name=folder_name, target_file_path=file_path, - enable_summary=enable_summary, - root_folder_id=root_folder_id, + root_folder_id=root_folder_id, task_logger=task_logger, log_entry=log_entry, ) @@ -1036,7 +1023,6 @@ async def _index_single_file( folder_path: str, folder_name: str, target_file_path: str, - enable_summary: bool, root_folder_id: int | None, task_logger, log_entry, @@ -1125,8 +1111,7 @@ async def _index_single_file( folder_name=folder_name, search_space_id=search_space_id, user_id=user_id, - enable_summary=enable_summary, - ) + ) if root_folder_id: connector_doc.folder_id = await _resolve_folder_for_file( @@ -1134,7 +1119,6 @@ async def _index_single_file( ) pipeline = IndexingPipelineService(session) - llm = await get_user_long_context_llm(session, user_id, search_space_id) documents = await pipeline.prepare_for_indexing([connector_doc]) if not documents: @@ -1142,7 +1126,7 @@ async def _index_single_file( db_doc = documents[0] - await pipeline.index(db_doc, connector_doc, llm) + await pipeline.index(db_doc, connector_doc) await session.refresh(db_doc) doc_meta = dict(db_doc.document_metadata or {}) @@ -1275,7 +1259,6 @@ async def index_uploaded_files( user_id: str, folder_name: str, root_folder_id: int, - enable_summary: bool, file_mappings: list[dict], on_heartbeat_callback: HeartbeatCallbackType | None = None, use_vision_llm: bool = False, @@ -1318,7 +1301,6 @@ async def index_uploaded_files( page_limit_service = PageLimitService(session) pipeline = IndexingPipelineService(session) - llm = await get_user_long_context_llm(session, user_id, search_space_id) vision_llm_instance = None if use_vision_llm: @@ -1414,8 +1396,7 @@ async def index_uploaded_files( folder_name=folder_name, search_space_id=search_space_id, user_id=user_id, - enable_summary=enable_summary, - ) + ) connector_doc.folder_id = await _resolve_folder_for_file( session, @@ -1432,7 +1413,7 @@ async def index_uploaded_files( db_doc = documents[0] - await pipeline.index(db_doc, connector_doc, llm) + await pipeline.index(db_doc, connector_doc) await session.refresh(db_doc) doc_meta = dict(db_doc.document_metadata or {}) diff --git a/surfsense_backend/app/tasks/connector_indexers/luma_indexer.py b/surfsense_backend/app/tasks/connector_indexers/luma_indexer.py index 555d60273..662bb6b96 100644 --- a/surfsense_backend/app/tasks/connector_indexers/luma_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/luma_indexer.py @@ -16,13 +16,11 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.connectors.luma_connector import LumaConnector from app.db import Document, DocumentStatus, DocumentType, SearchSourceConnectorType -from app.services.llm_service import get_user_long_context_llm from app.services.task_logging_service import TaskLoggingService from app.utils.document_converters import ( create_document_chunks, embed_text, generate_content_hash, - generate_document_summary, generate_unique_identifier_hash, ) @@ -437,38 +435,14 @@ async def index_luma_events( document.status = DocumentStatus.processing() await session.commit() - # Heavy processing (LLM, embeddings, chunks) - user_llm = await get_user_long_context_llm( - session, user_id, search_space_id - ) + # Heavy processing (embeddings, chunks) - if user_llm and connector.enable_summary: - document_metadata_for_summary = { - "event_id": item["event_id"], - "event_name": item["event_name"], - "event_url": item["event_url"], - "start_at": item["start_at"], - "end_at": item["end_at"], - "timezone": item["timezone"], - "location": item["location"] or "No location", - "city": item["city"], - "hosts": item["host_names"], - "document_type": "Luma Event", - "connector_type": "Luma", - } - ( - summary_content, - summary_embedding, - ) = await generate_document_summary( - item["event_markdown"], user_llm, document_metadata_for_summary - ) - else: - summary_content = ( - f"Luma Event: {item['event_name']}\n\n{item['event_markdown']}" - ) - summary_embedding = await asyncio.to_thread( - embed_text, summary_content - ) + summary_content = ( + f"Luma Event: {item['event_name']}\n\n{item['event_markdown']}" + ) + summary_embedding = await asyncio.to_thread( + embed_text, summary_content + ) chunks = await create_document_chunks(item["event_markdown"]) diff --git a/surfsense_backend/app/tasks/connector_indexers/notion_indexer.py b/surfsense_backend/app/tasks/connector_indexers/notion_indexer.py index 77aac795a..59589b7c7 100644 --- a/surfsense_backend/app/tasks/connector_indexers/notion_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/notion_indexer.py @@ -2,7 +2,7 @@ Notion connector indexer. Uses the shared IndexingPipelineService for document deduplication, -summarization, chunking, and embedding with bounded parallel indexing. +chunking, and embedding with bounded parallel indexing. """ from collections.abc import Awaitable, Callable @@ -19,7 +19,6 @@ from app.indexing_pipeline.indexing_pipeline_service import ( IndexingPipelineService, PlaceholderInfo, ) -from app.services.llm_service import get_user_long_context_llm from app.services.task_logging_service import TaskLoggingService from app.utils.notion_utils import process_blocks @@ -43,7 +42,6 @@ def _build_connector_doc( connector_id: int, search_space_id: int, user_id: str, - enable_summary: bool, ) -> ConnectorDocument: """Map a raw Notion page dict to a ConnectorDocument.""" page_id = page.get("page_id", "") @@ -57,8 +55,6 @@ def _build_connector_doc( "connector_type": "Notion", } - fallback_summary = f"Notion Page: {page_title}\n\n{markdown_content}" - return ConnectorDocument( title=page_title, source_markdown=markdown_content, @@ -67,8 +63,6 @@ def _build_connector_doc( search_space_id=search_space_id, connector_id=connector_id, created_by_id=user_id, - should_summarize=enable_summary, - fallback_summary=fallback_summary, metadata=metadata, ) @@ -314,8 +308,7 @@ async def index_notion_pages( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=connector.enable_summary, - ) + ) with session.no_autoflush: duplicate = await check_duplicate_document_by_hash( @@ -343,13 +336,8 @@ async def index_notion_pages( # ── Pipeline: migrate legacy docs + parallel index ──────────── await pipeline.migrate_legacy_docs(connector_docs) - - async def _get_llm(s): - return await get_user_long_context_llm(s, user_id, search_space_id) - _, documents_indexed, documents_failed = await pipeline.index_batch_parallel( connector_docs, - _get_llm, max_concurrency=3, on_heartbeat=on_heartbeat_callback, heartbeat_interval=HEARTBEAT_INTERVAL_SECONDS, diff --git a/surfsense_backend/app/tasks/connector_indexers/onedrive_indexer.py b/surfsense_backend/app/tasks/connector_indexers/onedrive_indexer.py index 2def799f3..5d783e497 100644 --- a/surfsense_backend/app/tasks/connector_indexers/onedrive_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/onedrive_indexer.py @@ -27,7 +27,6 @@ from app.db import Document, DocumentStatus, DocumentType, SearchSourceConnector from app.indexing_pipeline.connector_document import ConnectorDocument from app.indexing_pipeline.document_hashing import compute_identifier_hash from app.indexing_pipeline.indexing_pipeline_service import IndexingPipelineService -from app.services.llm_service import get_user_long_context_llm from app.services.page_limit_service import PageLimitService from app.services.task_logging_service import TaskLoggingService from app.tasks.connector_indexers.base import ( @@ -133,7 +132,6 @@ def _build_connector_doc( connector_id: int, search_space_id: int, user_id: str, - enable_summary: bool, ) -> ConnectorDocument: file_id = file.get("id", "") file_name = file.get("name", "Unknown") @@ -145,8 +143,6 @@ def _build_connector_doc( "connector_type": "OneDrive", } - fallback_summary = f"File: {file_name}\n\n{markdown[:4000]}" - return ConnectorDocument( title=file_name, source_markdown=markdown, @@ -155,8 +151,6 @@ def _build_connector_doc( search_space_id=search_space_id, connector_id=connector_id, created_by_id=user_id, - should_summarize=enable_summary, - fallback_summary=fallback_summary, metadata=metadata, ) @@ -168,7 +162,6 @@ async def _download_files_parallel( connector_id: int, search_space_id: int, user_id: str, - enable_summary: bool, max_concurrency: int = 3, on_heartbeat: HeartbeatCallbackType | None = None, vision_llm=None, @@ -198,7 +191,6 @@ async def _download_files_parallel( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=enable_summary, ) async with hb_lock: completed_count += 1 @@ -230,7 +222,6 @@ async def _download_and_index( connector_id: int, search_space_id: int, user_id: str, - enable_summary: bool, on_heartbeat: HeartbeatCallbackType | None = None, vision_llm=None, ) -> tuple[int, int]: @@ -241,7 +232,6 @@ async def _download_and_index( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=enable_summary, on_heartbeat=on_heartbeat, vision_llm=vision_llm, ) @@ -250,13 +240,8 @@ async def _download_and_index( batch_failed = 0 if connector_docs: pipeline = IndexingPipelineService(session) - - async def _get_llm(s): - return await get_user_long_context_llm(s, user_id, search_space_id) - _, batch_indexed, batch_failed = await pipeline.index_batch_parallel( connector_docs, - _get_llm, max_concurrency=3, on_heartbeat=on_heartbeat, ) @@ -294,7 +279,6 @@ async def _index_selected_files( connector_id: int, search_space_id: int, user_id: str, - enable_summary: bool, on_heartbeat: HeartbeatCallbackType | None = None, vision_llm=None, ) -> tuple[int, int, int, list[str]]: @@ -345,7 +329,6 @@ async def _index_selected_files( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=enable_summary, on_heartbeat=on_heartbeat, vision_llm=vision_llm, ) @@ -379,7 +362,6 @@ async def _index_full_scan( max_files: int, include_subfolders: bool = True, on_heartbeat_callback: HeartbeatCallbackType | None = None, - enable_summary: bool = True, vision_llm=None, ) -> tuple[int, int, int]: """Full scan indexing of a folder. @@ -454,7 +436,6 @@ async def _index_full_scan( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=enable_summary, on_heartbeat=on_heartbeat_callback, vision_llm=vision_llm, ) @@ -487,7 +468,6 @@ async def _index_with_delta_sync( log_entry: object, max_files: int, on_heartbeat_callback: HeartbeatCallbackType | None = None, - enable_summary: bool = True, vision_llm=None, ) -> tuple[int, int, int, str | None]: """Delta sync using OneDrive change tracking. @@ -579,7 +559,6 @@ async def _index_with_delta_sync( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=enable_summary, on_heartbeat=on_heartbeat_callback, vision_llm=vision_llm, ) @@ -651,7 +630,6 @@ async def index_onedrive_files( ) return 0, 0, error_msg, 0 - connector_enable_summary = getattr(connector, "enable_summary", True) connector_enable_vision_llm = getattr(connector, "enable_vision_llm", False) vision_llm = None if connector_enable_vision_llm: @@ -681,7 +659,6 @@ async def index_onedrive_files( connector_id=connector_id, search_space_id=search_space_id, user_id=user_id, - enable_summary=connector_enable_summary, vision_llm=vision_llm, ) total_indexed += indexed @@ -711,7 +688,6 @@ async def index_onedrive_files( task_logger, log_entry, max_files, - enable_summary=connector_enable_summary, vision_llm=vision_llm, ) total_indexed += indexed @@ -738,7 +714,6 @@ async def index_onedrive_files( log_entry, max_files, include_subfolders, - enable_summary=connector_enable_summary, vision_llm=vision_llm, ) total_indexed += ri @@ -758,7 +733,6 @@ async def index_onedrive_files( log_entry, max_files, include_subfolders, - enable_summary=connector_enable_summary, vision_llm=vision_llm, ) total_indexed += indexed diff --git a/surfsense_backend/app/tasks/connector_indexers/webcrawler_indexer.py b/surfsense_backend/app/tasks/connector_indexers/webcrawler_indexer.py index ada54e7fc..8538f28d2 100644 --- a/surfsense_backend/app/tasks/connector_indexers/webcrawler_indexer.py +++ b/surfsense_backend/app/tasks/connector_indexers/webcrawler_indexer.py @@ -15,13 +15,11 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.connectors.webcrawler_connector import WebCrawlerConnector from app.db import Document, DocumentStatus, DocumentType, SearchSourceConnectorType -from app.services.llm_service import get_user_long_context_llm from app.services.task_logging_service import TaskLoggingService from app.utils.document_converters import ( create_document_chunks, embed_text, generate_content_hash, - generate_document_summary, generate_unique_identifier_hash, ) from app.utils.webcrawler_utils import parse_webcrawler_urls @@ -372,29 +370,10 @@ async def index_crawled_urls( documents_skipped += 1 continue - # Generate summary with LLM - user_llm = await get_user_long_context_llm( - session, user_id, search_space_id - ) + # Select deterministic document content - if user_llm and connector.enable_summary: - document_metadata_for_summary = { - "url": url, - "title": title, - "description": description, - "language": language, - "document_type": "Crawled URL", - "crawler_type": crawler_type, - } - ( - summary_content, - summary_embedding, - ) = await generate_document_summary( - structured_document, user_llm, document_metadata_for_summary - ) - else: - summary_content = f"Crawled URL: {title}\n\nURL: {url}\n\n{content}" - summary_embedding = embed_text(summary_content) + summary_content = f"Crawled URL: {title}\n\nURL: {url}\n\n{content}" + summary_embedding = embed_text(summary_content) # Process chunks chunks = await create_document_chunks(content) diff --git a/surfsense_backend/app/tasks/document_processors/_save.py b/surfsense_backend/app/tasks/document_processors/_save.py index d633dd4f6..3b9616cbd 100644 --- a/surfsense_backend/app/tasks/document_processors/_save.py +++ b/surfsense_backend/app/tasks/document_processors/_save.py @@ -1,20 +1,15 @@ -""" -Unified document save/update logic for file processors. -""" +"""Unified document save/update logic for file processors.""" -import asyncio import logging from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.ext.asyncio import AsyncSession from app.db import Document, DocumentStatus, DocumentType -from app.services.llm_service import get_user_long_context_llm from app.utils.document_converters import ( create_document_chunks, embed_text, generate_content_hash, - generate_document_summary, ) from ._helpers import ( @@ -24,59 +19,6 @@ from ._helpers import ( ) from .base import get_current_timestamp, safe_set_chunks -# --------------------------------------------------------------------------- -# Summary generation -# --------------------------------------------------------------------------- - - -async def _generate_summary( - markdown_content: str, - file_name: str, - etl_service: str, - user_llm, - enable_summary: bool, -) -> tuple[str, list[float]]: - """ - Generate a document summary and embedding. - - Docling uses its own large-document summary strategy; other ETL services - use the standard ``generate_document_summary`` helper. - """ - if not enable_summary: - summary = f"File: {file_name}\n\n{markdown_content[:4000]}" - return summary, await asyncio.to_thread(embed_text, summary) - - if etl_service == "DOCLING": - from app.services.docling_service import create_docling_service - - docling_service = create_docling_service() - summary_text = await docling_service.process_large_document_summary( - content=markdown_content, llm=user_llm, document_title=file_name - ) - - meta = { - "file_name": file_name, - "etl_service": etl_service, - "document_type": "File Document", - } - parts = ["# DOCUMENT METADATA"] - for key, value in meta.items(): - if value: - formatted_key = key.replace("_", " ").title() - parts.append(f"**{formatted_key}:** {value}") - - enhanced = "\n".join(parts) + "\n\n# DOCUMENT SUMMARY\n\n" + summary_text - return enhanced, await asyncio.to_thread(embed_text, enhanced) - - # Standard summary (Unstructured / LlamaCloud / others) - meta = { - "file_name": file_name, - "etl_service": etl_service, - "document_type": "File Document", - } - return await generate_document_summary(markdown_content, user_llm, meta) - - # --------------------------------------------------------------------------- # Unified save function # --------------------------------------------------------------------------- @@ -90,7 +32,6 @@ async def save_file_document( user_id: str, etl_service: str, connector: dict | None = None, - enable_summary: bool = True, ) -> Document | None: """ Process and store a file document with deduplication and migration support. @@ -106,7 +47,6 @@ async def save_file_document( user_id: ID of the user etl_service: Name of the ETL service (UNSTRUCTURED, LLAMACLOUD, DOCLING) connector: Optional connector info for Google Drive files - enable_summary: Whether to generate an AI summary Returns: Document object if successful, None if duplicate detected @@ -133,24 +73,16 @@ async def save_file_document( if should_skip: return doc - user_llm = await get_user_long_context_llm(session, user_id, search_space_id) - if not user_llm: - raise RuntimeError( - f"No long context LLM configured for user {user_id} " - f"in search space {search_space_id}" - ) - - summary_content, summary_embedding = await _generate_summary( - markdown_content, file_name, etl_service, user_llm, enable_summary - ) + document_content = f"File: {file_name}\n\n{markdown_content[:4000]}" + document_embedding = embed_text(document_content) chunks = await create_document_chunks(markdown_content) doc_metadata = {"FILE_NAME": file_name, "ETL_SERVICE": etl_service} if existing_document: existing_document.title = file_name - existing_document.content = summary_content + existing_document.content = document_content existing_document.content_hash = content_hash - existing_document.embedding = summary_embedding + existing_document.embedding = document_embedding existing_document.document_metadata = doc_metadata await safe_set_chunks(session, existing_document, chunks) existing_document.source_markdown = markdown_content @@ -171,8 +103,8 @@ async def save_file_document( title=file_name, document_type=doc_type, document_metadata=doc_metadata, - content=summary_content, - embedding=summary_embedding, + content=document_content, + embedding=document_embedding, chunks=chunks, content_hash=content_hash, unique_identifier_hash=primary_hash, diff --git a/surfsense_backend/app/tasks/document_processors/circleback_processor.py b/surfsense_backend/app/tasks/document_processors/circleback_processor.py index a6b9568b9..ee36d5bc2 100644 --- a/surfsense_backend/app/tasks/document_processors/circleback_processor.py +++ b/surfsense_backend/app/tasks/document_processors/circleback_processor.py @@ -25,11 +25,10 @@ from app.db import ( SearchSourceConnectorType, SearchSpace, ) -from app.services.llm_service import get_document_summary_llm from app.utils.document_converters import ( create_document_chunks, + embed_text, generate_content_hash, - generate_document_summary, generate_unique_identifier_hash, ) @@ -176,34 +175,8 @@ async def add_circleback_meeting_document( # PHASE 3: Process the document content # ======================================================================= - # Get LLM for generating summary - llm = await get_document_summary_llm(session, search_space_id) - if not llm: - logger.warning( - f"No LLM configured for search space {search_space_id}. Using content as summary." - ) - # Use first 1000 chars as summary if no LLM available - summary_content = ( - markdown_content[:1000] + "..." - if len(markdown_content) > 1000 - else markdown_content - ) - summary_embedding = None - else: - # Generate summary with metadata - summary_metadata = { - "meeting_name": meeting_name, - "meeting_id": meeting_id, - "document_type": "Circleback Meeting", - **{ - k: v - for k, v in metadata.items() - if isinstance(v, str | int | float | bool) - }, - } - summary_content, summary_embedding = await generate_document_summary( - markdown_content, llm, summary_metadata - ) + summary_content = markdown_content + summary_embedding = embed_text(summary_content) # Process chunks chunks = await create_document_chunks(markdown_content) @@ -224,8 +197,7 @@ async def add_circleback_meeting_document( document.title = meeting_name document.content = summary_content document.content_hash = content_hash - if summary_embedding is not None: - document.embedding = summary_embedding + document.embedding = summary_embedding document.document_metadata = document_metadata await safe_set_chunks(session, document, chunks) document.source_markdown = markdown_content diff --git a/surfsense_backend/app/tasks/document_processors/extension_processor.py b/surfsense_backend/app/tasks/document_processors/extension_processor.py index 7320ec9fa..bdbc985fa 100644 --- a/surfsense_backend/app/tasks/document_processors/extension_processor.py +++ b/surfsense_backend/app/tasks/document_processors/extension_processor.py @@ -9,12 +9,11 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.db import Document, DocumentType from app.schemas import ExtensionDocumentContent -from app.services.llm_service import get_user_long_context_llm from app.services.task_logging_service import TaskLoggingService from app.utils.document_converters import ( create_document_chunks, + embed_text, generate_content_hash, - generate_document_summary, generate_unique_identifier_hash, ) @@ -123,26 +122,8 @@ async def add_extension_received_document( f"Content changed for URL {content.metadata.VisitedWebPageURL}. Updating document." ) - # Get user's long context LLM (needed for both create and update) - user_llm = await get_user_long_context_llm(session, user_id, search_space_id) - if not user_llm: - raise RuntimeError( - f"No long context LLM configured for user {user_id} in search space {search_space_id}" - ) - - # Generate summary with metadata - document_metadata = { - "session_id": content.metadata.BrowsingSessionId, - "url": content.metadata.VisitedWebPageURL, - "title": content.metadata.VisitedWebPageTitle, - "referrer": content.metadata.VisitedWebPageReffererURL, - "timestamp": content.metadata.VisitedWebPageDateWithTimeInISOString, - "duration_ms": content.metadata.VisitedWebPageVisitDurationInMilliseconds, - "document_type": "Browser Extension Capture", - } - summary_content, summary_embedding = await generate_document_summary( - combined_document_string, user_llm, document_metadata - ) + summary_content = combined_document_string + summary_embedding = embed_text(summary_content) # Process chunks chunks = await create_document_chunks(content.pageContent) diff --git a/surfsense_backend/app/tasks/document_processors/file_processors.py b/surfsense_backend/app/tasks/document_processors/file_processors.py index 137c27cda..805f5554d 100644 --- a/surfsense_backend/app/tasks/document_processors/file_processors.py +++ b/surfsense_backend/app/tasks/document_processors/file_processors.py @@ -10,7 +10,7 @@ from __future__ import annotations import contextlib import logging import os -from dataclasses import dataclass, field +from dataclasses import dataclass from fastapi import HTTPException from sqlalchemy.ext.asyncio import AsyncSession @@ -48,12 +48,6 @@ class _ProcessingContext: notification: Notification | None = None use_vision_llm: bool = False processing_mode: str = "basic" - enable_summary: bool = field(init=False) - - def __post_init__(self) -> None: - self.enable_summary = ( - self.connector.get("enable_summary", True) if self.connector else True - ) # --------------------------------------------------------------------------- @@ -261,7 +255,6 @@ async def _process_document_upload(ctx: _ProcessingContext) -> Document | None: ctx.user_id, etl_result.etl_service, ctx.connector, - enable_summary=ctx.enable_summary, ) if result: @@ -466,7 +459,6 @@ async def process_file_in_background_with_document( log_entry: Log, connector: dict | None = None, notification: Notification | None = None, - should_summarize: bool = False, use_vision_llm: bool = False, processing_mode: str = "basic", ) -> Document | None: @@ -482,7 +474,6 @@ async def process_file_in_background_with_document( from app.indexing_pipeline.adapters.file_upload_adapter import ( UploadDocumentAdapter, ) - from app.services.llm_service import get_user_long_context_llm from app.utils.document_converters import generate_content_hash from .base import check_duplicate_document @@ -522,8 +513,6 @@ async def process_file_in_background_with_document( stage="chunking", ) - user_llm = await get_user_long_context_llm(session, user_id, search_space_id) - adapter = UploadDocumentAdapter(session) await adapter.index( markdown_content=markdown_content, @@ -531,8 +520,6 @@ async def process_file_in_background_with_document( etl_service=etl_service, search_space_id=search_space_id, user_id=user_id, - llm=user_llm, - should_summarize=should_summarize, ) if billable_pages > 0: diff --git a/surfsense_backend/app/tasks/document_processors/markdown_processor.py b/surfsense_backend/app/tasks/document_processors/markdown_processor.py index 0ff340c0e..19a4df87d 100644 --- a/surfsense_backend/app/tasks/document_processors/markdown_processor.py +++ b/surfsense_backend/app/tasks/document_processors/markdown_processor.py @@ -8,12 +8,11 @@ from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.ext.asyncio import AsyncSession from app.db import Document, DocumentStatus, DocumentType -from app.services.llm_service import get_user_long_context_llm from app.services.task_logging_service import TaskLoggingService from app.utils.document_converters import ( create_document_chunks, + embed_text, generate_content_hash, - generate_document_summary, ) from ._helpers import ( @@ -183,21 +182,8 @@ async def add_received_markdown_file_document( return doc # Content changed - continue to update - # Get user's long context LLM (needed for both create and update) - user_llm = await get_user_long_context_llm(session, user_id, search_space_id) - if not user_llm: - raise RuntimeError( - f"No long context LLM configured for user {user_id} in search space {search_space_id}" - ) - - # Generate summary with metadata - document_metadata = { - "file_name": file_name, - "document_type": "Markdown File Document", - } - summary_content, summary_embedding = await generate_document_summary( - file_in_markdown, user_llm, document_metadata - ) + summary_content = f"File: {file_name}\n\n{file_in_markdown[:4000]}" + summary_embedding = embed_text(summary_content) # Process chunks chunks = await create_document_chunks(file_in_markdown) diff --git a/surfsense_backend/app/tasks/document_processors/youtube_processor.py b/surfsense_backend/app/tasks/document_processors/youtube_processor.py index 0ed2e57d2..96c7bda5f 100644 --- a/surfsense_backend/app/tasks/document_processors/youtube_processor.py +++ b/surfsense_backend/app/tasks/document_processors/youtube_processor.py @@ -17,12 +17,11 @@ from sqlalchemy.ext.asyncio import AsyncSession from youtube_transcript_api import YouTubeTranscriptApi from app.db import Document, DocumentStatus, DocumentType -from app.services.llm_service import get_user_long_context_llm from app.services.task_logging_service import TaskLoggingService from app.utils.document_converters import ( create_document_chunks, + embed_text, generate_content_hash, - generate_document_summary, generate_unique_identifier_hash, ) from app.utils.proxy_config import get_requests_proxies @@ -355,40 +354,8 @@ async def add_youtube_video_document( await session.commit() return document - # Get LLM for summary generation - await task_logger.log_task_progress( - log_entry, - f"Preparing for summary generation: {video_data.get('title', 'YouTube Video')}", - {"stage": "llm_setup"}, - ) - - # Get user's long context LLM - user_llm = await get_user_long_context_llm(session, user_id, search_space_id) - if not user_llm: - raise RuntimeError( - f"No long context LLM configured for user {user_id} in search space {search_space_id}" - ) - - # Generate summary - await task_logger.log_task_progress( - log_entry, - f"Generating summary for video: {video_data.get('title', 'YouTube Video')}", - {"stage": "summary_generation"}, - ) - - # Generate summary with metadata - document_metadata_for_summary = { - "url": url, - "video_id": video_id, - "title": video_data.get("title", "YouTube Video"), - "author": video_data.get("author_name", "Unknown"), - "thumbnail": video_data.get("thumbnail_url", ""), - "document_type": "YouTube Video Document", - "has_transcript": "No captions available" not in transcript_text, - } - summary_content, summary_embedding = await generate_document_summary( - combined_document_string, user_llm, document_metadata_for_summary - ) + summary_content = combined_document_string + summary_embedding = embed_text(summary_content) # Process chunks await task_logger.log_task_progress( diff --git a/surfsense_backend/app/utils/document_converters.py b/surfsense_backend/app/utils/document_converters.py index 059d91806..694ae22ac 100644 --- a/surfsense_backend/app/utils/document_converters.py +++ b/surfsense_backend/app/utils/document_converters.py @@ -9,7 +9,6 @@ from litellm import get_model_info, token_counter from app.config import config from app.db import Chunk, DocumentType -from app.prompts import SUMMARY_PROMPT_TEMPLATE logger = logging.getLogger(__name__) @@ -176,57 +175,6 @@ def optimize_content_for_context_window( return optimized_content -async def generate_document_summary( - content: str, - user_llm, - document_metadata: dict | None = None, -) -> tuple[str, list[float]]: - """ - Generate summary and embedding for document content with metadata. - - Args: - content: Document content - user_llm: User's LLM instance - document_metadata: Optional metadata dictionary to include in summary - - Returns: - Tuple of (enhanced_summary_content, summary_embedding) - """ - # Get model name from user_llm for token counting - model_name = getattr(user_llm, "model", "gpt-3.5-turbo") # Fallback to default - - # Optimize content to fit within context window - optimized_content = optimize_content_for_context_window( - content, document_metadata, model_name - ) - - summary_chain = SUMMARY_PROMPT_TEMPLATE | user_llm - content_with_metadata = f"\n\n{document_metadata}\n\n\n\n\n\n{optimized_content}\n\n" - summary_result = await summary_chain.ainvoke({"document": content_with_metadata}) - summary_content = summary_result.content - - # Combine summary with metadata if provided - if document_metadata: - metadata_parts = [] - metadata_parts.append("# DOCUMENT METADATA") - - for key, value in document_metadata.items(): - if value: # Only include non-empty values - formatted_key = key.replace("_", " ").title() - metadata_parts.append(f"**{formatted_key}:** {value}") - - metadata_section = "\n".join(metadata_parts) - enhanced_summary_content = ( - f"{metadata_section}\n\n# DOCUMENT SUMMARY\n\n{summary_content}" - ) - else: - enhanced_summary_content = summary_content - - summary_embedding = await asyncio.to_thread(embed_text, enhanced_summary_content) - - return enhanced_summary_content, summary_embedding - - async def create_document_chunks(content: str) -> list[Chunk]: """ Create chunks from document content. diff --git a/surfsense_backend/tests/e2e/fakes/llm.py b/surfsense_backend/tests/e2e/fakes/llm.py index 9d2370e2c..8172dd86a 100644 --- a/surfsense_backend/tests/e2e/fakes/llm.py +++ b/surfsense_backend/tests/e2e/fakes/llm.py @@ -7,13 +7,13 @@ The production indexing pipeline summarizes documents with: summary_content = summary_result.content The `llm` parameter is supplied per-document by -`app.services.llm_service.get_user_long_context_llm`. We patch THAT +`app.services.llm_service.get_agent_llm`. We patch THAT function to return a langchain-native FakeListChatModel so the rest of the chain works unchanged. No real LLM provider package is touched. Run-backend / run-celery use unittest.mock.patch.start() to install this at every binding site (the source module + every consumer that -did `from app.services.llm_service import get_user_long_context_llm` +did `from app.services.llm_service import get_agent_llm` at module load time). """ @@ -42,7 +42,7 @@ def _make_fake_llm() -> FakeListChatModel: return fake -async def fake_get_user_long_context_llm(*args: Any, **kwargs: Any) -> Any: - """Drop-in replacement for app.services.llm_service.get_user_long_context_llm.""" +async def fake_get_agent_llm(*args: Any, **kwargs: Any) -> Any: + """Drop-in replacement for app.services.llm_service.get_agent_llm.""" logger.info("[fake-llm] returning FakeListChatModel for E2E indexing") return _make_fake_llm() diff --git a/surfsense_backend/tests/e2e/run_backend.py b/surfsense_backend/tests/e2e/run_backend.py index 5a787ac52..6781b1634 100644 --- a/surfsense_backend/tests/e2e/run_backend.py +++ b/surfsense_backend/tests/e2e/run_backend.py @@ -206,23 +206,23 @@ def _patch_llm_bindings() -> None: fake_create_chat_litellm_from_agent_config, fake_create_chat_litellm_from_config, ) - from tests.e2e.fakes.llm import fake_get_user_long_context_llm + from tests.e2e.fakes.llm import fake_get_agent_llm targets = [ - "app.services.llm_service.get_user_long_context_llm", - "app.tasks.connector_indexers.confluence_indexer.get_user_long_context_llm", - "app.tasks.connector_indexers.google_drive_indexer.get_user_long_context_llm", - "app.tasks.connector_indexers.google_gmail_indexer.get_user_long_context_llm", - "app.tasks.connector_indexers.notion_indexer.get_user_long_context_llm", - "app.tasks.connector_indexers.onedrive_indexer.get_user_long_context_llm", - "app.tasks.connector_indexers.dropbox_indexer.get_user_long_context_llm", - "app.tasks.connector_indexers.local_folder_indexer.get_user_long_context_llm", - "app.tasks.document_processors._save.get_user_long_context_llm", - "app.tasks.document_processors.markdown_processor.get_user_long_context_llm", + "app.services.llm_service.get_agent_llm", + "app.tasks.connector_indexers.confluence_indexer.get_agent_llm", + "app.tasks.connector_indexers.google_drive_indexer.get_agent_llm", + "app.tasks.connector_indexers.google_gmail_indexer.get_agent_llm", + "app.tasks.connector_indexers.notion_indexer.get_agent_llm", + "app.tasks.connector_indexers.onedrive_indexer.get_agent_llm", + "app.tasks.connector_indexers.dropbox_indexer.get_agent_llm", + "app.tasks.connector_indexers.local_folder_indexer.get_agent_llm", + "app.tasks.document_processors._save.get_agent_llm", + "app.tasks.document_processors.markdown_processor.get_agent_llm", ] for target in targets: try: - p = patch(target, fake_get_user_long_context_llm) + p = patch(target, fake_get_agent_llm) p.start() _active_patches.append(p) logger.info("[fake-llm] patched %s", target) diff --git a/surfsense_backend/tests/e2e/run_celery.py b/surfsense_backend/tests/e2e/run_celery.py index e4091d689..d0fbb4760 100644 --- a/surfsense_backend/tests/e2e/run_celery.py +++ b/surfsense_backend/tests/e2e/run_celery.py @@ -183,23 +183,23 @@ def _patch_llm_bindings() -> None: fake_create_chat_litellm_from_agent_config, fake_create_chat_litellm_from_config, ) - from tests.e2e.fakes.llm import fake_get_user_long_context_llm + from tests.e2e.fakes.llm import fake_get_agent_llm targets = [ - "app.services.llm_service.get_user_long_context_llm", - "app.tasks.connector_indexers.confluence_indexer.get_user_long_context_llm", - "app.tasks.connector_indexers.google_drive_indexer.get_user_long_context_llm", - "app.tasks.connector_indexers.google_gmail_indexer.get_user_long_context_llm", - "app.tasks.connector_indexers.notion_indexer.get_user_long_context_llm", - "app.tasks.connector_indexers.onedrive_indexer.get_user_long_context_llm", - "app.tasks.connector_indexers.dropbox_indexer.get_user_long_context_llm", - "app.tasks.connector_indexers.local_folder_indexer.get_user_long_context_llm", - "app.tasks.document_processors._save.get_user_long_context_llm", - "app.tasks.document_processors.markdown_processor.get_user_long_context_llm", + "app.services.llm_service.get_agent_llm", + "app.tasks.connector_indexers.confluence_indexer.get_agent_llm", + "app.tasks.connector_indexers.google_drive_indexer.get_agent_llm", + "app.tasks.connector_indexers.google_gmail_indexer.get_agent_llm", + "app.tasks.connector_indexers.notion_indexer.get_agent_llm", + "app.tasks.connector_indexers.onedrive_indexer.get_agent_llm", + "app.tasks.connector_indexers.dropbox_indexer.get_agent_llm", + "app.tasks.connector_indexers.local_folder_indexer.get_agent_llm", + "app.tasks.document_processors._save.get_agent_llm", + "app.tasks.document_processors.markdown_processor.get_agent_llm", ] for target in targets: try: - p = patch(target, fake_get_user_long_context_llm) + p = patch(target, fake_get_agent_llm) p.start() _active_patches.append(p) logger.info("[fake-llm] patched %s in celery worker", target) diff --git a/surfsense_backend/tests/integration/chat/test_thread_visibility.py b/surfsense_backend/tests/integration/chat/test_thread_visibility.py new file mode 100644 index 000000000..464d389db --- /dev/null +++ b/surfsense_backend/tests/integration/chat/test_thread_visibility.py @@ -0,0 +1,279 @@ +"""Integration tests for new-chat thread visibility invariants. + +These tests exercise the route handlers directly with real DB-backed +users, memberships, and permissions. The important contract is that a +thread shared with a search space stays shared across normal metadata +updates until the creator explicitly makes it private again. +""" + +from __future__ import annotations + +import uuid + +import pytest +import pytest_asyncio +from fastapi import HTTPException +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db import ( + ChatVisibility, + SearchSpace, + SearchSpaceMembership, + SearchSpaceRole, + User, +) +from app.routes import new_chat_routes +from app.schemas.new_chat import ( + NewChatThreadCreate, + NewChatThreadUpdate, + NewChatThreadVisibilityUpdate, +) + +pytestmark = pytest.mark.integration + + +@pytest_asyncio.fixture +async def db_member(db_session: AsyncSession, db_search_space: SearchSpace) -> User: + member = User( + id=uuid.uuid4(), + email="member@surfsense.net", + hashed_password="hashed", + is_active=True, + is_superuser=False, + is_verified=True, + ) + db_session.add(member) + await db_session.flush() + + role = ( + ( + await db_session.execute( + select(SearchSpaceRole).where( + SearchSpaceRole.search_space_id == db_search_space.id, + SearchSpaceRole.name == "Editor", + ) + ) + ) + .scalars() + .one() + ) + db_session.add( + SearchSpaceMembership( + user_id=member.id, + search_space_id=db_search_space.id, + role_id=role.id, + is_owner=False, + ) + ) + await db_session.flush() + return member + + +async def _create_thread( + db_session: AsyncSession, + db_user: User, + db_search_space: SearchSpace, + *, + title: str = "Visibility Invariant Chat", +): + return await new_chat_routes.create_thread( + NewChatThreadCreate( + title=title, + archived=False, + search_space_id=db_search_space.id, + visibility=ChatVisibility.PRIVATE, + ), + session=db_session, + user=db_user, + ) + + +def _active_thread_ids(response) -> set[int]: + return {thread.id for thread in response.threads} + + +def _search_thread_ids(response) -> set[int]: + return {thread.id for thread in response} + + +async def test_private_thread_is_hidden_from_other_search_space_member( + db_session: AsyncSession, + db_user: User, + db_member: User, + db_search_space: SearchSpace, +): + thread = await _create_thread(db_session, db_user, db_search_space) + + member_threads = await new_chat_routes.list_threads( + search_space_id=db_search_space.id, + session=db_session, + user=db_member, + ) + member_search = await new_chat_routes.search_threads( + search_space_id=db_search_space.id, + title="Visibility", + session=db_session, + user=db_member, + ) + + assert thread.id not in _active_thread_ids(member_threads) + assert thread.id not in _search_thread_ids(member_search) + with pytest.raises(HTTPException) as exc_info: + await new_chat_routes.get_thread_full( + thread_id=thread.id, + session=db_session, + user=db_member, + ) + assert exc_info.value.status_code == 403 + + +async def test_creator_can_share_thread_and_member_can_list_search_read_it( + db_session: AsyncSession, + db_user: User, + db_member: User, + db_search_space: SearchSpace, +): + thread = await _create_thread(db_session, db_user, db_search_space) + + updated = await new_chat_routes.update_thread_visibility( + thread_id=thread.id, + visibility_update=NewChatThreadVisibilityUpdate( + visibility=ChatVisibility.SEARCH_SPACE, + ), + session=db_session, + user=db_user, + ) + + member_threads = await new_chat_routes.list_threads( + search_space_id=db_search_space.id, + session=db_session, + user=db_member, + ) + member_search = await new_chat_routes.search_threads( + search_space_id=db_search_space.id, + title="Visibility", + session=db_session, + user=db_member, + ) + full_thread = await new_chat_routes.get_thread_full( + thread_id=thread.id, + session=db_session, + user=db_member, + ) + + assert updated.visibility == ChatVisibility.SEARCH_SPACE + assert thread.id in _active_thread_ids(member_threads) + assert thread.id in _search_thread_ids(member_search) + assert full_thread["id"] == thread.id + assert full_thread["visibility"] == ChatVisibility.SEARCH_SPACE + + +async def test_rename_and_archive_do_not_reset_shared_visibility( + db_session: AsyncSession, + db_user: User, + db_search_space: SearchSpace, +): + thread = await _create_thread(db_session, db_user, db_search_space) + await new_chat_routes.update_thread_visibility( + thread_id=thread.id, + visibility_update=NewChatThreadVisibilityUpdate( + visibility=ChatVisibility.SEARCH_SPACE, + ), + session=db_session, + user=db_user, + ) + + renamed = await new_chat_routes.update_thread( + thread_id=thread.id, + thread_update=NewChatThreadUpdate(title="Renamed Shared Chat"), + session=db_session, + user=db_user, + ) + archived = await new_chat_routes.update_thread( + thread_id=thread.id, + thread_update=NewChatThreadUpdate(archived=True), + session=db_session, + user=db_user, + ) + + assert renamed.visibility == ChatVisibility.SEARCH_SPACE + assert archived.visibility == ChatVisibility.SEARCH_SPACE + assert archived.archived is True + + +async def test_non_creator_cannot_change_shared_thread_back_to_private( + db_session: AsyncSession, + db_user: User, + db_member: User, + db_search_space: SearchSpace, +): + thread = await _create_thread(db_session, db_user, db_search_space) + await new_chat_routes.update_thread_visibility( + thread_id=thread.id, + visibility_update=NewChatThreadVisibilityUpdate( + visibility=ChatVisibility.SEARCH_SPACE, + ), + session=db_session, + user=db_user, + ) + + with pytest.raises(HTTPException) as exc_info: + await new_chat_routes.update_thread_visibility( + thread_id=thread.id, + visibility_update=NewChatThreadVisibilityUpdate( + visibility=ChatVisibility.PRIVATE, + ), + session=db_session, + user=db_member, + ) + + assert exc_info.value.status_code == 403 + + +async def test_creator_can_make_shared_thread_private_again( + db_session: AsyncSession, + db_user: User, + db_member: User, + db_search_space: SearchSpace, +): + thread = await _create_thread(db_session, db_user, db_search_space) + await new_chat_routes.update_thread_visibility( + thread_id=thread.id, + visibility_update=NewChatThreadVisibilityUpdate( + visibility=ChatVisibility.SEARCH_SPACE, + ), + session=db_session, + user=db_user, + ) + + private_again = await new_chat_routes.update_thread_visibility( + thread_id=thread.id, + visibility_update=NewChatThreadVisibilityUpdate( + visibility=ChatVisibility.PRIVATE, + ), + session=db_session, + user=db_user, + ) + member_threads = await new_chat_routes.list_threads( + search_space_id=db_search_space.id, + session=db_session, + user=db_member, + ) + member_search = await new_chat_routes.search_threads( + search_space_id=db_search_space.id, + title="Visibility", + session=db_session, + user=db_member, + ) + + assert private_again.visibility == ChatVisibility.PRIVATE + assert thread.id not in _active_thread_ids(member_threads) + assert thread.id not in _search_thread_ids(member_search) + with pytest.raises(HTTPException) as exc_info: + await new_chat_routes.get_thread_full( + thread_id=thread.id, + session=db_session, + user=db_member, + ) + assert exc_info.value.status_code == 403 diff --git a/surfsense_backend/tests/integration/conftest.py b/surfsense_backend/tests/integration/conftest.py index e03101e63..19f8e3d0a 100644 --- a/surfsense_backend/tests/integration/conftest.py +++ b/surfsense_backend/tests/integration/conftest.py @@ -1,7 +1,7 @@ import importlib import sys import uuid -from unittest.mock import AsyncMock, MagicMock +from unittest.mock import MagicMock import pytest import pytest_asyncio @@ -123,26 +123,6 @@ async def db_search_space(db_session: AsyncSession, db_user: User) -> SearchSpac return space -@pytest.fixture -def patched_summarize(monkeypatch) -> AsyncMock: - mock = AsyncMock(return_value="Mocked summary.") - monkeypatch.setattr( - "app.indexing_pipeline.indexing_pipeline_service.summarize_document", - mock, - ) - return mock - - -@pytest.fixture -def patched_summarize_raises(monkeypatch) -> AsyncMock: - mock = AsyncMock(side_effect=RuntimeError("LLM unavailable")) - monkeypatch.setattr( - "app.indexing_pipeline.indexing_pipeline_service.summarize_document", - mock, - ) - return mock - - @pytest.fixture def patched_embed_texts(monkeypatch) -> MagicMock: mock = MagicMock(side_effect=lambda texts: [[0.1] * _EMBEDDING_DIM for _ in texts]) @@ -153,6 +133,16 @@ def patched_embed_texts(monkeypatch) -> MagicMock: return mock +@pytest.fixture +def patched_embed_texts_raises(monkeypatch) -> MagicMock: + mock = MagicMock(side_effect=RuntimeError("Embedding unavailable")) + monkeypatch.setattr( + "app.indexing_pipeline.indexing_pipeline_service.embed_texts", + mock, + ) + return mock + + @pytest.fixture def patched_chunk_text(monkeypatch) -> MagicMock: mock = MagicMock(return_value=["Test chunk content."]) diff --git a/surfsense_backend/tests/integration/document_upload/conftest.py b/surfsense_backend/tests/integration/document_upload/conftest.py index ff44e471a..13e3ab59c 100644 --- a/surfsense_backend/tests/integration/document_upload/conftest.py +++ b/surfsense_backend/tests/integration/document_upload/conftest.py @@ -68,7 +68,6 @@ class InlineTaskDispatcher: filename: str, search_space_id: int, user_id: str, - should_summarize: bool = False, use_vision_llm: bool = False, processing_mode: str = "basic", ) -> None: @@ -83,7 +82,6 @@ class InlineTaskDispatcher: filename, search_space_id, user_id, - should_summarize=should_summarize, use_vision_llm=use_vision_llm, processing_mode=processing_mode, ) @@ -266,10 +264,6 @@ async def page_limits(): @pytest.fixture(autouse=True) def _mock_external_apis(monkeypatch): """Mock LLM, embedding, and chunking — these are external API boundaries.""" - monkeypatch.setattr( - "app.indexing_pipeline.indexing_pipeline_service.summarize_document", - AsyncMock(return_value="Mocked summary."), - ) monkeypatch.setattr( "app.indexing_pipeline.indexing_pipeline_service.embed_texts", MagicMock(side_effect=lambda texts: [[0.1] * _EMBEDDING_DIM for _ in texts]), diff --git a/surfsense_backend/tests/integration/indexing_pipeline/adapters/test_file_upload_adapter.py b/surfsense_backend/tests/integration/indexing_pipeline/adapters/test_file_upload_adapter.py index 6bb1d2094..b3bb241a3 100644 --- a/surfsense_backend/tests/integration/indexing_pipeline/adapters/test_file_upload_adapter.py +++ b/surfsense_backend/tests/integration/indexing_pipeline/adapters/test_file_upload_adapter.py @@ -8,7 +8,7 @@ pytestmark = pytest.mark.integration @pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" +"patched_embed_texts", "patched_chunk_text" ) async def test_sets_status_ready(db_session, db_search_space, db_user, mocker): """Document status is READY after successful indexing.""" @@ -19,7 +19,6 @@ async def test_sets_status_ready(db_session, db_search_space, db_user, mocker): etl_service="UNSTRUCTURED", search_space_id=db_search_space.id, user_id=str(db_user.id), - llm=mocker.Mock(), ) result = await db_session.execute( @@ -31,10 +30,10 @@ async def test_sets_status_ready(db_session, db_search_space, db_user, mocker): @pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" +"patched_embed_texts", "patched_chunk_text" ) -async def test_content_is_summary(db_session, db_search_space, db_user, mocker): - """Document content is set to the LLM-generated summary.""" +async def test_content_is_source_markdown(db_session, db_search_space, db_user, mocker): + """Document content is set to the extracted source markdown.""" adapter = UploadDocumentAdapter(db_session) await adapter.index( markdown_content="## Hello\n\nSome content.", @@ -42,8 +41,6 @@ async def test_content_is_summary(db_session, db_search_space, db_user, mocker): etl_service="UNSTRUCTURED", search_space_id=db_search_space.id, user_id=str(db_user.id), - llm=mocker.Mock(), - should_summarize=True, ) result = await db_session.execute( @@ -51,11 +48,11 @@ async def test_content_is_summary(db_session, db_search_space, db_user, mocker): ) document = result.scalars().first() - assert document.content == "Mocked summary." + assert document.content == "## Hello\n\nSome content." @pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" +"patched_embed_texts", "patched_chunk_text" ) async def test_chunks_written_to_db(db_session, db_search_space, db_user, mocker): """Chunks derived from the source markdown are persisted in the DB.""" @@ -66,7 +63,6 @@ async def test_chunks_written_to_db(db_session, db_search_space, db_user, mocker etl_service="UNSTRUCTURED", search_space_id=db_search_space.id, user_id=str(db_user.id), - llm=mocker.Mock(), ) result = await db_session.execute( @@ -83,9 +79,7 @@ async def test_chunks_written_to_db(db_session, db_search_space, db_user, mocker assert chunks[0].content == "Test chunk content." -@pytest.mark.usefixtures( - "patched_summarize_raises", "patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts_raises", "patched_chunk_text") async def test_raises_on_indexing_failure(db_session, db_search_space, db_user, mocker): """RuntimeError is raised when the indexing step fails so the caller can fire a failure notification.""" adapter = UploadDocumentAdapter(db_session) @@ -96,8 +90,6 @@ async def test_raises_on_indexing_failure(db_session, db_search_space, db_user, etl_service="UNSTRUCTURED", search_space_id=db_search_space.id, user_id=str(db_user.id), - llm=mocker.Mock(), - should_summarize=True, ) @@ -107,10 +99,10 @@ async def test_raises_on_indexing_failure(db_session, db_search_space, db_user, @pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" +"patched_embed_texts", "patched_chunk_text" ) async def test_reindex_updates_content(db_session, db_search_space, db_user, mocker): - """Document content is updated to the new summary after reindexing.""" + """Document content is updated to the new source markdown after reindexing.""" adapter = UploadDocumentAdapter(db_session) await adapter.index( markdown_content="## Original\n\nOriginal content.", @@ -118,7 +110,6 @@ async def test_reindex_updates_content(db_session, db_search_space, db_user, moc etl_service="UNSTRUCTURED", search_space_id=db_search_space.id, user_id=str(db_user.id), - llm=mocker.Mock(), ) result = await db_session.execute( @@ -129,14 +120,14 @@ async def test_reindex_updates_content(db_session, db_search_space, db_user, moc document.source_markdown = "## Edited\n\nNew content after user edit." await db_session.flush() - await adapter.reindex(document=document, llm=mocker.Mock()) + await adapter.reindex(document=document) await db_session.refresh(document) - assert document.content == "Mocked summary." + assert document.content == "## Edited\n\nNew content after user edit." @pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" +"patched_embed_texts", "patched_chunk_text" ) async def test_reindex_updates_content_hash( db_session, db_search_space, db_user, mocker @@ -149,7 +140,6 @@ async def test_reindex_updates_content_hash( etl_service="UNSTRUCTURED", search_space_id=db_search_space.id, user_id=str(db_user.id), - llm=mocker.Mock(), ) result = await db_session.execute( @@ -161,14 +151,14 @@ async def test_reindex_updates_content_hash( document.source_markdown = "## Edited\n\nNew content after user edit." await db_session.flush() - await adapter.reindex(document=document, llm=mocker.Mock()) + await adapter.reindex(document=document) await db_session.refresh(document) assert document.content_hash != original_hash @pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" +"patched_embed_texts", "patched_chunk_text" ) async def test_reindex_sets_status_ready(db_session, db_search_space, db_user, mocker): """Document status is READY after successful reindexing.""" @@ -179,7 +169,6 @@ async def test_reindex_sets_status_ready(db_session, db_search_space, db_user, m etl_service="UNSTRUCTURED", search_space_id=db_search_space.id, user_id=str(db_user.id), - llm=mocker.Mock(), ) result = await db_session.execute( @@ -190,13 +179,13 @@ async def test_reindex_sets_status_ready(db_session, db_search_space, db_user, m document.source_markdown = "## Edited\n\nNew content after user edit." await db_session.flush() - await adapter.reindex(document=document, llm=mocker.Mock()) + await adapter.reindex(document=document) await db_session.refresh(document) assert DocumentStatus.is_state(document.status, DocumentStatus.READY) -@pytest.mark.usefixtures("patched_summarize", "patched_embed_texts") +@pytest.mark.usefixtures("patched_embed_texts") async def test_reindex_replaces_chunks(db_session, db_search_space, db_user, mocker): """Reindexing replaces old chunks with new content rather than appending.""" mocker.patch( @@ -211,7 +200,6 @@ async def test_reindex_replaces_chunks(db_session, db_search_space, db_user, moc etl_service="UNSTRUCTURED", search_space_id=db_search_space.id, user_id=str(db_user.id), - llm=mocker.Mock(), ) result = await db_session.execute( @@ -223,7 +211,7 @@ async def test_reindex_replaces_chunks(db_session, db_search_space, db_user, moc document.source_markdown = "## Edited\n\nNew content after user edit." await db_session.flush() - await adapter.reindex(document=document, llm=mocker.Mock()) + await adapter.reindex(document=document) chunks_result = await db_session.execute( select(Chunk).filter(Chunk.document_id == document_id) @@ -235,7 +223,7 @@ async def test_reindex_replaces_chunks(db_session, db_search_space, db_user, moc @pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" +"patched_embed_texts", "patched_chunk_text" ) async def test_reindex_clears_reindexing_flag( db_session, db_search_space, db_user, mocker @@ -248,7 +236,6 @@ async def test_reindex_clears_reindexing_flag( etl_service="UNSTRUCTURED", search_space_id=db_search_space.id, user_id=str(db_user.id), - llm=mocker.Mock(), ) result = await db_session.execute( @@ -260,19 +247,17 @@ async def test_reindex_clears_reindexing_flag( document.content_needs_reindexing = True await db_session.flush() - await adapter.reindex(document=document, llm=mocker.Mock()) + await adapter.reindex(document=document) await db_session.refresh(document) assert document.content_needs_reindexing is False @pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") -async def test_reindex_raises_on_failure(db_session, db_search_space, db_user, mocker): +async def test_reindex_raises_on_failure( + db_session, db_search_space, db_user, patched_embed_texts, mocker +): """RuntimeError is raised when reindexing fails so the caller can handle it.""" - mocker.patch( - "app.indexing_pipeline.indexing_pipeline_service.summarize_document", - return_value="Mocked summary.", - ) adapter = UploadDocumentAdapter(db_session) await adapter.index( @@ -281,7 +266,6 @@ async def test_reindex_raises_on_failure(db_session, db_search_space, db_user, m etl_service="UNSTRUCTURED", search_space_id=db_search_space.id, user_id=str(db_user.id), - llm=mocker.Mock(), ) result = await db_session.execute( @@ -292,13 +276,10 @@ async def test_reindex_raises_on_failure(db_session, db_search_space, db_user, m document.source_markdown = "## Edited\n\nNew content after user edit." await db_session.flush() - mocker.patch( - "app.indexing_pipeline.indexing_pipeline_service.summarize_document", - side_effect=RuntimeError("LLM unavailable"), - ) + patched_embed_texts.side_effect = RuntimeError("Embedding unavailable") with pytest.raises(RuntimeError, match=r"Embedding failed|Reindexing failed"): - await adapter.reindex(document=document, llm=mocker.Mock()) + await adapter.reindex(document=document) async def test_reindex_raises_on_empty_source_markdown( @@ -323,4 +304,4 @@ async def test_reindex_raises_on_empty_source_markdown( adapter = UploadDocumentAdapter(db_session) with pytest.raises(RuntimeError, match="no source_markdown"): - await adapter.reindex(document=document, llm=mocker.Mock()) + await adapter.reindex(document=document) diff --git a/surfsense_backend/tests/integration/indexing_pipeline/test_calendar_pipeline.py b/surfsense_backend/tests/integration/indexing_pipeline/test_calendar_pipeline.py index b2dd13e57..95afee5ef 100644 --- a/surfsense_backend/tests/integration/indexing_pipeline/test_calendar_pipeline.py +++ b/surfsense_backend/tests/integration/indexing_pipeline/test_calendar_pipeline.py @@ -25,8 +25,6 @@ def _cal_doc( search_space_id=search_space_id, connector_id=connector_id, created_by_id=user_id, - should_summarize=True, - fallback_summary=f"Calendar: Event {unique_id}", metadata={ "event_id": unique_id, "start_time": "2025-01-15T10:00:00", @@ -37,7 +35,7 @@ def _cal_doc( @pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" +"patched_embed_texts", "patched_chunk_text" ) async def test_calendar_pipeline_creates_ready_document( db_session, db_search_space, db_connector, db_user, mocker @@ -55,7 +53,7 @@ async def test_calendar_pipeline_creates_ready_document( prepared = await service.prepare_for_indexing([doc]) assert len(prepared) == 1 - await service.index(prepared[0], doc, llm=mocker.Mock()) + await service.index(prepared[0], doc) result = await db_session.execute( select(Document).filter(Document.search_space_id == space_id) @@ -68,7 +66,7 @@ async def test_calendar_pipeline_creates_ready_document( @pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" +"patched_embed_texts", "patched_chunk_text" ) async def test_calendar_legacy_doc_migrated( db_session, db_search_space, db_connector, db_user, mocker diff --git a/surfsense_backend/tests/integration/indexing_pipeline/test_drive_pipeline.py b/surfsense_backend/tests/integration/indexing_pipeline/test_drive_pipeline.py index d9900ea87..4e8b8a4a2 100644 --- a/surfsense_backend/tests/integration/indexing_pipeline/test_drive_pipeline.py +++ b/surfsense_backend/tests/integration/indexing_pipeline/test_drive_pipeline.py @@ -25,8 +25,6 @@ def _drive_doc( search_space_id=search_space_id, connector_id=connector_id, created_by_id=user_id, - should_summarize=True, - fallback_summary=f"File: {unique_id}.pdf", metadata={ "google_drive_file_id": unique_id, "google_drive_file_name": f"{unique_id}.pdf", @@ -36,7 +34,7 @@ def _drive_doc( @pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" +"patched_embed_texts", "patched_chunk_text" ) async def test_drive_pipeline_creates_ready_document( db_session, db_search_space, db_connector, db_user, mocker @@ -54,7 +52,7 @@ async def test_drive_pipeline_creates_ready_document( prepared = await service.prepare_for_indexing([doc]) assert len(prepared) == 1 - await service.index(prepared[0], doc, llm=mocker.Mock()) + await service.index(prepared[0], doc) result = await db_session.execute( select(Document).filter(Document.search_space_id == space_id) @@ -67,7 +65,7 @@ async def test_drive_pipeline_creates_ready_document( @pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" +"patched_embed_texts", "patched_chunk_text" ) async def test_drive_legacy_doc_migrated( db_session, db_search_space, db_connector, db_user, mocker diff --git a/surfsense_backend/tests/integration/indexing_pipeline/test_dropbox_pipeline.py b/surfsense_backend/tests/integration/indexing_pipeline/test_dropbox_pipeline.py index 83e4f7bb4..d2a8cefc5 100644 --- a/surfsense_backend/tests/integration/indexing_pipeline/test_dropbox_pipeline.py +++ b/surfsense_backend/tests/integration/indexing_pipeline/test_dropbox_pipeline.py @@ -24,8 +24,6 @@ def _dropbox_doc( search_space_id=search_space_id, connector_id=connector_id, created_by_id=user_id, - should_summarize=True, - fallback_summary=f"File: {unique_id}.docx", metadata={ "dropbox_file_id": unique_id, "dropbox_file_name": f"{unique_id}.docx", @@ -35,7 +33,7 @@ def _dropbox_doc( @pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" +"patched_embed_texts", "patched_chunk_text" ) async def test_dropbox_pipeline_creates_ready_document( db_session, db_search_space, db_connector, db_user, mocker @@ -53,7 +51,7 @@ async def test_dropbox_pipeline_creates_ready_document( prepared = await service.prepare_for_indexing([doc]) assert len(prepared) == 1 - await service.index(prepared[0], doc, llm=mocker.Mock()) + await service.index(prepared[0], doc) result = await db_session.execute( select(Document).filter(Document.search_space_id == space_id) @@ -66,7 +64,7 @@ async def test_dropbox_pipeline_creates_ready_document( @pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" +"patched_embed_texts", "patched_chunk_text" ) async def test_dropbox_duplicate_content_skipped( db_session, db_search_space, db_connector, db_user, mocker @@ -86,7 +84,7 @@ async def test_dropbox_duplicate_content_skipped( prepared = await service.prepare_for_indexing([doc]) assert len(prepared) == 1 - await service.index(prepared[0], doc, llm=mocker.Mock()) + await service.index(prepared[0], doc) result = await db_session.execute( select(Document).filter(Document.search_space_id == space_id) diff --git a/surfsense_backend/tests/integration/indexing_pipeline/test_gmail_pipeline.py b/surfsense_backend/tests/integration/indexing_pipeline/test_gmail_pipeline.py index b74d092c0..5b2efa1aa 100644 --- a/surfsense_backend/tests/integration/indexing_pipeline/test_gmail_pipeline.py +++ b/surfsense_backend/tests/integration/indexing_pipeline/test_gmail_pipeline.py @@ -28,8 +28,6 @@ def _gmail_doc( search_space_id=search_space_id, connector_id=connector_id, created_by_id=user_id, - should_summarize=True, - fallback_summary=f"Gmail: Subject for {unique_id}", metadata={ "message_id": unique_id, "from": "sender@example.com", @@ -39,7 +37,7 @@ def _gmail_doc( @pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" +"patched_embed_texts", "patched_chunk_text" ) async def test_gmail_pipeline_creates_ready_document( db_session, db_search_space, db_connector, db_user, mocker @@ -57,7 +55,7 @@ async def test_gmail_pipeline_creates_ready_document( prepared = await service.prepare_for_indexing([doc]) assert len(prepared) == 1 - await service.index(prepared[0], doc, llm=mocker.Mock()) + await service.index(prepared[0], doc) result = await db_session.execute( select(Document).filter(Document.search_space_id == space_id) @@ -71,7 +69,7 @@ async def test_gmail_pipeline_creates_ready_document( @pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" +"patched_embed_texts", "patched_chunk_text" ) async def test_gmail_legacy_doc_migrated_then_reused( db_session, db_search_space, db_connector, db_user, mocker diff --git a/surfsense_backend/tests/integration/indexing_pipeline/test_index_batch.py b/surfsense_backend/tests/integration/indexing_pipeline/test_index_batch.py index 847f7592c..59b7c8814 100644 --- a/surfsense_backend/tests/integration/indexing_pipeline/test_index_batch.py +++ b/surfsense_backend/tests/integration/indexing_pipeline/test_index_batch.py @@ -10,7 +10,7 @@ pytestmark = pytest.mark.integration @pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" +"patched_embed_texts", "patched_chunk_text" ) async def test_index_batch_creates_ready_documents( db_session, db_search_space, make_connector_document, mocker @@ -33,7 +33,7 @@ async def test_index_batch_creates_ready_documents( ] service = IndexingPipelineService(session=db_session) - results = await service.index_batch(docs, llm=mocker.Mock()) + results = await service.index_batch(docs) assert len(results) == 2 @@ -50,10 +50,10 @@ async def test_index_batch_creates_ready_documents( @pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" +"patched_embed_texts", "patched_chunk_text" ) async def test_index_batch_empty_returns_empty(db_session, mocker): """index_batch with empty input returns an empty list.""" service = IndexingPipelineService(session=db_session) - results = await service.index_batch([], llm=mocker.Mock()) + results = await service.index_batch([]) assert results == [] diff --git a/surfsense_backend/tests/integration/indexing_pipeline/test_index_document.py b/surfsense_backend/tests/integration/indexing_pipeline/test_index_document.py index a82148f96..ee895c61b 100644 --- a/surfsense_backend/tests/integration/indexing_pipeline/test_index_document.py +++ b/surfsense_backend/tests/integration/indexing_pipeline/test_index_document.py @@ -10,9 +10,7 @@ _EMBEDDING_DIM = app_config.embedding_model_instance.dimension pytestmark = pytest.mark.integration -@pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_sets_status_ready( db_session, db_search_space, @@ -27,7 +25,7 @@ async def test_sets_status_ready( document = prepared[0] document_id = document.id - await service.index(document, connector_doc, llm=mocker.Mock()) + await service.index(document, connector_doc) result = await db_session.execute( select(Document).filter(Document.id == document_id) @@ -37,16 +35,14 @@ async def test_sets_status_ready( assert DocumentStatus.is_state(reloaded.status, DocumentStatus.READY) -@pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" -) -async def test_content_is_summary_when_should_summarize_true( +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") +async def test_content_is_source_markdown_by_default( db_session, db_search_space, make_connector_document, mocker, ): - """Document content is set to the LLM-generated summary when should_summarize=True.""" + """Document content is set to source_markdown by default.""" connector_doc = make_connector_document(search_space_id=db_search_space.id) service = IndexingPipelineService(session=db_session) @@ -54,28 +50,25 @@ async def test_content_is_summary_when_should_summarize_true( document = prepared[0] document_id = document.id - await service.index(document, connector_doc, llm=mocker.Mock()) + await service.index(document, connector_doc) result = await db_session.execute( select(Document).filter(Document.id == document_id) ) reloaded = result.scalars().first() - assert reloaded.content == "Mocked summary." + assert reloaded.content == connector_doc.source_markdown -@pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" -) -async def test_content_is_source_markdown_when_should_summarize_false( +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") +async def test_content_is_source_markdown_when_custom_content( db_session, db_search_space, make_connector_document, ): - """Document content is set to source_markdown verbatim when should_summarize=False.""" + """Document content is set to source_markdown verbatim.""" connector_doc = make_connector_document( search_space_id=db_search_space.id, - should_summarize=False, source_markdown="## Raw content", ) service = IndexingPipelineService(session=db_session) @@ -84,7 +77,7 @@ async def test_content_is_source_markdown_when_should_summarize_false( document = prepared[0] document_id = document.id - await service.index(document, connector_doc, llm=None) + await service.index(document, connector_doc) result = await db_session.execute( select(Document).filter(Document.id == document_id) @@ -94,9 +87,7 @@ async def test_content_is_source_markdown_when_should_summarize_false( assert reloaded.content == "## Raw content" -@pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_chunks_written_to_db( db_session, db_search_space, @@ -111,7 +102,7 @@ async def test_chunks_written_to_db( document = prepared[0] document_id = document.id - await service.index(document, connector_doc, llm=mocker.Mock()) + await service.index(document, connector_doc) result = await db_session.execute( select(Chunk).filter(Chunk.document_id == document_id) @@ -122,9 +113,7 @@ async def test_chunks_written_to_db( assert chunks[0].content == "Test chunk content." -@pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_embedding_written_to_db( db_session, db_search_space, @@ -139,7 +128,7 @@ async def test_embedding_written_to_db( document = prepared[0] document_id = document.id - await service.index(document, connector_doc, llm=mocker.Mock()) + await service.index(document, connector_doc) result = await db_session.execute( select(Document).filter(Document.id == document_id) @@ -150,9 +139,7 @@ async def test_embedding_written_to_db( assert len(reloaded.embedding) == _EMBEDDING_DIM -@pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_updated_at_advances_after_indexing( db_session, db_search_space, @@ -172,7 +159,7 @@ async def test_updated_at_advances_after_indexing( ) updated_at_pending = result.scalars().first().updated_at - await service.index(document, connector_doc, llm=mocker.Mock()) + await service.index(document, connector_doc) result = await db_session.execute( select(Document).filter(Document.id == document_id) @@ -182,18 +169,15 @@ async def test_updated_at_advances_after_indexing( assert updated_at_ready > updated_at_pending -@pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_no_llm_falls_back_to_source_markdown( db_session, db_search_space, make_connector_document, ): - """When llm=None and no fallback_summary, content falls back to source_markdown.""" + """Content stays deterministic source markdown without an LLM.""" connector_doc = make_connector_document( search_space_id=db_search_space.id, - should_summarize=True, source_markdown="## Fallback content", ) service = IndexingPipelineService(session=db_session) @@ -202,7 +186,7 @@ async def test_no_llm_falls_back_to_source_markdown( document = prepared[0] document_id = document.id - await service.index(document, connector_doc, llm=None) + await service.index(document, connector_doc) result = await db_session.execute( select(Document).filter(Document.id == document_id) @@ -213,27 +197,23 @@ async def test_no_llm_falls_back_to_source_markdown( assert reloaded.content == "## Fallback content" -@pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" -) -async def test_fallback_summary_used_when_llm_unavailable( +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") +async def test_source_markdown_used_without_preview( db_session, db_search_space, make_connector_document, ): - """fallback_summary is used as content when llm=None and should_summarize=True.""" + """Source markdown is used without fallback preview fields.""" connector_doc = make_connector_document( search_space_id=db_search_space.id, - should_summarize=True, source_markdown="## Full raw content", - fallback_summary="Short pre-built summary.", ) service = IndexingPipelineService(session=db_session) prepared = await service.prepare_for_indexing([connector_doc]) document_id = prepared[0].id - await service.index(prepared[0], connector_doc, llm=None) + await service.index(prepared[0], connector_doc) result = await db_session.execute( select(Document).filter(Document.id == document_id) @@ -241,12 +221,10 @@ async def test_fallback_summary_used_when_llm_unavailable( reloaded = result.scalars().first() assert DocumentStatus.is_state(reloaded.status, DocumentStatus.READY) - assert reloaded.content == "Short pre-built summary." + assert reloaded.content == "## Full raw content" -@pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts", "patched_chunk_text") async def test_reindex_replaces_old_chunks( db_session, db_search_space, @@ -264,14 +242,14 @@ async def test_reindex_replaces_old_chunks( document = prepared[0] document_id = document.id - await service.index(document, connector_doc, llm=mocker.Mock()) + await service.index(document, connector_doc) updated_doc = make_connector_document( search_space_id=db_search_space.id, source_markdown="## v2", ) re_prepared = await service.prepare_for_indexing([updated_doc]) - await service.index(re_prepared[0], updated_doc, llm=mocker.Mock()) + await service.index(re_prepared[0], updated_doc) result = await db_session.execute( select(Chunk).filter(Chunk.document_id == document_id) @@ -281,16 +259,14 @@ async def test_reindex_replaces_old_chunks( assert len(chunks) == 1 -@pytest.mark.usefixtures( - "patched_summarize_raises", "patched_embed_texts", "patched_chunk_text" -) -async def test_llm_error_sets_status_failed( +@pytest.mark.usefixtures("patched_embed_texts_raises", "patched_chunk_text") +async def test_embedding_error_sets_status_failed( db_session, db_search_space, make_connector_document, mocker, ): - """Document status is FAILED when the LLM raises during indexing.""" + """Document status is FAILED when embedding raises during indexing.""" connector_doc = make_connector_document(search_space_id=db_search_space.id) service = IndexingPipelineService(session=db_session) @@ -298,7 +274,7 @@ async def test_llm_error_sets_status_failed( document = prepared[0] document_id = document.id - await service.index(document, connector_doc, llm=mocker.Mock()) + await service.index(document, connector_doc) result = await db_session.execute( select(Document).filter(Document.id == document_id) @@ -308,10 +284,8 @@ async def test_llm_error_sets_status_failed( assert DocumentStatus.is_state(reloaded.status, DocumentStatus.FAILED) -@pytest.mark.usefixtures( - "patched_summarize_raises", "patched_embed_texts", "patched_chunk_text" -) -async def test_llm_error_leaves_no_partial_data( +@pytest.mark.usefixtures("patched_embed_texts_raises", "patched_chunk_text") +async def test_embedding_error_leaves_no_partial_data( db_session, db_search_space, make_connector_document, @@ -325,7 +299,7 @@ async def test_llm_error_leaves_no_partial_data( document = prepared[0] document_id = document.id - await service.index(document, connector_doc, llm=mocker.Mock()) + await service.index(document, connector_doc) result = await db_session.execute( select(Document).filter(Document.id == document_id) diff --git a/surfsense_backend/tests/integration/indexing_pipeline/test_local_folder_pipeline.py b/surfsense_backend/tests/integration/indexing_pipeline/test_local_folder_pipeline.py index 4dc5742f7..2cd378343 100644 --- a/surfsense_backend/tests/integration/indexing_pipeline/test_local_folder_pipeline.py +++ b/surfsense_backend/tests/integration/indexing_pipeline/test_local_folder_pipeline.py @@ -21,7 +21,6 @@ from app.db import ( pytestmark = pytest.mark.integration UNIFIED_FIXTURES = ( - "patched_summarize", "patched_embed_texts", "patched_chunk_text", ) @@ -787,7 +786,7 @@ class TestPipelineIntegration: assert len(prepared) == 1 db_doc = prepared[0] - result = await service.index(db_doc, doc, llm=mocker.Mock()) + result = await service.index(db_doc, doc) assert result is not None docs = ( @@ -1272,7 +1271,7 @@ class TestIndexingProgressFlag: original_index = IndexingPipelineService.index flag_observed = [] - async def patched_index(self_pipe, document, connector_doc, llm): + async def patched_index(self_pipe, document, connector_doc): folder = ( await db_session.execute( select(Folder).where( @@ -1284,7 +1283,7 @@ class TestIndexingProgressFlag: if folder: meta = folder.folder_metadata or {} flag_observed.append(meta.get("indexing_in_progress", False)) - return await original_index(self_pipe, document, connector_doc, llm) + return await original_index(self_pipe, document, connector_doc) IndexingPipelineService.index = patched_index try: diff --git a/surfsense_backend/tests/integration/indexing_pipeline/test_onedrive_pipeline.py b/surfsense_backend/tests/integration/indexing_pipeline/test_onedrive_pipeline.py index 541e3a38e..41ac6894b 100644 --- a/surfsense_backend/tests/integration/indexing_pipeline/test_onedrive_pipeline.py +++ b/surfsense_backend/tests/integration/indexing_pipeline/test_onedrive_pipeline.py @@ -24,8 +24,6 @@ def _onedrive_doc( search_space_id=search_space_id, connector_id=connector_id, created_by_id=user_id, - should_summarize=True, - fallback_summary=f"File: {unique_id}.docx", metadata={ "onedrive_file_id": unique_id, "onedrive_file_name": f"{unique_id}.docx", @@ -35,7 +33,7 @@ def _onedrive_doc( @pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" +"patched_embed_texts", "patched_chunk_text" ) async def test_onedrive_pipeline_creates_ready_document( db_session, db_search_space, db_connector, db_user, mocker @@ -53,7 +51,7 @@ async def test_onedrive_pipeline_creates_ready_document( prepared = await service.prepare_for_indexing([doc]) assert len(prepared) == 1 - await service.index(prepared[0], doc, llm=mocker.Mock()) + await service.index(prepared[0], doc) result = await db_session.execute( select(Document).filter(Document.search_space_id == space_id) @@ -66,7 +64,7 @@ async def test_onedrive_pipeline_creates_ready_document( @pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" +"patched_embed_texts", "patched_chunk_text" ) async def test_onedrive_duplicate_content_skipped( db_session, db_search_space, db_connector, db_user, mocker @@ -86,7 +84,7 @@ async def test_onedrive_duplicate_content_skipped( prepared = await service.prepare_for_indexing([doc]) assert len(prepared) == 1 - await service.index(prepared[0], doc, llm=mocker.Mock()) + await service.index(prepared[0], doc) result = await db_session.execute( select(Document).filter(Document.search_space_id == space_id) diff --git a/surfsense_backend/tests/integration/indexing_pipeline/test_prepare_for_indexing.py b/surfsense_backend/tests/integration/indexing_pipeline/test_prepare_for_indexing.py index 776180b9a..d0b8c7fed 100644 --- a/surfsense_backend/tests/integration/indexing_pipeline/test_prepare_for_indexing.py +++ b/surfsense_backend/tests/integration/indexing_pipeline/test_prepare_for_indexing.py @@ -33,7 +33,7 @@ async def test_new_document_is_persisted_with_pending_status( @pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" +"patched_embed_texts", "patched_chunk_text" ) async def test_unchanged_ready_document_is_skipped( db_session, @@ -47,7 +47,7 @@ async def test_unchanged_ready_document_is_skipped( # Index fully so the document reaches ready state prepared = await service.prepare_for_indexing([doc]) - await service.index(prepared[0], doc, llm=mocker.Mock()) + await service.index(prepared[0], doc) # Same content on the next run — a ready document must be skipped results = await service.prepare_for_indexing([doc]) @@ -56,7 +56,7 @@ async def test_unchanged_ready_document_is_skipped( @pytest.mark.usefixtures( - "patched_summarize", "patched_embed_texts", "patched_chunk_text" +"patched_embed_texts", "patched_chunk_text" ) async def test_title_only_change_updates_title_in_db( db_session, @@ -72,7 +72,7 @@ async def test_title_only_change_updates_title_in_db( prepared = await service.prepare_for_indexing([original]) document_id = prepared[0].id - await service.index(prepared[0], original, llm=mocker.Mock()) + await service.index(prepared[0], original) renamed = make_connector_document( search_space_id=db_search_space.id, title="Updated Title" @@ -338,9 +338,7 @@ async def test_same_content_from_different_source_is_skipped( assert len(result.scalars().all()) == 1 -@pytest.mark.usefixtures( - "patched_summarize_raises", "patched_embed_texts", "patched_chunk_text" -) +@pytest.mark.usefixtures("patched_embed_texts_raises", "patched_chunk_text") async def test_failed_document_with_unchanged_content_is_requeued( db_session, db_search_space, @@ -351,10 +349,10 @@ async def test_failed_document_with_unchanged_content_is_requeued( doc = make_connector_document(search_space_id=db_search_space.id) service = IndexingPipelineService(session=db_session) - # First run: document is created and indexing crashes → status = failed + # First run: document is created and indexing crashes, so status becomes failed. prepared = await service.prepare_for_indexing([doc]) document_id = prepared[0].id - await service.index(prepared[0], doc, llm=mocker.Mock()) + await service.index(prepared[0], doc) result = await db_session.execute( select(Document).filter(Document.id == document_id) diff --git a/surfsense_backend/tests/unit/agents/new_chat/tools/test_resume_page_limits.py b/surfsense_backend/tests/unit/agents/new_chat/tools/test_resume_page_limits.py index 4f93ad732..f9212a45c 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/tools/test_resume_page_limits.py +++ b/surfsense_backend/tests/unit/agents/new_chat/tools/test_resume_page_limits.py @@ -101,7 +101,7 @@ async def test_generate_resume_defaults_to_one_page_target(monkeypatch) -> None: llm = SimpleNamespace(ainvoke=AsyncMock(side_effect=_llm_invoke)) monkeypatch.setattr( resume_tool, - "get_document_summary_llm", + "get_agent_llm", AsyncMock(return_value=llm), ) monkeypatch.setattr(resume_tool, "_compile_typst", lambda _source: b"pdf") @@ -130,7 +130,7 @@ async def test_generate_resume_compresses_when_over_limit(monkeypatch) -> None: llm = SimpleNamespace(ainvoke=AsyncMock(side_effect=responses)) monkeypatch.setattr( resume_tool, - "get_document_summary_llm", + "get_agent_llm", AsyncMock(return_value=llm), ) monkeypatch.setattr(resume_tool, "_compile_typst", lambda _source: b"pdf") @@ -165,7 +165,7 @@ async def test_generate_resume_returns_ready_when_target_not_met(monkeypatch) -> llm = SimpleNamespace(ainvoke=AsyncMock(side_effect=responses)) monkeypatch.setattr( resume_tool, - "get_document_summary_llm", + "get_agent_llm", AsyncMock(return_value=llm), ) monkeypatch.setattr(resume_tool, "_compile_typst", lambda _source: b"pdf") @@ -198,7 +198,7 @@ async def test_generate_resume_fails_when_hard_limit_exceeded(monkeypatch) -> No llm = SimpleNamespace(ainvoke=AsyncMock(side_effect=responses)) monkeypatch.setattr( resume_tool, - "get_document_summary_llm", + "get_agent_llm", AsyncMock(return_value=llm), ) monkeypatch.setattr(resume_tool, "_compile_typst", lambda _source: b"pdf") diff --git a/surfsense_backend/tests/unit/automations/actions/builtin/agent_task/test_dependencies.py b/surfsense_backend/tests/unit/automations/actions/builtin/agent_task/test_dependencies.py index ac20b2608..79da12933 100644 --- a/surfsense_backend/tests/unit/automations/actions/builtin/agent_task/test_dependencies.py +++ b/surfsense_backend/tests/unit/automations/actions/builtin/agent_task/test_dependencies.py @@ -14,8 +14,8 @@ from typing import Any import pytest -import app.automations.actions.agent_task.dependencies as deps_mod -from app.automations.actions.agent_task.dependencies import ( +import app.automations.actions.builtin.agent_task.dependencies as deps_mod +from app.automations.actions.builtin.agent_task.dependencies import ( DependencyError, build_dependencies, ) diff --git a/surfsense_backend/tests/unit/connector_indexers/test_confluence_parallel.py b/surfsense_backend/tests/unit/connector_indexers/test_confluence_parallel.py index a8cf05269..ff85096d4 100644 --- a/surfsense_backend/tests/unit/connector_indexers/test_confluence_parallel.py +++ b/surfsense_backend/tests/unit/connector_indexers/test_confluence_parallel.py @@ -71,7 +71,6 @@ async def test_build_connector_doc_produces_correct_fields(): connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, ) assert doc.title == "Engineering Handbook" @@ -81,7 +80,6 @@ async def test_build_connector_doc_produces_correct_fields(): assert doc.search_space_id == _SEARCH_SPACE_ID assert doc.connector_id == _CONNECTOR_ID assert doc.created_by_id == _USER_ID - assert doc.should_summarize is True assert doc.metadata["page_id"] == "abc-123" assert doc.metadata["page_title"] == "Engineering Handbook" assert doc.metadata["space_id"] == "ENG" @@ -89,21 +87,6 @@ async def test_build_connector_doc_produces_correct_fields(): assert doc.metadata["connector_id"] == _CONNECTOR_ID assert doc.metadata["document_type"] == "Confluence Page" assert doc.metadata["connector_type"] == "Confluence" - assert doc.fallback_summary is not None - assert "Engineering Handbook" in doc.fallback_summary - assert markdown in doc.fallback_summary - - -async def test_build_connector_doc_summary_disabled(): - doc = _build_connector_doc( - _make_page(), - _to_markdown(_make_page()), - connector_id=_CONNECTOR_ID, - search_space_id=_SEARCH_SPACE_ID, - user_id=_USER_ID, - enable_summary=False, - ) - assert doc.should_summarize is False # --------------------------------------------------------------------------- @@ -111,10 +94,9 @@ async def test_build_connector_doc_summary_disabled(): # --------------------------------------------------------------------------- -def _mock_connector(enable_summary: bool = True): +def _mock_connector(): c = MagicMock() c.config = {"access_token": "tok"} - c.enable_summary = enable_summary c.last_indexed_at = None return c diff --git a/surfsense_backend/tests/unit/connector_indexers/test_dropbox_parallel.py b/surfsense_backend/tests/unit/connector_indexers/test_dropbox_parallel.py index 9ba87207a..694caed06 100644 --- a/surfsense_backend/tests/unit/connector_indexers/test_dropbox_parallel.py +++ b/surfsense_backend/tests/unit/connector_indexers/test_dropbox_parallel.py @@ -71,7 +71,6 @@ async def test_single_file_returns_one_connector_document( connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, ) assert len(docs) == 1 @@ -97,7 +96,6 @@ async def test_multiple_files_all_produce_documents( connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, ) assert len(docs) == 3 @@ -125,7 +123,6 @@ async def test_one_download_exception_does_not_block_others( connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, ) assert len(docs) == 2 @@ -152,7 +149,6 @@ async def test_etl_error_counts_as_download_failure( connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, ) assert len(docs) == 1 @@ -191,7 +187,6 @@ async def test_concurrency_bounded_by_semaphore( connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, max_concurrency=2, ) @@ -231,7 +226,6 @@ async def test_heartbeat_fires_during_parallel_downloads( connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, on_heartbeat=_on_heartbeat, ) @@ -324,7 +318,6 @@ async def _run_full_scan(mocks, monkeypatch, page_files, *, max_files=500): mocks["task_logger"], mocks["log_entry"], max_files, - enable_summary=True, ) @@ -434,7 +427,6 @@ async def _run_selected(mocks, file_tuples): connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, ) @@ -569,7 +561,6 @@ async def test_delta_sync_deletions_call_remove_document(monkeypatch): mock_task_logger, MagicMock(), max_files=500, - enable_summary=True, ) assert sorted(remove_calls) == ["id:del1", "id:del2"] @@ -608,7 +599,6 @@ async def test_delta_sync_upserts_filtered_and_downloaded(monkeypatch): mock_task_logger, MagicMock(), max_files=500, - enable_summary=True, ) assert indexed == 2 @@ -670,7 +660,6 @@ async def test_delta_sync_mix_deletions_and_upserts(monkeypatch): mock_task_logger, MagicMock(), max_files=500, - enable_summary=True, ) assert sorted(remove_calls) == ["id:del1", "id:del2"] @@ -704,7 +693,6 @@ async def test_delta_sync_returns_new_cursor(monkeypatch): mock_task_logger, MagicMock(), max_files=500, - enable_summary=True, ) assert cursor == "brand-new-cursor-xyz" @@ -725,7 +713,7 @@ def orchestrator_mocks(monkeypatch): mock_connector = MagicMock() mock_connector.config = {"_token_encrypted": False} mock_connector.last_indexed_at = None - mock_connector.enable_summary = True + mock_connector.enable_vision_llm = True monkeypatch.setattr( _mod, diff --git a/surfsense_backend/tests/unit/connector_indexers/test_google_drive_parallel.py b/surfsense_backend/tests/unit/connector_indexers/test_google_drive_parallel.py index 7e968514c..65be05593 100644 --- a/surfsense_backend/tests/unit/connector_indexers/test_google_drive_parallel.py +++ b/surfsense_backend/tests/unit/connector_indexers/test_google_drive_parallel.py @@ -66,7 +66,6 @@ async def test_single_file_returns_one_connector_document( connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, ) assert len(docs) == 1 @@ -91,7 +90,6 @@ async def test_multiple_files_all_produce_documents( connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, ) assert len(docs) == 3 @@ -119,7 +117,6 @@ async def test_one_download_exception_does_not_block_others( connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, ) assert len(docs) == 2 @@ -146,7 +143,6 @@ async def test_etl_error_counts_as_download_failure( connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, ) assert len(docs) == 1 @@ -186,7 +182,6 @@ async def test_concurrency_bounded_by_semaphore( connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, max_concurrency=2, ) @@ -226,7 +221,6 @@ async def test_heartbeat_fires_during_parallel_downloads( connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, on_heartbeat=_on_heartbeat, ) @@ -300,12 +294,6 @@ def full_scan_mocks(mock_drive_client, monkeypatch): MagicMock(return_value=pipeline_mock), ) - monkeypatch.setattr( - _mod, - "get_user_long_context_llm", - AsyncMock(return_value=MagicMock()), - ) - return { "drive_client": mock_drive_client, "session": mock_session, @@ -333,7 +321,6 @@ async def _run_full_scan(mocks, *, max_files=500, include_subfolders=False): mocks["log_entry"], max_files, include_subfolders=include_subfolders, - enable_summary=True, ) @@ -487,12 +474,6 @@ async def test_delta_sync_removals_serial_rest_parallel(monkeypatch): "IndexingPipelineService", MagicMock(return_value=pipeline_mock), ) - monkeypatch.setattr( - _mod, - "get_user_long_context_llm", - AsyncMock(return_value=MagicMock()), - ) - mock_session, _ = _make_page_limit_session() mock_task_logger = MagicMock() mock_task_logger.log_task_progress = AsyncMock() @@ -509,7 +490,6 @@ async def test_delta_sync_removals_serial_rest_parallel(monkeypatch): mock_task_logger, MagicMock(), max_files=500, - enable_summary=True, ) assert sorted(remove_calls) == ["del1", "del2", "trash1"] @@ -577,7 +557,6 @@ async def _run_selected(mocks, file_ids): connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, ) diff --git a/surfsense_backend/tests/unit/connector_indexers/test_linear_parallel.py b/surfsense_backend/tests/unit/connector_indexers/test_linear_parallel.py index ef17aae06..f057a6352 100644 --- a/surfsense_backend/tests/unit/connector_indexers/test_linear_parallel.py +++ b/surfsense_backend/tests/unit/connector_indexers/test_linear_parallel.py @@ -70,7 +70,6 @@ async def test_build_connector_doc_produces_correct_fields(): connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, ) assert doc.title == "ENG-42: Fix login bug" @@ -80,7 +79,6 @@ async def test_build_connector_doc_produces_correct_fields(): assert doc.search_space_id == _SEARCH_SPACE_ID assert doc.connector_id == _CONNECTOR_ID assert doc.created_by_id == _USER_ID - assert doc.should_summarize is True assert doc.metadata["issue_id"] == "abc-123" assert doc.metadata["issue_identifier"] == "ENG-42" assert doc.metadata["issue_title"] == "Fix login bug" @@ -90,24 +88,6 @@ async def test_build_connector_doc_produces_correct_fields(): assert doc.metadata["connector_id"] == _CONNECTOR_ID assert doc.metadata["document_type"] == "Linear Issue" assert doc.metadata["connector_type"] == "Linear" - assert doc.fallback_summary is not None - assert "ENG-42" in doc.fallback_summary - assert markdown in doc.fallback_summary - - -async def test_build_connector_doc_summary_disabled(): - """When enable_summary is False, should_summarize is False.""" - doc = _build_connector_doc( - _make_issue(), - _make_formatted_issue(), - "# content", - connector_id=_CONNECTOR_ID, - search_space_id=_SEARCH_SPACE_ID, - user_id=_USER_ID, - enable_summary=False, - ) - - assert doc.should_summarize is False # --------------------------------------------------------------------------- @@ -115,10 +95,9 @@ async def test_build_connector_doc_summary_disabled(): # --------------------------------------------------------------------------- -def _mock_connector(enable_summary: bool = True): +def _mock_connector(): c = MagicMock() c.config = {"access_token": "tok"} - c.enable_summary = enable_summary c.last_indexed_at = None return c diff --git a/surfsense_backend/tests/unit/connector_indexers/test_notion_parallel.py b/surfsense_backend/tests/unit/connector_indexers/test_notion_parallel.py index 651524015..e40f739d8 100644 --- a/surfsense_backend/tests/unit/connector_indexers/test_notion_parallel.py +++ b/surfsense_backend/tests/unit/connector_indexers/test_notion_parallel.py @@ -41,7 +41,6 @@ async def test_build_connector_doc_produces_correct_fields(): connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, ) assert doc.title == "My Notion Page" @@ -51,29 +50,11 @@ async def test_build_connector_doc_produces_correct_fields(): assert doc.search_space_id == _SEARCH_SPACE_ID assert doc.connector_id == _CONNECTOR_ID assert doc.created_by_id == _USER_ID - assert doc.should_summarize is True assert doc.metadata["page_title"] == "My Notion Page" assert doc.metadata["page_id"] == "abc-123" assert doc.metadata["connector_id"] == _CONNECTOR_ID assert doc.metadata["document_type"] == "Notion Page" assert doc.metadata["connector_type"] == "Notion" - assert doc.fallback_summary is not None - assert "My Notion Page" in doc.fallback_summary - assert markdown in doc.fallback_summary - - -async def test_build_connector_doc_summary_disabled(): - """When enable_summary is False, should_summarize is False.""" - doc = _build_connector_doc( - _make_page(), - "# content", - connector_id=_CONNECTOR_ID, - search_space_id=_SEARCH_SPACE_ID, - user_id=_USER_ID, - enable_summary=False, - ) - - assert doc.should_summarize is False # --------------------------------------------------------------------------- @@ -81,10 +62,9 @@ async def test_build_connector_doc_summary_disabled(): # --------------------------------------------------------------------------- -def _mock_connector(enable_summary: bool = True): +def _mock_connector(): c = MagicMock() c.config = {"access_token": "tok"} - c.enable_summary = enable_summary c.last_indexed_at = None return c diff --git a/surfsense_backend/tests/unit/connector_indexers/test_onedrive_parallel.py b/surfsense_backend/tests/unit/connector_indexers/test_onedrive_parallel.py index 396d79e73..eb1451938 100644 --- a/surfsense_backend/tests/unit/connector_indexers/test_onedrive_parallel.py +++ b/surfsense_backend/tests/unit/connector_indexers/test_onedrive_parallel.py @@ -65,7 +65,6 @@ async def test_single_file_returns_one_connector_document( connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, ) assert len(docs) == 1 @@ -91,7 +90,6 @@ async def test_multiple_files_all_produce_documents( connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, ) assert len(docs) == 3 @@ -119,7 +117,6 @@ async def test_one_download_exception_does_not_block_others( connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, ) assert len(docs) == 2 @@ -146,7 +143,6 @@ async def test_etl_error_counts_as_download_failure( connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, ) assert len(docs) == 1 @@ -185,7 +181,6 @@ async def test_concurrency_bounded_by_semaphore( connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, max_concurrency=2, ) @@ -225,7 +220,6 @@ async def test_heartbeat_fires_during_parallel_downloads( connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, on_heartbeat=_on_heartbeat, ) diff --git a/surfsense_backend/tests/unit/connector_indexers/test_page_limits.py b/surfsense_backend/tests/unit/connector_indexers/test_page_limits.py index 573ee43d8..a79ed7858 100644 --- a/surfsense_backend/tests/unit/connector_indexers/test_page_limits.py +++ b/surfsense_backend/tests/unit/connector_indexers/test_page_limits.py @@ -180,7 +180,6 @@ async def _run_gdrive_selected(mocks, file_ids): connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, ) @@ -336,10 +335,6 @@ def gdrive_full_scan_mocks(monkeypatch): monkeypatch.setattr( _mod, "IndexingPipelineService", MagicMock(return_value=pipeline_mock) ) - monkeypatch.setattr( - _mod, "get_user_long_context_llm", AsyncMock(return_value=MagicMock()) - ) - return { "mod": _mod, "session": session, @@ -366,7 +361,6 @@ async def _run_gdrive_full_scan(mocks, max_files=500): MagicMock(), max_files, include_subfolders=False, - enable_summary=True, ) @@ -454,10 +448,6 @@ async def test_gdrive_delta_sync_skips_over_quota(monkeypatch): monkeypatch.setattr( _mod, "IndexingPipelineService", MagicMock(return_value=pipeline_mock) ) - monkeypatch.setattr( - _mod, "get_user_long_context_llm", AsyncMock(return_value=MagicMock()) - ) - mock_task_logger = MagicMock() mock_task_logger.log_task_progress = AsyncMock() @@ -473,7 +463,6 @@ async def test_gdrive_delta_sync_skips_over_quota(monkeypatch): mock_task_logger, MagicMock(), max_files=500, - enable_summary=True, ) call_files = download_mock.call_args[0][1] @@ -539,7 +528,6 @@ async def _run_onedrive_selected(mocks, file_ids): connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, ) @@ -641,7 +629,6 @@ async def _run_dropbox_selected(mocks, file_paths): connector_id=_CONNECTOR_ID, search_space_id=_SEARCH_SPACE_ID, user_id=_USER_ID, - enable_summary=True, ) diff --git a/surfsense_backend/tests/unit/gateway/test_webhook_routes.py b/surfsense_backend/tests/unit/gateway/test_webhook_routes.py index 34d0651ab..338a35c39 100644 --- a/surfsense_backend/tests/unit/gateway/test_webhook_routes.py +++ b/surfsense_backend/tests/unit/gateway/test_webhook_routes.py @@ -69,6 +69,13 @@ def _signed_slack_request(payload: dict, *, secret: str = "signing-secret") -> R ) +def _enable_slack_gateway(monkeypatch): + monkeypatch.setattr(routes.config, "GATEWAY_SLACK_ENABLED", True) + monkeypatch.setattr(routes.config, "GATEWAY_SLACK_CLIENT_ID", "client-id") + monkeypatch.setattr(routes.config, "GATEWAY_SLACK_CLIENT_SECRET", "client-secret") + monkeypatch.setattr(routes.config, "GATEWAY_SLACK_SIGNING_SECRET", "signing-secret") + + async def _call_webhook(*, request: RequestStub, account_id: int, session): return await routes.telegram_webhook( request=request, @@ -207,7 +214,7 @@ def test_verify_slack_signature_accepts_valid_signature(): @pytest.mark.asyncio async def test_slack_webhook_url_verification(monkeypatch, mocker): - monkeypatch.setattr(routes.config, "GATEWAY_SLACK_SIGNING_SECRET", "signing-secret") + _enable_slack_gateway(monkeypatch) request = _signed_slack_request({"type": "url_verification", "challenge": "abc123"}) response = await routes.slack_webhook(request=request, session=mocker.AsyncMock()) @@ -218,7 +225,7 @@ async def test_slack_webhook_url_verification(monkeypatch, mocker): @pytest.mark.asyncio async def test_slack_webhook_persists_event(monkeypatch, mocker): - monkeypatch.setattr(routes.config, "GATEWAY_SLACK_SIGNING_SECRET", "signing-secret") + _enable_slack_gateway(monkeypatch) session = mocker.AsyncMock() monkeypatch.setattr(routes, "get_slack_account_by_team", mocker.AsyncMock(return_value=_slack_account())) persist = mocker.AsyncMock(return_value=100) @@ -248,7 +255,7 @@ async def test_slack_webhook_persists_event(monkeypatch, mocker): @pytest.mark.asyncio async def test_slack_webhook_ignores_self_event(monkeypatch, mocker): - monkeypatch.setattr(routes.config, "GATEWAY_SLACK_SIGNING_SECRET", "signing-secret") + _enable_slack_gateway(monkeypatch) session = mocker.AsyncMock() monkeypatch.setattr(routes, "get_slack_account_by_team", mocker.AsyncMock(return_value=_slack_account())) persist = mocker.AsyncMock(return_value=100) @@ -275,7 +282,7 @@ async def test_slack_webhook_ignores_self_event(monkeypatch, mocker): @pytest.mark.asyncio -async def test_discord_gateway_install_returns_oauth_url(monkeypatch): +async def test_discord_gateway_install_returns_oauth_url(monkeypatch, mocker): monkeypatch.setattr(routes.config, "DISCORD_CLIENT_ID", "discord-client") monkeypatch.setattr( routes.config, @@ -283,10 +290,12 @@ async def test_discord_gateway_install_returns_oauth_url(monkeypatch): "http://localhost:8000/api/v1/gateway/discord/callback", ) monkeypatch.setattr(routes.config, "SECRET_KEY", "test-secret") + monkeypatch.setattr(routes, "check_search_space_access", mocker.AsyncMock()) response = await routes.install_discord_gateway( search_space_id=123, user=SimpleNamespace(id="00000000-0000-0000-0000-000000000001"), + session=mocker.AsyncMock(), ) assert response["auth_url"].startswith("https://discord.com/api/oauth2/authorize?") diff --git a/surfsense_backend/tests/unit/indexing_pipeline/test_connector_document.py b/surfsense_backend/tests/unit/indexing_pipeline/test_connector_document.py index 2136f2152..f85c632ef 100644 --- a/surfsense_backend/tests/unit/indexing_pipeline/test_connector_document.py +++ b/surfsense_backend/tests/unit/indexing_pipeline/test_connector_document.py @@ -18,7 +18,6 @@ def test_valid_document_created_with_required_fields(): connector_id=42, created_by_id="00000000-0000-0000-0000-000000000001", ) - assert doc.should_summarize is True assert doc.should_use_code_chunker is False assert doc.metadata == {} assert doc.connector_id == 42 diff --git a/surfsense_backend/tests/unit/indexing_pipeline/test_document_summarizer.py b/surfsense_backend/tests/unit/indexing_pipeline/test_document_summarizer.py deleted file mode 100644 index eee32357f..000000000 --- a/surfsense_backend/tests/unit/indexing_pipeline/test_document_summarizer.py +++ /dev/null @@ -1,41 +0,0 @@ -from unittest.mock import MagicMock - -import pytest - -from app.indexing_pipeline.document_summarizer import summarize_document - -pytestmark = pytest.mark.unit - - -@pytest.mark.usefixtures("patched_summarizer_chain") -async def test_without_metadata_returns_raw_summary(): - """Summarizer returns the LLM output directly when no metadata is provided.""" - result = await summarize_document("# Content", llm=MagicMock(model="gpt-4")) - - assert result == "The summary." - - -@pytest.mark.usefixtures("patched_summarizer_chain") -async def test_with_metadata_includes_metadata_values_in_output(): - """Non-empty metadata values are prepended to the summary output.""" - result = await summarize_document( - "# Content", - llm=MagicMock(model="gpt-4"), - metadata={"author": "Alice", "source": "Notion"}, - ) - - assert "Alice" in result - assert "Notion" in result - - -@pytest.mark.usefixtures("patched_summarizer_chain") -async def test_with_metadata_omits_empty_fields_from_output(): - """Empty metadata fields are omitted from the summary output.""" - result = await summarize_document( - "# Content", - llm=MagicMock(model="gpt-4"), - metadata={"author": "Alice", "description": ""}, - ) - - assert "Alice" in result - assert "description" not in result.lower() diff --git a/surfsense_backend/tests/unit/indexing_pipeline/test_index_batch.py b/surfsense_backend/tests/unit/indexing_pipeline/test_index_batch.py index dd9940503..963ac6792 100644 --- a/surfsense_backend/tests/unit/indexing_pipeline/test_index_batch.py +++ b/surfsense_backend/tests/unit/indexing_pipeline/test_index_batch.py @@ -37,12 +37,10 @@ async def test_calls_prepare_then_index_per_document(pipeline, make_connector_do orm2 = MagicMock(spec=Document) orm2.unique_identifier_hash = compute_unique_identifier_hash(doc2) - mock_llm = MagicMock() - pipeline.prepare_for_indexing = AsyncMock(return_value=[orm1, orm2]) - pipeline.index = AsyncMock(side_effect=lambda doc, cdoc, llm: doc) + pipeline.index = AsyncMock(side_effect=lambda doc, cdoc: doc) - results = await pipeline.index_batch([doc1, doc2], mock_llm) + results = await pipeline.index_batch([doc1, doc2]) pipeline.prepare_for_indexing.assert_awaited_once_with([doc1, doc2]) assert pipeline.index.await_count == 2 @@ -53,7 +51,7 @@ async def test_empty_input_returns_empty(pipeline): """Empty connector_docs list returns empty result.""" pipeline.prepare_for_indexing = AsyncMock(return_value=[]) - results = await pipeline.index_batch([], MagicMock()) + results = await pipeline.index_batch([]) assert results == [] @@ -74,7 +72,7 @@ async def test_skips_document_without_matching_connector_doc( pipeline.prepare_for_indexing = AsyncMock(return_value=[orphan_orm]) pipeline.index = AsyncMock() - results = await pipeline.index_batch([doc1], MagicMock()) + results = await pipeline.index_batch([doc1]) pipeline.index.assert_not_awaited() assert results == [] diff --git a/surfsense_backend/tests/unit/indexing_pipeline/test_index_batch_parallel.py b/surfsense_backend/tests/unit/indexing_pipeline/test_index_batch_parallel.py index 07e388836..3a1b77d90 100644 --- a/surfsense_backend/tests/unit/indexing_pipeline/test_index_batch_parallel.py +++ b/surfsense_backend/tests/unit/indexing_pipeline/test_index_batch_parallel.py @@ -51,11 +51,6 @@ async def test_index_calls_embed_and_chunk_via_to_thread( return await original_to_thread(func, *args, **kwargs) monkeypatch.setattr(asyncio, "to_thread", tracking_to_thread) - - monkeypatch.setattr( - "app.indexing_pipeline.indexing_pipeline_service.summarize_document", - AsyncMock(return_value="Summary."), - ) mock_chunk_hybrid = MagicMock(return_value=["chunk1"]) mock_chunk_hybrid.__name__ = "chunk_text_hybrid" monkeypatch.setattr( @@ -85,7 +80,7 @@ async def test_index_calls_embed_and_chunk_via_to_thread( document.id = 1 document.status = DocumentStatus.pending() - await pipeline.index(document, connector_doc, llm=MagicMock()) + await pipeline.index(document, connector_doc) # Either chunker entry point satisfies the "chunking runs off the event # loop" contract this test guards. Routing between the two is verified @@ -104,10 +99,6 @@ async def test_non_code_documents_use_hybrid_chunker( mid-row. Only documents flagged with ``should_use_code_chunker=True`` should take the ``chunk_text`` path. """ - monkeypatch.setattr( - "app.indexing_pipeline.indexing_pipeline_service.summarize_document", - AsyncMock(return_value="Summary."), - ) mock_chunk_hybrid = MagicMock(return_value=["chunk1"]) mock_chunk_hybrid.__name__ = "chunk_text_hybrid" monkeypatch.setattr( @@ -139,7 +130,7 @@ async def test_non_code_documents_use_hybrid_chunker( document.id = 1 document.status = DocumentStatus.pending() - await pipeline.index(document, connector_doc, llm=MagicMock()) + await pipeline.index(document, connector_doc) mock_chunk_hybrid.assert_called_once() mock_chunk_code.assert_not_called() @@ -192,19 +183,14 @@ async def test_batch_parallel_indexes_all_documents( index_calls = [] - async def fake_index(self, document, connector_doc, llm): + async def fake_index(self, document, connector_doc): index_calls.append(document.id) document.status = DocumentStatus.ready() return document monkeypatch.setattr(IndexingPipelineService, "index", fake_index) - async def mock_get_llm(session): - return MagicMock() - - _, indexed, failed = await pipeline.index_batch_parallel( - docs, mock_get_llm, max_concurrency=2 - ) + _, indexed, failed = await pipeline.index_batch_parallel(docs, max_concurrency=2) assert indexed == 3 assert failed == 0 @@ -233,20 +219,15 @@ async def test_batch_parallel_one_failure_does_not_affect_others( _mock_session_factory(orm_by_id), ) - async def failing_index(self, document, connector_doc, llm): + async def failing_index(self, document, connector_doc): if document.id == 2: - raise RuntimeError("LLM exploded") + raise RuntimeError("Indexing exploded") document.status = DocumentStatus.ready() return document monkeypatch.setattr(IndexingPipelineService, "index", failing_index) - async def mock_get_llm(session): - return MagicMock() - - _, indexed, failed = await pipeline.index_batch_parallel( - docs, mock_get_llm, max_concurrency=4 - ) + _, indexed, failed = await pipeline.index_batch_parallel(docs, max_concurrency=4) assert indexed == 2 assert failed == 1 diff --git a/surfsense_backend/tests/unit/tasks/chat/streaming/test_parallel_refactor_parity.py b/surfsense_backend/tests/unit/tasks/chat/streaming/test_parallel_refactor_parity.py index e014bb911..fa6d8b9e2 100644 --- a/surfsense_backend/tests/unit/tasks/chat/streaming/test_parallel_refactor_parity.py +++ b/surfsense_backend/tests/unit/tasks/chat/streaming/test_parallel_refactor_parity.py @@ -246,6 +246,8 @@ def test_new_chat_runtime_context_prefers_accepted_folder_ids() -> None: mentioned_document_ids=[1, 2], accepted_folder_ids=[10], mentioned_folder_ids=[20, 30], + mentioned_connector_ids=None, + mentioned_connectors=None, request_id="req", turn_id="t1", ) @@ -263,6 +265,8 @@ def test_new_chat_runtime_context_falls_back_to_mentioned_folder_ids() -> None: mentioned_document_ids=None, accepted_folder_ids=[], mentioned_folder_ids=[20, 30], + mentioned_connector_ids=None, + mentioned_connectors=None, request_id=None, turn_id="t2", ) diff --git a/surfsense_evals/README.md b/surfsense_evals/README.md index c6314af80..c755c4de6 100644 --- a/surfsense_evals/README.md +++ b/surfsense_evals/README.md @@ -137,15 +137,14 @@ Notes: - `--skip-unanswerable` (run) — drop unanswerable questions - `--docs ,` (run) — scope to specific docs -## Ingestion knobs (vision LLM, processing mode, summarize) +## Ingestion knobs (vision LLM, processing mode) -The harness exposes `POST /api/v1/documents/fileupload`'s three knobs on every `ingest` subcommand: +The harness exposes `POST /api/v1/documents/fileupload`'s ingest knobs on every `ingest` subcommand: | Flag pair | Effect | |--------------------------------------------|-----------------------------------------------------------------------------------------| | `--use-vision-llm` / `--no-vision-llm` | Walk every embedded image in the PDF and inline image-derived text at the image's position (see below). | | `--processing-mode {basic,premium}` | `premium` carries a 10× page multiplier and routes to a stronger ETL (e.g. LlamaCloud). | -| `--should-summarize` / `--no-summarize` | Generate a per-document summary at ingest. | The "Default ingest" column in the benchmarks table is what runs if you don't pass any flag. Whatever was actually used is recorded as a `__settings__` header in the doc map (`data//maps/_*_map.jsonl`) and as `extra.ingest_settings` in `run_artifact.json`, then surfaced in the report — no need to hunt through CLI history. diff --git a/surfsense_evals/src/surfsense_evals/core/clients/documents.py b/surfsense_evals/src/surfsense_evals/core/clients/documents.py index 02bcf74da..362aae53b 100644 --- a/surfsense_evals/src/surfsense_evals/core/clients/documents.py +++ b/surfsense_evals/src/surfsense_evals/core/clients/documents.py @@ -110,7 +110,6 @@ class DocumentsClient: files: Iterable[Path], *, search_space_id: int, - should_summarize: bool = False, use_vision_llm: bool = False, processing_mode: str = "basic", ) -> FileUploadResult: @@ -149,7 +148,6 @@ class DocumentsClient: f"{self._base}/api/v1/documents/fileupload", data={ "search_space_id": str(search_space_id), - "should_summarize": "true" if should_summarize else "false", "use_vision_llm": "true" if use_vision_llm else "false", "processing_mode": processing_mode, }, diff --git a/surfsense_evals/src/surfsense_evals/core/clients/search_space.py b/surfsense_evals/src/surfsense_evals/core/clients/search_space.py index 37fa69f80..e2d37694d 100644 --- a/surfsense_evals/src/surfsense_evals/core/clients/search_space.py +++ b/surfsense_evals/src/surfsense_evals/core/clients/search_space.py @@ -83,7 +83,6 @@ class LlmPreferences: """ agent_llm_id: int | None - document_summary_llm_id: int | None image_generation_config_id: int | None vision_llm_config_id: int | None agent_llm: dict[str, Any] | None @@ -93,7 +92,6 @@ class LlmPreferences: def from_payload(cls, payload: dict[str, Any]) -> LlmPreferences: return cls( agent_llm_id=payload.get("agent_llm_id"), - document_summary_llm_id=payload.get("document_summary_llm_id"), image_generation_config_id=payload.get("image_generation_config_id"), vision_llm_config_id=payload.get("vision_llm_config_id"), agent_llm=payload.get("agent_llm"), @@ -154,7 +152,6 @@ class SearchSpaceClient: search_space_id: int, *, agent_llm_id: int | None = None, - document_summary_llm_id: int | None = None, image_generation_config_id: int | None = None, vision_llm_config_id: int | None = None, ) -> LlmPreferences: @@ -167,8 +164,6 @@ class SearchSpaceClient: body: dict[str, Any] = {} if agent_llm_id is not None: body["agent_llm_id"] = agent_llm_id - if document_summary_llm_id is not None: - body["document_summary_llm_id"] = document_summary_llm_id if image_generation_config_id is not None: body["image_generation_config_id"] = image_generation_config_id if vision_llm_config_id is not None: diff --git a/surfsense_evals/src/surfsense_evals/core/ingest_settings.py b/surfsense_evals/src/surfsense_evals/core/ingest_settings.py index 5cdece577..8328e0d46 100644 --- a/surfsense_evals/src/surfsense_evals/core/ingest_settings.py +++ b/surfsense_evals/src/surfsense_evals/core/ingest_settings.py @@ -8,15 +8,13 @@ exactly three knobs (verified at * ``processing_mode`` — ``"basic"`` (default) | ``"premium"`` * ``use_vision_llm`` — ``bool`` (run vision LLM during ingest to extract image content / captions / tables) -* ``should_summarize`` — ``bool`` (generate document summary) This module gives every benchmark a uniform way to: 1. Receive sensible per-benchmark defaults (text-only benchmarks default vision off; image-bearing benchmarks default vision on). 2. Accept CLI overrides (``--use-vision-llm`` / ``--no-vision-llm``, - ``--processing-mode {basic,premium}``, - ``--should-summarize`` / ``--no-summarize``). + ``--processing-mode {basic,premium}``). 3. Persist the *actual* settings used into the doc-map manifest and the run artifact so reports can show "vision=ON, mode=premium → 65% accuracy" head-to-head with "vision=OFF, mode=basic → 52%". @@ -71,13 +69,11 @@ class IngestSettings: use_vision_llm: bool = False processing_mode: str = "basic" - should_summarize: bool = False def to_dict(self) -> dict[str, Any]: return { "use_vision_llm": self.use_vision_llm, "processing_mode": self.processing_mode, - "should_summarize": self.should_summarize, } @classmethod @@ -87,14 +83,13 @@ class IngestSettings: ``opts`` is the kwargs dict built by ``core.cli`` from the argparse namespace (see ``_cmd_ingest`` / ``_cmd_run``). Keys we look for: ``use_vision_llm`` (bool or None), ``processing_mode`` - (str or None), ``should_summarize`` (bool or None). Anything + (str or None). Anything else is ignored so benchmarks can pass through their own opts. """ return cls( use_vision_llm=_coerce_bool(opts.get("use_vision_llm"), defaults.use_vision_llm), processing_mode=_coerce_mode(opts.get("processing_mode"), defaults.processing_mode), - should_summarize=_coerce_bool(opts.get("should_summarize"), defaults.should_summarize), ) def render_label(self) -> str: @@ -102,8 +97,7 @@ class IngestSettings: return ( f"vision={'on' if self.use_vision_llm else 'off'}, " - f"mode={self.processing_mode}, " - f"summarize={'on' if self.should_summarize else 'off'}" + f"mode={self.processing_mode}" ) @@ -179,14 +173,14 @@ def add_ingest_settings_args( *, defaults: IngestSettings, ) -> None: - """Attach the three ingest-settings flag pairs to ``parser``. + """Attach ingest-settings flags to ``parser``. - Each bool exposes a mutually exclusive ``--foo`` / ``--no-foo`` - pair so an operator can flip either direction without restating - every flag. Default is ``None`` so that "operator didn't pass the - flag" is distinguishable from "operator explicitly passed false" - — ``IngestSettings.merge`` then folds in the benchmark default - only when the operator was silent. + The vision bool exposes a mutually exclusive ``--foo`` / ``--no-foo`` + pair so an operator can flip either direction without restating every + flag. Default is ``None`` so that "operator didn't pass the flag" is + distinguishable from "operator explicitly passed false" — + ``IngestSettings.merge`` then folds in the benchmark default only when + the operator was silent. """ settings_group = parser.add_argument_group( @@ -217,18 +211,6 @@ def add_ingest_settings_args( f"Default for this benchmark: {defaults.processing_mode!r}." ), ) - _add_bool_pair( - settings_group, - dest="should_summarize", - on_flag="--should-summarize", - off_flag="--no-summarize", - on_help=( - "Have SurfSense generate a document summary at ingest " - f"(default for this benchmark: " - f"{'on' if defaults.should_summarize else 'off'})." - ), - off_help="Skip per-document summary generation.", - ) # --------------------------------------------------------------------------- @@ -292,10 +274,9 @@ def format_ingest_settings_md(settings: Any) -> str: return "- SurfSense ingest settings: (not recorded — re-ingest to capture)" vision = "on" if settings.get("use_vision_llm") else "off" mode = settings.get("processing_mode") or "basic" - summarize = "on" if settings.get("should_summarize") else "off" return ( f"- SurfSense ingest settings: vision_llm=`{vision}`, " - f"processing_mode=`{mode}`, summarize=`{summarize}`" + f"processing_mode=`{mode}`" ) diff --git a/surfsense_evals/src/surfsense_evals/suites/medical/cure/ingest.py b/surfsense_evals/src/surfsense_evals/suites/medical/cure/ingest.py index 6eca8810c..275e28ce5 100644 --- a/surfsense_evals/src/surfsense_evals/suites/medical/cure/ingest.py +++ b/surfsense_evals/src/surfsense_evals/suites/medical/cure/ingest.py @@ -160,8 +160,7 @@ async def run_ingest( upload_result = await docs_client.upload( files=[b.path for b in batches], search_space_id=ctx.search_space_id, - should_summarize=settings.should_summarize, - use_vision_llm=settings.use_vision_llm, + use_vision_llm=settings.use_vision_llm, processing_mode=settings.processing_mode, ) new_doc_ids = list(upload_result.document_ids) diff --git a/surfsense_evals/src/surfsense_evals/suites/medical/cure/runner.py b/surfsense_evals/src/surfsense_evals/suites/medical/cure/runner.py index 416912b14..041e0e8b5 100644 --- a/surfsense_evals/src/surfsense_evals/suites/medical/cure/runner.py +++ b/surfsense_evals/src/surfsense_evals/suites/medical/cure/runner.py @@ -63,7 +63,6 @@ _DESCRIPTION = "CUREv1 retrieval (single-arm SurfSense): Recall@k / MRR / nDCG@1 _DEFAULT_INGEST_SETTINGS = IngestSettings( use_vision_llm=False, processing_mode="basic", - should_summarize=False, ) diff --git a/surfsense_evals/src/surfsense_evals/suites/medical/medxpertqa/ingest.py b/surfsense_evals/src/surfsense_evals/suites/medical/medxpertqa/ingest.py index 5293e116f..ff43c7049 100644 --- a/surfsense_evals/src/surfsense_evals/suites/medical/medxpertqa/ingest.py +++ b/surfsense_evals/src/surfsense_evals/suites/medical/medxpertqa/ingest.py @@ -208,7 +208,6 @@ async def _upload_pdfs( result = await docs_client.upload( files=batch, search_space_id=ctx.search_space_id, - should_summarize=settings.should_summarize, use_vision_llm=settings.use_vision_llm, processing_mode=settings.processing_mode, ) diff --git a/surfsense_evals/src/surfsense_evals/suites/medical/medxpertqa/runner.py b/surfsense_evals/src/surfsense_evals/suites/medical/medxpertqa/runner.py index 75646ef32..e1a830138 100644 --- a/surfsense_evals/src/surfsense_evals/suites/medical/medxpertqa/runner.py +++ b/surfsense_evals/src/surfsense_evals/suites/medical/medxpertqa/runner.py @@ -169,7 +169,6 @@ _DESCRIPTION = ( _DEFAULT_INGEST_SETTINGS = IngestSettings( use_vision_llm=True, processing_mode="basic", - should_summarize=False, ) diff --git a/surfsense_evals/src/surfsense_evals/suites/medical/mirage/ingest.py b/surfsense_evals/src/surfsense_evals/suites/medical/mirage/ingest.py index 9769d078b..59006b6c0 100644 --- a/surfsense_evals/src/surfsense_evals/suites/medical/mirage/ingest.py +++ b/surfsense_evals/src/surfsense_evals/suites/medical/mirage/ingest.py @@ -480,7 +480,6 @@ async def run_ingest( upload_result = await docs_client.upload( files=[b.path for b in batches], search_space_id=ctx.search_space_id, - should_summarize=settings.should_summarize, use_vision_llm=settings.use_vision_llm, processing_mode=settings.processing_mode, ) diff --git a/surfsense_evals/src/surfsense_evals/suites/medical/mirage/runner.py b/surfsense_evals/src/surfsense_evals/suites/medical/mirage/runner.py index 0f336c0d5..b01b645a9 100644 --- a/surfsense_evals/src/surfsense_evals/suites/medical/mirage/runner.py +++ b/surfsense_evals/src/surfsense_evals/suites/medical/mirage/runner.py @@ -48,7 +48,6 @@ _DESCRIPTION = "MIRAGE (7,663 medical MCQs) — single-arm SurfSense per-task ac _DEFAULT_INGEST_SETTINGS = IngestSettings( use_vision_llm=False, processing_mode="basic", - should_summarize=False, ) diff --git a/surfsense_evals/src/surfsense_evals/suites/multimodal_doc/mmlongbench/ingest.py b/surfsense_evals/src/surfsense_evals/suites/multimodal_doc/mmlongbench/ingest.py index cf0572df8..15cdbeb77 100644 --- a/surfsense_evals/src/surfsense_evals/suites/multimodal_doc/mmlongbench/ingest.py +++ b/surfsense_evals/src/surfsense_evals/suites/multimodal_doc/mmlongbench/ingest.py @@ -225,7 +225,6 @@ async def _upload_pdfs( result = await docs_client.upload( files=batch, search_space_id=ctx.search_space_id, - should_summarize=settings.should_summarize, use_vision_llm=settings.use_vision_llm, processing_mode=settings.processing_mode, ) diff --git a/surfsense_evals/src/surfsense_evals/suites/multimodal_doc/mmlongbench/runner.py b/surfsense_evals/src/surfsense_evals/suites/multimodal_doc/mmlongbench/runner.py index 0e352d7ae..95a1e15eb 100644 --- a/surfsense_evals/src/surfsense_evals/suites/multimodal_doc/mmlongbench/runner.py +++ b/surfsense_evals/src/surfsense_evals/suites/multimodal_doc/mmlongbench/runner.py @@ -178,7 +178,6 @@ _TEXT_ONLY_HINTS = ("gpt-5.4-mini", "gpt-3.5", "text-only", "instruct-") _DEFAULT_INGEST_SETTINGS = IngestSettings( use_vision_llm=True, processing_mode="basic", - should_summarize=False, ) diff --git a/surfsense_evals/src/surfsense_evals/suites/research/crag/ingest.py b/surfsense_evals/src/surfsense_evals/suites/research/crag/ingest.py index aad6a70bf..4e0c2bdc5 100644 --- a/surfsense_evals/src/surfsense_evals/suites/research/crag/ingest.py +++ b/surfsense_evals/src/surfsense_evals/suites/research/crag/ingest.py @@ -189,7 +189,6 @@ async def _upload_pages( result = await docs_client.upload( files=batch, search_space_id=ctx.search_space_id, - should_summarize=settings.should_summarize, use_vision_llm=settings.use_vision_llm, processing_mode=settings.processing_mode, ) @@ -306,8 +305,7 @@ async def run_ingest( settings = settings or IngestSettings( use_vision_llm=False, processing_mode="basic", - should_summarize=False, - ) + ) bench_dir = ctx.benchmark_data_dir() pages_dir = bench_dir / "pages" raw_cache = bench_dir / ".raw_cache" diff --git a/surfsense_evals/src/surfsense_evals/suites/research/crag/runner.py b/surfsense_evals/src/surfsense_evals/suites/research/crag/runner.py index 710f76744..8b759e0d8 100644 --- a/surfsense_evals/src/surfsense_evals/suites/research/crag/runner.py +++ b/surfsense_evals/src/surfsense_evals/suites/research/crag/runner.py @@ -177,7 +177,6 @@ _DESCRIPTION = ( _DEFAULT_INGEST_SETTINGS = IngestSettings( use_vision_llm=False, processing_mode="basic", - should_summarize=False, ) diff --git a/surfsense_evals/src/surfsense_evals/suites/research/frames/ingest.py b/surfsense_evals/src/surfsense_evals/suites/research/frames/ingest.py index 9780be4ed..98e035f28 100644 --- a/surfsense_evals/src/surfsense_evals/suites/research/frames/ingest.py +++ b/surfsense_evals/src/surfsense_evals/suites/research/frames/ingest.py @@ -136,7 +136,6 @@ async def _upload_markdowns( result = await docs_client.upload( files=batch, search_space_id=ctx.search_space_id, - should_summarize=settings.should_summarize, use_vision_llm=settings.use_vision_llm, processing_mode=settings.processing_mode, ) @@ -240,8 +239,7 @@ async def run_ingest( settings = settings or IngestSettings( use_vision_llm=False, processing_mode="basic", - should_summarize=False, - ) + ) bench_dir = ctx.benchmark_data_dir() wiki_cache = bench_dir / "wiki" wiki_cache.mkdir(parents=True, exist_ok=True) diff --git a/surfsense_evals/src/surfsense_evals/suites/research/frames/runner.py b/surfsense_evals/src/surfsense_evals/suites/research/frames/runner.py index a8dde0dd2..9c0e16b00 100644 --- a/surfsense_evals/src/surfsense_evals/suites/research/frames/runner.py +++ b/surfsense_evals/src/surfsense_evals/suites/research/frames/runner.py @@ -153,7 +153,6 @@ _DESCRIPTION = ( _DEFAULT_INGEST_SETTINGS = IngestSettings( use_vision_llm=False, processing_mode="basic", - should_summarize=False, ) diff --git a/surfsense_evals/tests/core/test_clients.py b/surfsense_evals/tests/core/test_clients.py index 9e2c4ad75..611408703 100644 --- a/surfsense_evals/tests/core/test_clients.py +++ b/surfsense_evals/tests/core/test_clients.py @@ -69,7 +69,7 @@ async def test_set_llm_preferences_partial_update(respx_mock, http): 200, json={ "agent_llm_id": -10042, - "document_summary_llm_id": None, + "agent_llm_id": None, "image_generation_config_id": None, "vision_llm_config_id": None, "agent_llm": { diff --git a/surfsense_evals/tests/core/test_ingest_settings.py b/surfsense_evals/tests/core/test_ingest_settings.py index acfac57a6..fd7e7818a 100644 --- a/surfsense_evals/tests/core/test_ingest_settings.py +++ b/surfsense_evals/tests/core/test_ingest_settings.py @@ -4,7 +4,7 @@ Covers: * ``IngestSettings.merge`` honours operator overrides and falls back to per-benchmark defaults when the operator is silent. -* ``add_ingest_settings_args`` exposes the three flag pairs and +* ``add_ingest_settings_args`` exposes ingest settings flags and argparse defaults of ``None`` correctly distinguish "not passed" from "explicitly false". * ``settings_header_line`` / ``read_settings_header`` round-trip @@ -40,7 +40,7 @@ from surfsense_evals.core.ingest_settings import ( class TestMerge: def test_silent_operator_uses_defaults(self) -> None: - defaults = IngestSettings(use_vision_llm=True, processing_mode="basic", should_summarize=True) + defaults = IngestSettings(use_vision_llm=True, processing_mode="basic") merged = IngestSettings.merge(defaults, {}) assert merged == defaults @@ -111,17 +111,16 @@ class TestMerge: assert merged.processing_mode == "basic" def test_to_dict_round_trips(self) -> None: - s = IngestSettings(use_vision_llm=True, processing_mode="premium", should_summarize=False) + s = IngestSettings(use_vision_llm=True, processing_mode="premium") d = s.to_dict() assert d == { "use_vision_llm": True, "processing_mode": "premium", - "should_summarize": False, } def test_render_label_format(self) -> None: - s = IngestSettings(use_vision_llm=True, processing_mode="premium", should_summarize=True) - assert s.render_label() == "vision=on, mode=premium, summarize=on" + s = IngestSettings(use_vision_llm=True, processing_mode="premium") + assert s.render_label() == "vision=on, mode=premium" # --------------------------------------------------------------------------- @@ -136,7 +135,7 @@ class TestAddArgs: add_ingest_settings_args( p, defaults=IngestSettings( - use_vision_llm=False, processing_mode="basic", should_summarize=False + use_vision_llm=False, processing_mode="basic" ), ) return p @@ -145,7 +144,6 @@ class TestAddArgs: args = parser.parse_args([]) assert args.use_vision_llm is None assert args.processing_mode is None - assert args.should_summarize is None def test_use_vision_llm_flag(self, parser: argparse.ArgumentParser) -> None: args = parser.parse_args(["--use-vision-llm"]) @@ -166,12 +164,6 @@ class TestAddArgs: with pytest.raises(SystemExit): parser.parse_args(["--processing-mode", "exotic"]) - def test_summarize_flag_pair(self, parser: argparse.ArgumentParser) -> None: - on = parser.parse_args(["--should-summarize"]) - assert on.should_summarize is True - off = parser.parse_args(["--no-summarize"]) - assert off.should_summarize is False - def test_vision_flags_mutually_exclusive( self, parser: argparse.ArgumentParser ) -> None: @@ -185,11 +177,11 @@ class TestAddArgs: ["--use-vision-llm", "--processing-mode", "premium"] ) defaults = IngestSettings( - use_vision_llm=False, processing_mode="basic", should_summarize=False + use_vision_llm=False, processing_mode="basic" ) merged = IngestSettings.merge(defaults, vars(args)) assert merged == IngestSettings( - use_vision_llm=True, processing_mode="premium", should_summarize=False + use_vision_llm=True, processing_mode="premium" ) @@ -249,19 +241,17 @@ class TestHeader: class TestFormatMd: def test_full_settings(self) -> None: out = format_ingest_settings_md( - {"use_vision_llm": True, "processing_mode": "premium", "should_summarize": True} + {"use_vision_llm": True, "processing_mode": "premium"} ) assert "vision_llm=`on`" in out assert "processing_mode=`premium`" in out - assert "summarize=`on`" in out def test_default_off(self) -> None: out = format_ingest_settings_md( - {"use_vision_llm": False, "processing_mode": "basic", "should_summarize": False} + {"use_vision_llm": False, "processing_mode": "basic"} ) assert "vision_llm=`off`" in out assert "processing_mode=`basic`" in out - assert "summarize=`off`" in out def test_missing_returns_re_ingest_hint(self) -> None: # Empty dict + None + non-mapping should all degrade gracefully. diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-definition-section.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-definition-section.tsx index 4ff9b8b8c..4d6382a60 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-definition-section.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-definition-section.tsx @@ -1,6 +1,8 @@ "use client"; -import { ListOrdered, Settings2, Tag, Target } from "lucide-react"; +import { Dot } from "lucide-react"; +import { useState } from "react"; import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"; import type { AutomationDefinition } from "@/contracts/types/automation.types"; import { ExecutionSummary } from "./execution-summary"; import { InputsSchemaPreview } from "./inputs-schema-preview"; @@ -11,34 +13,30 @@ interface AutomationDefinitionSectionProps { } /** - * The Definition card. Read view; editing happens on the sibling /edit - * route (Edit button in the header). Layout is top-down: - * goal → tags → execution defaults → inputs schema (if any) → plan - * - * The schema_version is rendered as a small badge next to the section - * title so it's discoverable but doesn't fight for attention. + * User-facing read view of the saved automation definition. Editing happens on + * the sibling /edit route; this card should summarize behavior, not expose the + * raw persisted schema. */ export function AutomationDefinitionSection({ definition }: AutomationDefinitionSectionProps) { const hasTags = definition.metadata.tags.length > 0; const hasInputs = !!definition.inputs; + const [advancedOpen, setAdvancedOpen] = useState(false); + const stepCount = `${definition.plan.length} step${definition.plan.length === 1 ? "" : "s"}`; return ( - - Definition - - v{definition.schema_version} - + + Automation details {definition.goal && ( - +

{definition.goal}

)} {hasTags && ( - +
{definition.metadata.tags.map((tag) => ( )} - - - - {hasInputs && ( - + {definition.inputs && } )} + Plan + + {stepCount} + + } >
{definition.plan.map((step, idx) => ( ))}
+ + + {advancedOpen ? "Hide advanced options" : "Advanced options"} + + +
+
+ Execution defaults +
+ +
+
+
@@ -78,20 +90,15 @@ export function AutomationDefinitionSection({ definition }: AutomationDefinition } function Field({ - icon: Icon, label, children, }: { - icon: typeof Target; - label: string; + label: React.ReactNode; children: React.ReactNode; }) { return (
-
- - {label} -
+
{label}
{children}
); diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-detail-header.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-detail-header.tsx index 0bce3fa2d..4a6537385 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-detail-header.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-detail-header.tsx @@ -8,7 +8,6 @@ import { updateAutomationMutationAtom } from "@/atoms/automations/automations-mu import { Button } from "@/components/ui/button"; import { Spinner } from "@/components/ui/spinner"; import type { Automation } from "@/contracts/types/automation.types"; -import { AutomationStatusBadge } from "../../components/automation-status-badge"; import { DeleteAutomationDialog } from "../../components/delete-automation-dialog"; interface AutomationDetailHeaderProps { @@ -70,12 +69,9 @@ export function AutomationDetailHeader({
-
-

- {automation.name} -

- -
+

+ {automation.name} +

{automation.description && (

{automation.description}

)} @@ -83,9 +79,15 @@ export function AutomationDetailHeader({
{canUpdate && ( - @@ -93,28 +95,30 @@ export function AutomationDetailHeader({ {canToggle && ( )} {canDelete && ( )} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-runs-section.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-runs-section.tsx index d31bd696d..bd683fe57 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-runs-section.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-runs-section.tsx @@ -27,7 +27,6 @@ export function AutomationRunsSection({ automationId }: AutomationRunsSectionPro
- Recent runs

diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-triggers-section.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-triggers-section.tsx index 558a089ac..2f4eea7b8 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-triggers-section.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/automation-triggers-section.tsx @@ -27,7 +27,7 @@ export function AutomationTriggersSection({ Triggers

- When this automation fires. v1 supports scheduled triggers only. + When this automation runs

diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/execution-summary.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/execution-summary.tsx index 5c4dc381c..82abce173 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/execution-summary.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/execution-summary.tsx @@ -15,8 +15,8 @@ export function ExecutionSummary({ execution }: ExecutionSummaryProps) {
- - + + {execution.on_failure.length > 0 && ( diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/inputs-schema-preview.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/inputs-schema-preview.tsx index 29d79d99b..dce6ac4a7 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/inputs-schema-preview.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/inputs-schema-preview.tsx @@ -1,5 +1,4 @@ "use client"; -import { JsonView } from "@/components/json-view"; import type { Inputs } from "@/contracts/types/automation.types"; interface InputsSchemaPreviewProps { @@ -13,9 +12,63 @@ interface InputsSchemaPreviewProps { * is null. */ export function InputsSchemaPreview({ inputs }: InputsSchemaPreviewProps) { + const fields = getInputFields(inputs.schema); + + if (fields.length === 0) { + return

No extra inputs are required.

; + } + return ( -
- +
+ {fields.map((field) => ( +
+
+
{field.name}
+ {field.description ? ( +
{field.description}
+ ) : null} +
+
+ {field.type} + {field.required ? " · required" : ""} +
+
+ ))}
); } + +function getInputFields(schema: Record): { + name: string; + type: string; + description?: string; + required: boolean; +}[] { + const properties = schema.properties; + if (!properties || typeof properties !== "object" || Array.isArray(properties)) { + return []; + } + + const required = new Set(Array.isArray(schema.required) ? schema.required : []); + return Object.entries(properties as Record).map(([name, value]) => { + const field = value && typeof value === "object" && !Array.isArray(value) ? value : {}; + return { + name, + type: formatType((field as Record).type), + description: + typeof (field as Record).description === "string" + ? ((field as Record).description as string) + : undefined, + required: required.has(name), + }; + }); +} + +function formatType(value: unknown): string { + if (Array.isArray(value)) return value.join(" or "); + if (typeof value === "string") return value; + return "value"; +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/plan-step-card.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/plan-step-card.tsx index 27cecf3bf..15a285322 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/plan-step-card.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/plan-step-card.tsx @@ -1,6 +1,4 @@ "use client"; -import { ArrowRightCircle, GitCommitHorizontal } from "lucide-react"; -import { JsonView } from "@/components/json-view"; import type { PlanStep } from "@/contracts/types/automation.types"; interface PlanStepCardProps { @@ -9,62 +7,35 @@ interface PlanStepCardProps { } /** - * Read-only view of one plan step. Renders the step_id + action prominently, - * then a definition list of the per-step knobs, and finally the params as - * formatted JSON. Editable mode is out of scope here — definition edits live - * on the (future) raw-JSON path. + * Read-only view of one plan step. Keep this user-facing: summarize what the + * step does and only show advanced step controls when they are explicitly set. */ export function PlanStepCard({ step, index }: PlanStepCardProps) { + const title = getStepTitle(step); + const details = getStepDetails(step); + return ( -
-
- +
+
+ {index + 1} - {step.step_id} - - {step.action} -
- -
- {(step.when || - step.output_as || - step.max_retries != null || - step.timeout_seconds != null) && ( -
- {step.when && ( - {step.when}} /> - )} - {step.output_as && ( - {step.output_as}} - /> - )} - {step.max_retries != null && ( - - )} - {step.timeout_seconds != null && ( - - )} -
- )} - -
-
- - Params -
-
- -
+
+

{title}

+ {details.length > 0 ? ( +
+ {details.map((detail) => ( + + ))} +
+ ) : null}
); } -function DefRow({ label, value }: { label: string; value: React.ReactNode }) { +function DefRow({ label, value }: { label: string; value: string }) { return (
{label}:
@@ -72,3 +43,104 @@ function DefRow({ label, value }: { label: string; value: React.ReactNode }) {
); } + +function getStepTitle(step: PlanStep): string { + if (step.action === "agent_task") { + return readStringParam(step.params, "query") ?? "Run an agent task"; + } + return sentenceCase(formatAction(step.action)); +} + +function getStepDetails(step: PlanStep): { label: string; value: string }[] { + const details: { label: string; value: string }[] = []; + + if (step.action === "agent_task") { + if (typeof step.params.auto_approve_all === "boolean") { + details.push({ + label: "Approval", + value: step.params.auto_approve_all ? "Auto-approve agent actions" : "Ask before actions", + }); + } + + const mentionSummary = summarizeMentions(step.params); + if (mentionSummary) { + details.push({ label: "Scope", value: mentionSummary }); + } + } else { + const readableParams = Object.entries(step.params) + .filter(([, value]) => value !== null && value !== undefined && value !== "") + .map(([key, value]) => `${sentenceCase(formatKey(key))}: ${formatValue(value)}`); + if (readableParams.length > 0) { + details.push({ label: "Details", value: readableParams.join(" · ") }); + } + } + + if (step.when) details.push({ label: "Runs when", value: step.when }); + if (step.output_as) details.push({ label: "Saves output as", value: step.output_as }); + if (step.max_retries != null) details.push({ label: "Max retries", value: String(step.max_retries) }); + if (step.timeout_seconds != null) details.push({ label: "Timeout", value: `${step.timeout_seconds}s` }); + + return details; +} + +function readStringParam(params: Record, key: string): string | null { + const value = params[key]; + return typeof value === "string" && value.trim() ? value : null; +} + +function summarizeMentions(params: Record): string | null { + const parts: string[] = []; + addMentionTitles(parts, params.mentioned_documents, "Documents and folders"); + addMentionTitles(parts, params.mentioned_connectors, "Connectors"); + if (parts.length === 0) { + addCount(parts, params.mentioned_document_ids, "document"); + addCount(parts, params.mentioned_folder_ids, "folder"); + addCount(parts, params.mentioned_connector_ids, "connector"); + } + return parts.length > 0 ? parts.join(", ") : null; +} + +function addMentionTitles(parts: string[], value: unknown, label: string): void { + if (!Array.isArray(value) || value.length === 0) return; + const titles = value + .map((entry) => { + const record = asRecord(entry); + const title = typeof record.title === "string" ? record.title : null; + const accountName = typeof record.account_name === "string" ? record.account_name : null; + return title ?? accountName; + }) + .filter((title): title is string => !!title); + if (titles.length === 0) return; + parts.push(`${label}: ${titles.join(", ")}`); +} + +function addCount(parts: string[], value: unknown, singular: string): void { + if (!Array.isArray(value) || value.length === 0) return; + parts.push(`${value.length} ${singular}${value.length === 1 ? "" : "s"}`); +} + +function formatAction(action: string): string { + return formatKey(action); +} + +function formatKey(key: string): string { + return key.replace(/_/g, " "); +} + +function sentenceCase(value: string): string { + return value.charAt(0).toUpperCase() + value.slice(1); +} + +function asRecord(value: unknown): Record { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : {}; +} + +function formatValue(value: unknown): string { + if (typeof value === "boolean") return value ? "Yes" : "No"; + if (typeof value === "string" || typeof value === "number") return String(value); + if (Array.isArray(value)) return `${value.length} item${value.length === 1 ? "" : "s"}`; + if (value && typeof value === "object") return "Configured"; + return String(value); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/run-details-panel.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/run-details-panel.tsx index 1a54ac0e5..ab82589dc 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/run-details-panel.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/run-details-panel.tsx @@ -53,7 +53,11 @@ export function RunDetailsPanel({ const isTerminal = liveStatus !== "pending" && liveStatus !== "running"; // Defer the REST round-trip until the run can actually carry heavy // fields — output/artifacts/error are only written at terminal mark. - const { data: run, isLoading, error } = useAutomationRun(automationId, runId, { + const { + data: run, + isLoading, + error, + } = useAutomationRun(automationId, runId, { enabled: isTerminal, }); diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/trigger-card.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/trigger-card.tsx index 681877523..6e6f84bd0 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/trigger-card.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/components/trigger-card.tsx @@ -1,15 +1,36 @@ "use client"; import { useAtomValue } from "jotai"; -import { AlertCircle, CalendarClock, Clock, Pencil, Save, Trash2 } from "lucide-react"; +import { AlertCircle, MoreHorizontal, Pencil, Trash2 } from "lucide-react"; import { useState } from "react"; import { updateTriggerMutationAtom } from "@/atoms/automations/automations-mutation.atoms"; -import { JsonView } from "@/components/json-view"; +import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; import { Button } from "@/components/ui/button"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; +import { Input } from "@/components/ui/input"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; import { Spinner } from "@/components/ui/spinner"; import { Switch } from "@/components/ui/switch"; import { type Trigger, triggerUpdateRequest } from "@/contracts/types/automation.types"; import { describeCron } from "@/lib/automations/describe-cron"; -import { formatRelativeDate, formatRelativeFutureDate } from "@/lib/format-date"; +import { formatRelativeFutureDate } from "@/lib/format-date"; +import { + DEFAULT_SCHEDULE, + fromCron, + type ScheduleFrequency, + toCron, +} from "@/lib/automations/schedule-builder"; +import { TimezoneCombobox } from "../../components/builder/timezone-combobox"; import { DeleteTriggerDialog } from "./delete-trigger-dialog"; interface TriggerCardProps { @@ -19,27 +40,58 @@ interface TriggerCardProps { canDelete: boolean; } +type SimpleFrequency = Extract | "custom"; + interface TriggerDraft { - params: Record; - static_inputs: Record; + frequency: SimpleFrequency; + hour: number; + minute: number; + timezone: string; + cron: string; } +const SIMPLE_FREQUENCIES = new Set(["hourly", "daily", "weekdays"]); + function draftFromTrigger(trigger: Trigger): TriggerDraft { + const cron = typeof trigger.params.cron === "string" ? trigger.params.cron : ""; + const timezone = typeof trigger.params.timezone === "string" ? trigger.params.timezone : "UTC"; + const model = fromCron(cron); + if (model && SIMPLE_FREQUENCIES.has(model.frequency)) { + return { + frequency: model.frequency as SimpleFrequency, + hour: model.hour, + minute: model.minute, + timezone, + cron, + }; + } return { - params: trigger.params, - static_inputs: trigger.static_inputs ?? {}, + frequency: "custom", + hour: DEFAULT_SCHEDULE.hour, + minute: DEFAULT_SCHEDULE.minute, + timezone, + cron, }; } +function pad(value: number): string { + return value.toString().padStart(2, "0"); +} + +function clampInt(raw: string, min: number, max: number): number { + const value = Number.parseInt(raw, 10); + if (Number.isNaN(value)) return min; + return Math.min(max, Math.max(min, value)); +} + /** * One trigger row in the Triggers section of the detail page. Renders: - * - type icon + human-readable schedule + timezone - * - last_fired_at / next_fire_at hints - * - static_inputs as formatted JSON (when present) - * - enable toggle + remove button + inline edit (each gated independently) + * - human-readable schedule + * - compact enable toggle + * - dropdown actions for edit/remove * - * Inline edit covers ``params`` and ``static_inputs`` — the two fields the - * backend ``PATCH /triggers/[id]`` endpoint accepts beyond ``enabled``. + * Inline edit keeps schedule editing intentionally small: common frequencies, + * time, timezone, and raw cron only for schedules outside the simple model. * ``enabled`` stays on the Switch so the two surfaces don't fight. */ export function TriggerCard({ trigger, automationId, canUpdate, canDelete }: TriggerCardProps) { @@ -51,10 +103,9 @@ export function TriggerCard({ trigger, automationId, canUpdate, canDelete }: Tri const [issues, setIssues] = useState([]); const cron = typeof trigger.params.cron === "string" ? trigger.params.cron : undefined; - const tz = typeof trigger.params.timezone === "string" ? trigger.params.timezone : "UTC"; const human = cron ? describeCron(cron) : trigger.type; - const triggerLabel = cron ? `${human} · ${tz}` : trigger.type; - const hasStaticInputs = Object.keys(trigger.static_inputs ?? {}).length > 0; + const triggerLabel = human; + const showActions = (canUpdate && !isEditing) || canDelete; async function handleToggle(checked: boolean) { await updateTrigger({ @@ -77,7 +128,22 @@ export function TriggerCard({ trigger, automationId, canUpdate, canDelete }: Tri async function saveEdit() { setIssues([]); - const result = triggerUpdateRequest.safeParse(draft); + const params = + draft.frequency === "custom" + ? { cron: draft.cron.trim(), timezone: draft.timezone } + : { + cron: toCron({ + ...DEFAULT_SCHEDULE, + frequency: draft.frequency, + hour: draft.hour, + minute: draft.minute, + }), + timezone: draft.timezone, + }; + const result = triggerUpdateRequest.safeParse({ + params, + static_inputs: trigger.static_inputs ?? {}, + }); if (!result.success) { setIssues( result.error.issues.map((issue) => `${issue.path.join(".") || "(root)"}: ${issue.message}`) @@ -98,134 +164,206 @@ export function TriggerCard({ trigger, automationId, canUpdate, canDelete }: Tri return ( <> -
-
-
- -
-
- {human} - · {tz} -
- {cron && {cron}} -
-
+
+
+
{human}
-
+
{canUpdate && ( -
- - {trigger.enabled ? "Enabled" : "Off"} - - -
+ )} - {canUpdate && !isEditing && ( - - )} - {canDelete && ( - + {showActions && ( + + + + + + {canUpdate && !isEditing && ( + + + Edit + + )} + {canDelete && ( + setDeleteOpen(true)}> + + Delete + + )} + + )}
-
- {isEditing ? ( - <> -
- setDraft(next as TriggerDraft)} - collapsed={false} - /> + {!isEditing && trigger.next_fire_at ? ( +
+
+ Next fire: +
+
+ {formatRelativeFutureDate(trigger.next_fire_at)} +
+
+ ) : null} + + {isEditing ? ( +
+
+
+ +
- {issues.length > 0 && ( -
-
- - {issues.length === 1 ? "1 issue" : `${issues.length} issues`} -
-
    + {draft.frequency === "hourly" ? ( +
    + + + setDraft((prev) => ({ + ...prev, + minute: clampInt(event.target.value, 0, 59), + })) + } + /> +
    + ) : draft.frequency !== "custom" ? ( +
    + + { + const [hour, minute] = event.target.value.split(":"); + setDraft((prev) => ({ + ...prev, + hour: clampInt(hour, 0, 23), + minute: clampInt(minute, 0, 59), + })); + }} + /> +
    + ) : ( +
    + + + setDraft((prev) => ({ ...prev, cron: event.target.value })) + } + /> +
    + )} + +
    +
    Timezone
    + setDraft((prev) => ({ ...prev, timezone }))} + /> +
    +
+ + {issues.length > 0 && ( + + + + {issues.length === 1 ? "1 issue" : `${issues.length} issues`} + + +
    {issues.map((issue) => (
  • {issue}
  • ))}
-
- )} + + + )} -
- - -
- - ) : ( - <> - {(trigger.last_fired_at || trigger.next_fire_at) && ( -
- {trigger.next_fire_at && ( - - )} - {trigger.last_fired_at && ( - - )} -
- )} - - {hasStaticInputs && ( -
-
Static inputs
-
- -
-
- )} - - )} -
+
+ + +
+
+ ) : null}
{canDelete && ( @@ -240,35 +378,3 @@ export function TriggerCard({ trigger, automationId, canUpdate, canDelete }: Tri ); } - -function TimeRow({ - label, - iso, - tense, - highlight = false, -}: { - label: string; - iso: string; - tense: "past" | "future"; - highlight?: boolean; -}) { - const formatted = tense === "future" ? formatRelativeFutureDate(iso) : formatRelativeDate(iso); - return ( - <> -
- - {label} -
-
- {formatted} -
- - ); -} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/edit/automation-edit-content.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/edit/automation-edit-content.tsx index 2c9db217d..c05bff7d9 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/edit/automation-edit-content.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/edit/automation-edit-content.tsx @@ -51,9 +51,17 @@ export function AutomationEditContent({ searchSpaceId, automationId }: Automatio } return ( - <> - - - + ( + + )} + /> ); } diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/edit/components/automation-edit-header.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/edit/components/automation-edit-header.tsx index 6b2a31822..ca477220e 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/edit/components/automation-edit-header.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/[automation_id]/edit/components/automation-edit-header.tsx @@ -1,15 +1,21 @@ "use client"; import { ArrowLeft } from "lucide-react"; import Link from "next/link"; +import type { ReactNode } from "react"; import { Button } from "@/components/ui/button"; import type { Automation } from "@/contracts/types/automation.types"; interface AutomationEditHeaderProps { automation: Automation; searchSpaceId: number; + modeSwitcher?: ReactNode; } -export function AutomationEditHeader({ automation, searchSpaceId }: AutomationEditHeaderProps) { +export function AutomationEditHeader({ + automation, + searchSpaceId, + modeSwitcher, +}: AutomationEditHeaderProps) { const detailHref = `/dashboard/${searchSpaceId}/automations/${automation.id}`; return ( @@ -20,11 +26,11 @@ export function AutomationEditHeader({ automation, searchSpaceId }: AutomationEd Back to automation -
+

Edit automation

-

{automation.name}

+ {modeSwitcher ?
{modeSwitcher}
: null}
); diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/automations-content.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/automations-content.tsx index 756221d38..d9c949058 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/automations-content.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/automations-content.tsx @@ -1,5 +1,6 @@ "use client"; -import { ShieldAlert } from "lucide-react"; +import { AlertCircle, ShieldAlert } from "lucide-react"; +import { Alert, AlertDescription } from "@/components/ui/alert"; import { useAutomations } from "@/hooks/use-automations"; import { AutomationsEmptyState } from "./components/automations-empty-state"; import { AutomationsHeader } from "./components/automations-header"; @@ -60,9 +61,10 @@ export function AutomationsContent({ searchSpaceId }: AutomationsContentProps) { loading={false} canCreate={perms.canCreate} /> -
-

Couldn't load automations. {error.message}

-
+ + + Couldn't load automations {error.message} + ); } diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/components/automation-row-actions.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/components/automation-row-actions.tsx index 229a417dc..95ee23445 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/components/automation-row-actions.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/components/automation-row-actions.tsx @@ -8,7 +8,6 @@ import { DropdownMenu, DropdownMenuContent, DropdownMenuItem, - DropdownMenuSeparator, DropdownMenuTrigger, } from "@/components/ui/dropdown-menu"; import type { AutomationSummary } from "@/contracts/types/automation.types"; @@ -58,25 +57,21 @@ export function AutomationRowActions({ - + {canToggle && ( {pauseLabel} )} - {canToggle && canDelete && } {canDelete && ( - setDeleteOpen(true)} - className="text-destructive focus:text-destructive" - > + setDeleteOpen(true)}> Delete diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/components/automation-row.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/components/automation-row.tsx index a59fb4527..74c95cee4 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/components/automation-row.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/components/automation-row.tsx @@ -26,35 +26,30 @@ export function AutomationRow({ canDelete, }: AutomationRowProps) { return ( - - -
- - {automation.name} - - {automation.description && ( - - {automation.description} - - )} -
+ + + + {automation.name} + - + - + {formatRelativeDate(automation.updated_at)} - - + +
+ +
); diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/components/automation-status-badge.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/components/automation-status-badge.tsx index ecf171e78..22e1be222 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/components/automation-status-badge.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/components/automation-status-badge.tsx @@ -1,5 +1,4 @@ "use client"; -import { Archive, CircleDot, Pause } from "lucide-react"; import type { AutomationStatus } from "@/contracts/types/automation.types"; import { cn } from "@/lib/utils"; @@ -8,41 +7,37 @@ interface AutomationStatusBadgeProps { className?: string; } -// Color + icon per status. Active = green, paused = amber, archived = muted. +// Small borderless status pills, matching model-selector badges. const STATUS_STYLES: Record< AutomationStatus, - { label: string; icon: typeof CircleDot; classes: string } + { label: string; classes: string } > = { active: { label: "Active", - icon: CircleDot, classes: - "bg-emerald-50 text-emerald-700 border border-emerald-200 dark:bg-emerald-950/40 dark:text-emerald-300 dark:border-emerald-900/50", + "bg-emerald-100 text-emerald-700 dark:bg-emerald-900/50 dark:text-emerald-300", }, paused: { label: "Paused", - icon: Pause, classes: - "bg-amber-50 text-amber-700 border border-amber-200 dark:bg-amber-950/40 dark:text-amber-300 dark:border-amber-900/50", + "bg-amber-100 text-amber-700 dark:bg-amber-900/50 dark:text-amber-300", }, archived: { label: "Archived", - icon: Archive, - classes: "bg-muted text-muted-foreground border border-border/60", + classes: "bg-muted text-muted-foreground", }, }; export function AutomationStatusBadge({ status, className }: AutomationStatusBadgeProps) { - const { label, icon: Icon, classes } = STATUS_STYLES[status]; + const { label, classes } = STATUS_STYLES[status]; return ( - {label} ); diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/components/automations-empty-state.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/components/automations-empty-state.tsx index cc54c5e94..b2e7b2532 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/components/automations-empty-state.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/components/automations-empty-state.tsx @@ -1,5 +1,5 @@ "use client"; -import { MessageSquarePlus, SquarePen, Workflow } from "lucide-react"; +import { Workflow } from "lucide-react"; import Link from "next/link"; import { Button } from "@/components/ui/button"; @@ -28,16 +28,14 @@ export function AutomationsEmptyState({ searchSpaceId, canCreate }: AutomationsE {canCreate ? (
-
) : ( diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/components/automations-header.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/components/automations-header.tsx index 137727f60..5c1fcb507 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/components/automations-header.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/components/automations-header.tsx @@ -1,5 +1,4 @@ "use client"; -import { MessageSquarePlus, SquarePen } from "lucide-react"; import Link from "next/link"; import { Button } from "@/components/ui/button"; @@ -41,17 +40,16 @@ export function AutomationsHeader({
{canCreate && showCreateCta && (
-
)} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/components/automations-table.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/components/automations-table.tsx index ec3aeeef5..8314a5179 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/components/automations-table.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/components/automations-table.tsx @@ -1,5 +1,5 @@ "use client"; -import { Activity, CalendarDays, Workflow } from "lucide-react"; +import { CalendarDays, Info, Workflow } from "lucide-react"; import { Table, TableBody, TableHead, TableHeader, TableRow } from "@/components/ui/table"; import type { AutomationSummary } from "@/contracts/types/automation.types"; import { AutomationRow } from "./automation-row"; @@ -37,7 +37,7 @@ export function AutomationsTable({ - + Status diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/advanced-section.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/advanced-section.tsx index 740f199af..110de57f6 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/advanced-section.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/advanced-section.tsx @@ -58,7 +58,7 @@ export function AdvancedSection({ return (
- + - + - + {BACKOFF_OPTIONS.map((option) => ( {option.label} @@ -105,7 +105,7 @@ export function AdvancedSection({ - + {CONCURRENCY_OPTIONS.map((option) => ( {option.label} diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/automation-builder-form.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/automation-builder-form.tsx index 39904dfa0..59967080f 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/automation-builder-form.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/automation-builder-form.tsx @@ -1,8 +1,8 @@ "use client"; import { useAtomValue } from "jotai"; -import { Code2, LayoutList, Save } from "lucide-react"; -import Link from "next/link"; +import { AlertCircle, Code2, LayoutList } from "lucide-react"; import { useRouter } from "next/navigation"; +import type { ReactNode } from "react"; import { useMemo, useState } from "react"; import type { z } from "zod"; import { @@ -12,9 +12,12 @@ import { updateAutomationMutationAtom, updateTriggerMutationAtom, } from "@/atoms/automations/automations-mutation.atoms"; +import { Alert, AlertDescription } from "@/components/ui/alert"; import { Button } from "@/components/ui/button"; import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Separator } from "@/components/ui/separator"; import { Spinner } from "@/components/ui/spinner"; +import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; import { type Automation, @@ -35,7 +38,6 @@ import { hasResolvedModels, hydrateForm, } from "@/lib/automations/builder-schema"; -import { cn } from "@/lib/utils"; import { AdvancedSection } from "./advanced-section"; import { AutomationModelFields } from "./automation-model-fields"; import { BasicsSection } from "./basics-section"; @@ -56,6 +58,7 @@ interface AutomationBuilderFormProps { * eligibility itself is now owned by the in-form pickers. */ submitDisabledReason?: string; + renderModeSwitcher?: (modeSwitcher: ReactNode) => ReactNode; } type Mode = "form" | "json"; @@ -78,6 +81,7 @@ export function AutomationBuilderForm({ searchSpaceId, automation, submitDisabledReason, + renderModeSwitcher, }: AutomationBuilderFormProps) { const router = useRouter(); const { mutateAsync: createAutomation } = useAtomValue(createAutomationMutationAtom); @@ -97,7 +101,7 @@ export function AutomationBuilderForm({ return { mode: "json" as Mode, form: createEmptyForm(), - notice: `This automation ${result.reason}, which the form can't show. Edit it as JSON below.`, + notice: `This automation ${result.reason}, which the form can't show. Edit it as JSON below`, }; } return { mode: "form" as Mode, form: createEmptyForm(), notice: undefined }; @@ -116,11 +120,6 @@ export function AutomationBuilderForm({ const [submitting, setSubmitting] = useState(false); - const cancelHref = - mode === "edit" && automation - ? `/dashboard/${searchSpaceId}/automations/${automation.id}` - : `/dashboard/${searchSpaceId}/automations`; - // Eligible models + the search-space-seeded defaults. Models are chosen per // automation on create; in edit mode the backend preserves the captured // snapshot, so the picker is create-only. @@ -192,7 +191,7 @@ export function AutomationBuilderForm({ // form's own validation enforces completeness on submit. const definition = jsonValue.definition; if (!definition || typeof definition !== "object") { - return { ok: false, issues: [], notice: "Add a definition before switching to the form." }; + return { ok: false, issues: [], notice: "Add a definition before switching to the form" }; } const name = @@ -210,7 +209,7 @@ export function AutomationBuilderForm({ const h = hydrateForm(name, description, definition, triggers); return h.formable ? { ok: true, form: h.form } - : { ok: false, issues: [], notice: `Can't show in the form: it ${h.reason}.` }; + : { ok: false, issues: [], notice: `Can't show in the form: it ${h.reason}` }; } function validateForm(): Record | null { @@ -328,119 +327,133 @@ export function AutomationBuilderForm({ : undefined); // Only gate creation; editing an existing automation isn't blocked here. const submitBlocked = mode === "create" && !!effectiveDisabledReason; + const modeSwitcher = ( + { + if (value === activeMode) return; + if (value === "form") switchToForm(); + else if (value === "json") switchToJson(); + }} + > + + + + Form + + + + Edit as JSON + + + + ); return (
-
-
- (activeMode === "form" ? undefined : switchToForm())} - /> - (activeMode === "json" ? undefined : switchToJson())} - /> -
-
+ {renderModeSwitcher ? ( + renderModeSwitcher(modeSwitcher) + ) : ( +
{modeSwitcher}
+ )} {activeMode === "json" ? ( - - - - - + ) : (
-
- - - Basics - - - - - - - - - Tasks - - - patchForm({ tasks })} - /> - patchForm({ unattended })} - /> - - - - - - Schedule - - - patchForm({ schedule })} - onTimezoneChange={(timezone) => patchForm({ timezone })} - /> - - - - - - Models - - - patchForm({ models: { ...form.models, ...patch } })} - /> - - - - - - Settings - - - - patchForm({ execution: { ...form.execution, ...patch } }) - } - onTagsChange={(tags) => patchForm({ tags })} - /> - +
+ +
+ + Basics + + + + +
+ +
+ + Tasks + + + patchForm({ tasks })} + /> + patchForm({ unattended })} + /> + +
+ +
+ + Schedule + + + patchForm({ schedule })} + onTimezoneChange={(timezone) => patchForm({ timezone })} + /> + +
+ +
+ + Models + + + patchForm({ models: { ...form.models, ...patch } })} + /> + +
+ +
+ + Settings + + + + patchForm({ execution: { ...form.execution, ...patch } }) + } + onTagsChange={(tags) => patchForm({ tags })} + /> + +
- + Summary @@ -452,12 +465,14 @@ export function AutomationBuilderForm({
)} - {rootError &&

{rootError}

} + {rootError && ( + + + {rootError} + + )}
- {submitBlocked ? ( @@ -470,7 +485,6 @@ export function AutomationBuilderForm({ className="cursor-not-allowed opacity-50" onClick={(event) => event.preventDefault()} > - {submitLabel} @@ -481,14 +495,11 @@ export function AutomationBuilderForm({ type="button" size="sm" disabled={submitting} + className="relative" onClick={() => (activeMode === "json" ? submitJson() : submitForm())} > - {submitting ? ( - - ) : ( - - )} - {submitLabel} + {submitLabel} + {submitting && } )}
@@ -496,34 +507,6 @@ export function AutomationBuilderForm({ ); } -function ModeButton({ - active, - icon: Icon, - label, - onClick, -}: { - active: boolean; - icon: typeof Code2; - label: string; - onClick: () => void; -}) { - return ( - - ); -} - function extractTriggers(raw: unknown): HydratableTrigger[] { if (!Array.isArray(raw)) return []; return raw.map((entry) => { diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/automation-model-fields.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/automation-model-fields.tsx index 8ca8d839c..2c4a0bf60 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/automation-model-fields.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/automation-model-fields.tsx @@ -118,15 +118,14 @@ const ModelSelectField = memo(function ModelSelectField({ if (kind.options.length === 0) { return ( - + No eligible models - - Automations need a premium or your own (BYOK) model. Set one up in{" "} + + Use a premium model or your own (BYOK) model in{" "} role settings - . @@ -155,7 +154,7 @@ const ModelSelectField = memo(function ModelSelectField({ )} - + {premium.length > 0 ? ( Premium diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/builder-summary.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/builder-summary.tsx index 21a77cb5f..55059ab53 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/builder-summary.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/builder-summary.tsx @@ -1,5 +1,5 @@ "use client"; -import { CalendarClock, CheckCircle2, ListOrdered, type LucideIcon, XCircle } from "lucide-react"; +import { Dot } from "lucide-react"; import { type BuilderForm, scheduleToCron } from "@/lib/automations/builder-schema"; import { describeCron } from "@/lib/automations/describe-cron"; @@ -12,85 +12,70 @@ interface BuilderSummaryProps { * chat ``AutomationDraftPreview`` so the two creation paths feel consistent. */ export function BuilderSummary({ form }: BuilderSummaryProps) { - const scheduleLabel = form.schedule - ? `${describeCron(scheduleToCron(form.schedule))} · ${form.timezone}` - : "No schedule — won't run automatically"; + const automationName = form.name.trim() || "Untitled automation"; + const scheduleDescription = form.schedule ? describeCron(scheduleToCron(form.schedule)) : null; + const taskCountLabel = `${form.tasks.length} task${form.tasks.length === 1 ? "" : "s"}`; + const visibleTasks = form.tasks.slice(0, 2); + const hiddenTaskCount = form.tasks.length - visibleTasks.length; return ( -
-
-

{form.name.trim() || "Untitled automation"}

- {form.description?.trim() && ( -

{form.description.trim()}

- )} +
+
+

+ {automationName} +

-
-

{scheduleLabel}

-
+
-
-
    - {form.tasks.map((task, index) => ( -
  1. - - {index + 1} - - - - {task.query.trim() || ( - No instructions yet - )} +
    + + {scheduleDescription ? ( + + {scheduleDescription} + + {form.timezone} + + ) : ( + No schedule — won't run automatically + )} + + + +
      + {visibleTasks.map((task, index) => ( +
    1. + {index + 1}. + + {task.query.trim() || "No instructions yet"} - {task.mentions.length > 0 && ( - - {task.mentions.map((mention) => ( - - @{mention.title} - - ))} - - )} - -
    2. - ))} -
    -
+ + ))} + {hiddenTaskCount > 0 && ( +
  • +{hiddenTaskCount} more tasks
  • + )} + + -
    - {form.unattended ? ( - - ) : ( - - )} - {form.unattended ? "Runs without approval prompts" : "Will reject approval prompts"} + + {form.unattended ? "Runs without approval prompts" : "Approval prompts are rejected"} +
    ); } -function Section({ - icon: Icon, +function SummaryRow({ label, children, }: { - icon: LucideIcon; label: string; children: React.ReactNode; }) { return ( -
    -
    - - {label} -
    - {children} +
    +
    {label}
    +
    {children}
    ); } diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/json-mode-panel.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/json-mode-panel.tsx index 1f25f8a61..412533d36 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/json-mode-panel.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/json-mode-panel.tsx @@ -1,6 +1,7 @@ "use client"; -import { AlertCircle } from "lucide-react"; +import { AlertCircle, TriangleAlert } from "lucide-react"; import { JsonView } from "@/components/json-view"; +import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; interface JsonModePanelProps { value: Record; @@ -19,32 +20,32 @@ export function JsonModePanel({ value, issues, notice, onChange }: JsonModePanel return (
    {notice && ( -
    - {notice} -
    + + + {notice} + )} -
    - onChange(next as Record)} - collapsed={false} - /> -
    + onChange(next as Record)} + collapsed={false} + className="max-h-144 overflow-auto rounded-md border border-accent bg-accent/20" + /> {issues.length > 0 && ( -
    -
    - - {issues.length === 1 ? "1 issue" : `${issues.length} issues`} -
    -
      - {issues.map((issue) => ( -
    • {issue}
    • - ))} -
    -
    + + + {issues.length === 1 ? "1 issue" : `${issues.length} issues`} + +
      + {issues.map((issue) => ( +
    • {issue}
    • + ))} +
    +
    +
    )}
    ); diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/schedule-section.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/schedule-section.tsx index 401b4f5cb..810984acd 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/schedule-section.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/schedule-section.tsx @@ -1,5 +1,5 @@ "use client"; -import { CalendarClock, CalendarOff, Plus, X } from "lucide-react"; +import { CalendarClock, CalendarOff, Dot, Plus, X } from "lucide-react"; import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; import { @@ -70,17 +70,18 @@ export function ScheduleSection({ return (
    -
    +
    {label} - · {timezone} + + {timezone}
    diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/timezone-combobox.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/timezone-combobox.tsx index bc3b97542..ed0808bb3 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/timezone-combobox.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/timezone-combobox.tsx @@ -35,22 +35,26 @@ export function TimezoneCombobox({ value, onChange }: TimezoneComboboxProps) { variant="outline" role="combobox" aria-expanded={open} - className="w-full justify-between font-normal" + className="w-full justify-between border-popover-border bg-transparent font-normal hover:bg-transparent" > {value || "Select timezone"} - - + + No timezone found. - + {timezones.map((tz) => ( { onChange(tz); setOpen(false); diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/unattended-toggle.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/unattended-toggle.tsx index ba665445f..861f22204 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/unattended-toggle.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/components/builder/unattended-toggle.tsx @@ -1,7 +1,5 @@ "use client"; -import { Info } from "lucide-react"; import { Switch } from "@/components/ui/switch"; -import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; interface UnattendedToggleProps { checked: boolean; @@ -15,26 +13,15 @@ interface UnattendedToggleProps { */ export function UnattendedToggle({ checked, onChange }: UnattendedToggleProps) { return ( -
    +
    Run without asking for approvals - - - - - - Automations run unattended. With this off, any approval the agent asks for is - rejected, which can stall a step. - -

    - Auto-approve actions the agent would normally pause to confirm. + Tasks run automatically without asking for confirmation

    - - - + ( + + )} + /> ); } diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/new/components/automation-new-header.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/new/components/automation-new-header.tsx index ccfbbc9fa..de9e2412b 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/new/components/automation-new-header.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/new/components/automation-new-header.tsx @@ -1,38 +1,38 @@ "use client"; -import { ArrowLeft, MessageSquarePlus } from "lucide-react"; +import { ArrowLeft } from "lucide-react"; import Link from "next/link"; +import type { ReactNode } from "react"; import { Button } from "@/components/ui/button"; interface AutomationNewHeaderProps { searchSpaceId: number; + modeSwitcher?: ReactNode; } -export function AutomationNewHeader({ searchSpaceId }: AutomationNewHeaderProps) { +export function AutomationNewHeader({ searchSpaceId, modeSwitcher }: AutomationNewHeaderProps) { return (
    - +
    + + {modeSwitcher ?
    {modeSwitcher}
    : null} +

    New automation

    - Set up a task and a schedule. Prefer natural language? Use chat instead. + Configure the task, schedule, and execution settings for this automation.

    - + {modeSwitcher ?
    {modeSwitcher}
    : null}
    ); diff --git a/surfsense_web/app/dashboard/[search_space_id]/automations/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/automations/page.tsx index b77cb20f4..0502d2310 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/automations/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/automations/page.tsx @@ -8,7 +8,7 @@ export default async function AutomationsPage({ const { search_space_id } = await params; return ( -
    +
    ); diff --git a/surfsense_web/app/dashboard/[search_space_id]/buy-more/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/buy-more/page.tsx index 74bcaff2e..b4ec015b7 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/buy-more/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/buy-more/page.tsx @@ -1,6 +1,5 @@ "use client"; -import { motion } from "motion/react"; import { useState } from "react"; import { BuyPagesContent } from "@/components/settings/buy-pages-content"; import { BuyTokensContent } from "@/components/settings/buy-tokens-content"; @@ -17,12 +16,7 @@ export default function BuyMorePage() { const [activeTab, setActiveTab] = useState("pages"); return ( - +
    { @@ -49,6 +43,6 @@ export default function BuyMorePage() { - +
    ); } diff --git a/surfsense_web/app/dashboard/[search_space_id]/client-layout.tsx b/surfsense_web/app/dashboard/[search_space_id]/client-layout.tsx index 759539ce3..7c9fcb1a0 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/client-layout.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/client-layout.tsx @@ -47,14 +47,9 @@ export function DashboardClientLayout({ const { mutateAsync: updatePreferences } = useAtomValue(updateLLMPreferencesMutationAtom); const isOnboardingComplete = useCallback(() => { - // Check that both LLM IDs are set (including 0 for Auto mode) - return ( - preferences.agent_llm_id !== null && - preferences.agent_llm_id !== undefined && - preferences.document_summary_llm_id !== null && - preferences.document_summary_llm_id !== undefined - ); - }, [preferences]); + // Check that the Agent LLM ID is set, including 0 for Auto mode. + return preferences.agent_llm_id !== null && preferences.agent_llm_id !== undefined; + }, [preferences.agent_llm_id]); const { data: access = null, isLoading: accessLoading } = useAtomValue(myAccessAtom); const [hasCheckedOnboarding, setHasCheckedOnboarding] = useState(false); @@ -100,7 +95,6 @@ export function DashboardClientLayout({ search_space_id: Number(searchSpaceId), data: { agent_llm_id: firstGlobalConfig.id, - document_summary_llm_id: firstGlobalConfig.id, }, }); diff --git a/surfsense_web/app/dashboard/[search_space_id]/more-pages/loading.tsx b/surfsense_web/app/dashboard/[search_space_id]/more-pages/loading.tsx deleted file mode 100644 index ccb3b35e3..000000000 --- a/surfsense_web/app/dashboard/[search_space_id]/more-pages/loading.tsx +++ /dev/null @@ -1,10 +0,0 @@ -import { Skeleton } from "@/components/ui/skeleton"; - -export default function Loading() { - return ( -
    - - -
    - ); -} diff --git a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx index f8ca9bbc2..75cfa4184 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx @@ -18,6 +18,7 @@ import { disabledToolsAtom } from "@/atoms/agent-tools/agent-tools.atoms"; import { clearTargetCommentIdAtom, currentThreadAtom, + setCurrentThreadMetadataAtom, setTargetCommentIdAtom, } from "@/atoms/chat/current-thread.atom"; import { @@ -36,7 +37,7 @@ import { closeReportPanelAtom } from "@/atoms/chat/report-panel.atom"; import { type AgentCreatedDocument, agentCreatedDocumentsAtom } from "@/atoms/documents/ui.atoms"; import { closeEditorPanelAtom } from "@/atoms/editor/editor-panel.atom"; import { membersAtom } from "@/atoms/members/members-query.atoms"; -import { removeChatTabAtom, updateChatTabTitleAtom } from "@/atoms/tabs/tabs.atom"; +import { removeChatTabAtom, syncChatTabAtom, updateChatTabTitleAtom } from "@/atoms/tabs/tabs.atom"; import { currentUserAtom } from "@/atoms/user/user-query.atoms"; import { EditMessageDialog, @@ -50,6 +51,7 @@ import { TokenUsageProvider, } from "@/components/assistant-ui/token-usage-context"; import { Button } from "@/components/ui/button"; +import { Skeleton } from "@/components/ui/skeleton"; import { type HitlDecision, PendingInterruptProvider, @@ -64,6 +66,7 @@ import { } from "@/hooks/use-agent-actions-query"; import { useChatSessionStateSync } from "@/hooks/use-chat-session-state"; import { useMessagesSync } from "@/hooks/use-messages-sync"; +import { useThreadDetail, useThreadMessages } from "@/hooks/use-thread-queries"; import { getAgentFilesystemSelection } from "@/lib/agent-filesystem"; import { documentsApiService } from "@/lib/apis/documents-api.service"; import { getBearerToken } from "@/lib/auth-utils"; @@ -100,8 +103,6 @@ import { appendMessage, createThread, getRegenerateUrl, - getThreadFull, - getThreadMessages, type ThreadListItem, type ThreadListResponse, type ThreadRecord, @@ -119,7 +120,7 @@ import { trackChatMessageSent, trackChatResponseReceived, } from "@/lib/posthog/events"; -import Loading from "../loading"; +import { cacheKeys } from "@/lib/query-client/cache-keys"; const MobileEditorPanel = dynamic( () => @@ -287,11 +288,78 @@ function computeFallbackTurnCancellingRetryDelay(attempt: number): number { return Math.min(raw, TURN_CANCELLING_MAX_DELAY_MS); } +function parseUrlChatId(id: string | string[] | undefined): number { + let parsed = 0; + if (Array.isArray(id) && id.length > 0) { + parsed = Number.parseInt(id[0], 10); + } else if (typeof id === "string") { + parsed = Number.parseInt(id, 10); + } + return Number.isNaN(parsed) ? 0 : parsed; +} + +function ThreadMessagesSkeleton() { + return ( +
    +
    +
    +
    +
    + +
    + +
    + + + +
    + +
    + +
    + +
    + + + +
    + +
    + +
    +
    + +
    +
    + +
    +
    +
    +
    + ); +} + export default function NewChatPage() { const params = useParams(); const queryClient = useQueryClient(); - const [isInitializing, setIsInitializing] = useState(true); - const [threadId, setThreadId] = useState(null); + const urlChatId = useMemo(() => parseUrlChatId(params.chat_id), [params.chat_id]); + const [threadId, setThreadId] = useState(() => (urlChatId > 0 ? urlChatId : null)); + const activeThreadId = urlChatId > 0 ? urlChatId : threadId; + const handledLoadErrorThreadRef = useRef(null); const [currentThread, setCurrentThread] = useState(null); const [messages, setMessages] = useState([]); const [isRunning, setIsRunning] = useState(false); @@ -375,12 +443,14 @@ export default function NewChatPage() { const mentionedDocuments = useAtomValue(mentionedDocumentsAtom); const messageDocumentsMap = useAtomValue(messageDocumentsMapAtom); const setMentionedDocuments = useSetAtom(mentionedDocumentsAtom); - const setCurrentThreadState = useSetAtom(currentThreadAtom); + const currentThreadState = useAtomValue(currentThreadAtom); + const setCurrentThreadMetadata = useSetAtom(setCurrentThreadMetadataAtom); const setPremiumAlertForThread = useSetAtom(setPremiumAlertForThreadAtom); const setTargetCommentId = useSetAtom(setTargetCommentIdAtom); const clearTargetCommentId = useSetAtom(clearTargetCommentIdAtom); const closeReportPanel = useSetAtom(closeReportPanelAtom); const closeEditorPanel = useSetAtom(closeEditorPanelAtom); + const syncChatTab = useSetAtom(syncChatTabAtom); const updateChatTabTitle = useSetAtom(updateChatTabTitleAtom); const removeChatTab = useSetAtom(removeChatTabAtom); const setAgentCreatedDocuments = useSetAtom(agentCreatedDocumentsAtom); @@ -402,9 +472,11 @@ export default function NewChatPage() { const { data: currentUser } = useAtomValue(currentUserAtom); const { data: agentFlags } = useAtomValue(agentFlagsAtom); const localFilesystemEnabled = agentFlags?.enable_desktop_local_filesystem === true; + const threadDetailQuery = useThreadDetail(activeThreadId); + const threadMessagesQuery = useThreadMessages(activeThreadId); // Live collaboration: sync session state and messages via Zero - useChatSessionStateSync(threadId); + useChatSessionStateSync(activeThreadId); const { data: membersData } = useAtomValue(membersAtom); const handleSyncedMessagesUpdate = useCallback( @@ -465,7 +537,7 @@ export default function NewChatPage() { [isRunning, membersData] ); - useMessagesSync(threadId, handleSyncedMessagesUpdate); + useMessagesSync(activeThreadId, handleSyncedMessagesUpdate); // Extract search_space_id from URL params const searchSpaceId = useMemo(() => { @@ -479,19 +551,7 @@ export default function NewChatPage() { // per-turn Revert button all read). Hydrates from // ``GET /threads/{id}/actions`` and is updated incrementally by the // SSE handlers + revert-batch results below — no atom side-channel. - const { items: agentActionItems } = useAgentActionsQuery(threadId); - - // Extract chat_id from URL params - const urlChatId = useMemo(() => { - const id = params.chat_id; - let parsed = 0; - if (Array.isArray(id) && id.length > 0) { - parsed = Number.parseInt(id[0], 10); - } else if (typeof id === "string") { - parsed = Number.parseInt(id, 10); - } - return Number.isNaN(parsed) ? 0 : parsed; - }, [params.chat_id]); + const { items: agentActionItems } = useAgentActionsQuery(activeThreadId); const handleChatFailure = useCallback( async ({ @@ -630,14 +690,19 @@ export default function NewChatPage() { }); }, []); - // Initialize thread and load messages - // For new chats (no urlChatId), we use lazy creation - thread is created on first message - const initializeThread = useCallback(async () => { - setIsInitializing(true); + const hydratedMessagesRef = useRef<{ + threadId: number | null; + data: typeof threadMessagesQuery.data; + }>({ threadId: null, data: undefined }); - // Reset all state when switching between chats/search spaces to prevent stale data + // Reset thread-local runtime state on route/search-space changes. Data fetching + // is handled by React Query below so the chat shell can render immediately. + useEffect(() => { + const nextThreadId = urlChatId > 0 ? urlChatId : null; + handledLoadErrorThreadRef.current = null; + hydratedMessagesRef.current = { threadId: null, data: undefined }; + setThreadId(nextThreadId); setMessages([]); - setThreadId(null); setCurrentThread(null); setMentionedDocuments([]); tokenUsageStore.clear(); @@ -647,82 +712,105 @@ export default function NewChatPage() { closeEditorPanel(); // Note: agent-action data is keyed by threadId in react-query so // switching threads naturally swaps caches; no explicit reset. - - try { - if (urlChatId > 0) { - // Thread exists - load thread data and messages - setThreadId(urlChatId); - - // Load thread data (for visibility info) and messages in parallel - const [threadData, messagesResponse] = await Promise.all([ - getThreadFull(urlChatId), - getThreadMessages(urlChatId), - ]); - - setCurrentThread(threadData); - - if (messagesResponse.messages && messagesResponse.messages.length > 0) { - const loadedMessages = reconcileInterruptedAssistantMessages( - messagesResponse.messages - ).map(convertToThreadMessage); - setMessages(loadedMessages); - - for (const msg of messagesResponse.messages) { - if (msg.token_usage) { - tokenUsageStore.set(`msg-${msg.id}`, msg.token_usage as TokenUsageData); - } - } - - const restoredDocsMap: Record = {}; - for (const msg of messagesResponse.messages) { - if (msg.role === "user") { - const docs = extractMentionedDocuments(msg.content); - if (docs.length > 0) { - restoredDocsMap[`msg-${msg.id}`] = docs; - } - } - } - if (Object.keys(restoredDocsMap).length > 0) { - setMessageDocumentsMap(restoredDocsMap); - } - } - } - // For new chats (urlChatId === 0), don't create thread yet - // Thread will be created lazily when user sends first message - // This improves UX (instant load) and avoids orphan threads - } catch (error) { - console.error("[NewChatPage] Failed to initialize thread:", error); - if (urlChatId > 0 && error instanceof NotFoundError) { - removeChatTab(urlChatId); - if (typeof window !== "undefined") { - window.history.replaceState(null, "", `/dashboard/${searchSpaceId}/new-chat`); - } - toast.error("This chat was deleted."); - return; - } - // Keep threadId as null - don't use Date.now() as it creates an invalid ID - // that will cause 404 errors on subsequent API calls - setThreadId(null); - setCurrentThread(null); - toast.error("Failed to load chat. Please try again."); - } finally { - setIsInitializing(false); - } }, [ urlChatId, - setMessageDocumentsMap, setMentionedDocuments, + setMessageDocumentsMap, + tokenUsageStore, closeReportPanel, closeEditorPanel, - removeChatTab, - searchSpaceId, + ]); + + useEffect(() => { + if (!activeThreadId) { + setCurrentThread(null); + return; + } + if (threadDetailQuery.data?.id === activeThreadId) { + const thread = threadDetailQuery.data; + setCurrentThread(thread); + syncChatTab({ + chatId: thread.id, + title: thread.title, + chatUrl: `/dashboard/${thread.search_space_id ?? searchSpaceId}/new-chat/${thread.id}`, + searchSpaceId: thread.search_space_id ?? searchSpaceId, + visibility: thread.visibility, + hasComments: thread.has_comments ?? false, + }); + } + }, [activeThreadId, searchSpaceId, syncChatTab, threadDetailQuery.data]); + + useEffect(() => { + const messagesResponse = threadMessagesQuery.data; + if (!activeThreadId || !messagesResponse) return; + + if ( + hydratedMessagesRef.current.threadId === activeThreadId && + hydratedMessagesRef.current.data === messagesResponse + ) { + return; + } + + if (isRunning) { + return; + } + + const loadedMessages = reconcileInterruptedAssistantMessages(messagesResponse.messages).map( + convertToThreadMessage + ); + setMessages(loadedMessages); + + tokenUsageStore.clear(); + const restoredDocsMap: Record = {}; + for (const msg of messagesResponse.messages) { + if (msg.token_usage) { + tokenUsageStore.set(`msg-${msg.id}`, msg.token_usage as TokenUsageData); + } + if (msg.role === "user") { + const docs = extractMentionedDocuments(msg.content); + if (docs.length > 0) { + restoredDocsMap[`msg-${msg.id}`] = docs; + } + } + } + setMessageDocumentsMap(restoredDocsMap); + hydratedMessagesRef.current = { threadId: activeThreadId, data: messagesResponse }; + }, [ + activeThreadId, + isRunning, + setMessageDocumentsMap, + threadMessagesQuery.data, tokenUsageStore, ]); - // Initialize on mount, and re-init when switching search spaces (even if urlChatId is the same) useEffect(() => { - initializeThread(); - }, [initializeThread]); + const loadError = threadDetailQuery.error ?? threadMessagesQuery.error; + if (!activeThreadId || !loadError) return; + if (handledLoadErrorThreadRef.current === activeThreadId) return; + + handledLoadErrorThreadRef.current = activeThreadId; + console.error("[NewChatPage] Failed to load thread:", loadError); + + if (loadError instanceof NotFoundError) { + removeChatTab(activeThreadId); + if (typeof window !== "undefined") { + window.history.replaceState(null, "", `/dashboard/${searchSpaceId}/new-chat`); + } + setThreadId(null); + setCurrentThread(null); + setMessages([]); + toast.error("This chat was deleted."); + return; + } + + toast.error("Failed to load chat. Please try again."); + }, [ + activeThreadId, + removeChatTab, + searchSpaceId, + threadDetailQuery.error, + threadMessagesQuery.error, + ]); // Prefetch document titles for @ mention picker // Runs when user lands on page so data is ready when they type @ @@ -750,7 +838,7 @@ export default function NewChatPage() { const readAndApplyCommentId = () => { const params = new URLSearchParams(window.location.search); const raw = params.get("commentId"); - if (raw && !isInitializing) { + if (raw && activeThreadId) { const commentId = Number.parseInt(raw, 10); if (!Number.isNaN(commentId)) { setTargetCommentId(commentId); @@ -768,17 +856,42 @@ export default function NewChatPage() { window.removeEventListener("popstate", readAndApplyCommentId); clearTargetCommentId(); }; - }, [isInitializing, setTargetCommentId, clearTargetCommentId]); + }, [activeThreadId, setTargetCommentId, clearTargetCommentId]); // Sync current thread state to atom useEffect(() => { - setCurrentThreadState((prev) => ({ - ...prev, - id: currentThread?.id ?? null, - visibility: currentThread?.visibility ?? null, - hasComments: currentThread?.has_comments ?? false, - })); - }, [currentThread, setCurrentThreadState]); + if (!currentThread) { + if (activeThreadId) { + return; + } + setCurrentThreadMetadata({ + id: null, + searchSpaceId: null, + visibility: null, + hasComments: false, + }); + return; + } + + const visibility = + currentThreadState.id === currentThread.id && currentThreadState.visibility !== null + ? currentThreadState.visibility + : currentThread.visibility; + + setCurrentThreadMetadata({ + id: currentThread.id, + searchSpaceId: currentThread.search_space_id ?? searchSpaceId, + visibility, + hasComments: currentThread.has_comments ?? false, + }); + }, [ + activeThreadId, + currentThread, + currentThreadState.id, + currentThreadState.visibility, + searchSpaceId, + setCurrentThreadMetadata, + ]); // Cleanup on unmount - abort any in-flight requests useEffect(() => { @@ -862,6 +975,8 @@ export default function NewChatPage() { setThreadId(currentThreadId); // Set currentThread so share button in header appears immediately setCurrentThread(newThread); + queryClient.setQueryData(cacheKeys.threads.detail(newThread.id), newThread); + queryClient.setQueryData(cacheKeys.threads.messages(newThread.id), { messages: [] }); // Track chat creation trackChatCreated(searchSpaceId, currentThreadId); @@ -1369,6 +1484,14 @@ export default function NewChatPage() { } finally { setIsRunning(false); abortControllerRef.current = null; + if (currentThreadId) { + void queryClient.invalidateQueries({ + queryKey: cacheKeys.threads.messages(currentThreadId), + }); + void queryClient.invalidateQueries({ + queryKey: cacheKeys.threads.detail(currentThreadId), + }); + } } }, [ @@ -1717,6 +1840,12 @@ export default function NewChatPage() { } finally { setIsRunning(false); abortControllerRef.current = null; + void queryClient.invalidateQueries({ + queryKey: cacheKeys.threads.messages(resumeThreadId), + }); + void queryClient.invalidateQueries({ + queryKey: cacheKeys.threads.detail(resumeThreadId), + }); } }, [ @@ -2210,6 +2339,12 @@ export default function NewChatPage() { } finally { setIsRunning(false); abortControllerRef.current = null; + void queryClient.invalidateQueries({ + queryKey: cacheKeys.threads.messages(threadId), + }); + void queryClient.invalidateQueries({ + queryKey: cacheKeys.threads.detail(threadId), + }); } }, [ @@ -2396,22 +2531,25 @@ export default function NewChatPage() { onCancel: cancelRun, }); - // Show loading state only when loading an existing thread - if (isInitializing) { - return ; - } + const threadLoadError = activeThreadId + ? (threadDetailQuery.error ?? threadMessagesQuery.error) + : null; + const shouldShowThreadLoadError = + !!threadLoadError && !!activeThreadId && !currentThread && messages.length === 0; + const isThreadMessagesLoading = + !!activeThreadId && + threadMessagesQuery.isPending && + messages.length === 0 && + !threadMessagesQuery.error; - // Show error state only if we tried to load an existing thread but failed - // For new chats (urlChatId === 0), threadId being null is expected (lazy creation) - if (!threadId && urlChatId > 0) { + if (shouldShowThreadLoadError) { return (
    Failed to load chat
    - -
    - - members +
    +
    +

    Members

    +
    +
    + +
    @@ -319,51 +320,54 @@ export function TeamContent({ searchSpaceId }: TeamContentProps) { return (
    -
    - {canInvite && - (rolesLoading ? ( - - ) : ( - - ))} - {canInvite && - (invitesLoading ? ( - - ) : ( - activeInvites.length > 0 && ( +
    +
    +

    Members

    +

    + {members.length} {members.length === 1 ? "member" : "members"} +

    +
    + {canInvite && ( +
    + {rolesLoading ? ( + + ) : ( + + )} + {invitesLoading ? ( + + ) : ( - ) - ))} -

    - {members.length} {members.length === 1 ? "member" : "members"} -

    + )} +
    + )}
    @@ -859,7 +863,11 @@ function AllInvitesDialog({ return ( - - - -
    - )} - {}} - onConnectNonOAuth={hasDocumentSummaryLLM ? handleConnectNonOAuth : () => {}} - onCreateWebcrawler={ - hasDocumentSummaryLLM ? handleCreateWebcrawler : () => {} - } - onCreateYouTubeCrawler={ - hasDocumentSummaryLLM ? handleCreateYouTubeCrawler : () => {} - } + onConnectOAuth={handleConnectOAuth} + onConnectNonOAuth={handleConnectNonOAuth} + onCreateWebcrawler={handleCreateWebcrawler} + onCreateYouTubeCrawler={handleCreateYouTubeCrawler} onManage={handleStartEdit} onViewAccountsList={handleViewAccountsList} /> diff --git a/surfsense_web/components/assistant-ui/connector-popup/components/summary-config.tsx b/surfsense_web/components/assistant-ui/connector-popup/components/summary-config.tsx deleted file mode 100644 index b9ff69f5f..000000000 --- a/surfsense_web/components/assistant-ui/connector-popup/components/summary-config.tsx +++ /dev/null @@ -1,25 +0,0 @@ -"use client"; - -import type { FC } from "react"; -import { Switch } from "@/components/ui/switch"; - -interface SummaryConfigProps { - enabled: boolean; - onEnabledChange: (enabled: boolean) => void; -} - -export const SummaryConfig: FC = ({ enabled, onEnabledChange }) => { - return ( -
    -
    -
    -

    Enable AI Summary

    -

    - Improves search quality but adds latency during indexing -

    -
    - -
    -
    - ); -}; diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-edit-view.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-edit-view.tsx index 2b86daf65..d61460d48 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-edit-view.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-edit-view.tsx @@ -17,7 +17,6 @@ import { BACKEND_URL } from "@/lib/env-config"; import { cn } from "@/lib/utils"; import { DateRangeSelector } from "../../components/date-range-selector"; import { PeriodicSyncConfig } from "../../components/periodic-sync-config"; -import { SummaryConfig } from "../../components/summary-config"; import { VisionLLMConfig } from "../../components/vision-llm-config"; import { LIVE_CONNECTOR_TYPES } from "../../constants/connector-constants"; import { getConnectorDisplayName } from "../../tabs/all-connectors-tab"; @@ -38,7 +37,6 @@ interface ConnectorEditViewProps { endDate: Date | undefined; periodicEnabled: boolean; frequencyMinutes: string; - enableSummary: boolean; enableVisionLlm: boolean; isSaving: boolean; isDisconnecting: boolean; @@ -48,7 +46,6 @@ interface ConnectorEditViewProps { onEndDateChange: (date: Date | undefined) => void; onPeriodicEnabledChange: (enabled: boolean) => void; onFrequencyChange: (frequency: string) => void; - onEnableSummaryChange: (enabled: boolean) => void; onEnableVisionLlmChange: (enabled: boolean) => void; onSave: () => void; onDisconnect: () => void; @@ -64,7 +61,6 @@ export const ConnectorEditView: FC = ({ endDate, periodicEnabled, frequencyMinutes, - enableSummary, enableVisionLlm, isSaving, isDisconnecting, @@ -74,7 +70,6 @@ export const ConnectorEditView: FC = ({ onEndDateChange, onPeriodicEnabledChange, onFrequencyChange, - onEnableSummaryChange, onEnableVisionLlmChange, onSave, onDisconnect, @@ -87,9 +82,13 @@ export const ConnectorEditView: FC = ({ const isAuthExpired = connector.config?.auth_expired === true; const reauthEndpoint = getReauthEndpoint(connector); const [reauthing, setReauthing] = useState(false); + const isMCPBacked = Boolean(connector.config?.server_config); + const isLive = isMCPBacked || LIVE_CONNECTOR_TYPES.has(connector.connector_type); const supportsVisionLlm = VISION_LLM_CONNECTOR_TYPES.has(connector.connector_type); - const showsAiToggles = - connector.is_indexable || connector.connector_type === EnumConnectorName.OBSIDIAN_CONNECTOR; + const showsVisionToggle = + !isLive && + supportsVisionLlm && + (connector.is_indexable || connector.connector_type === EnumConnectorName.OBSIDIAN_CONNECTOR); const handleReauth = useCallback(async () => { const spaceId = searchSpaceId ?? searchSpaceIdAtom; @@ -121,9 +120,6 @@ export const ConnectorEditView: FC = ({ } }, [searchSpaceId, searchSpaceIdAtom, reauthEndpoint, connector.id]); - const isMCPBacked = Boolean(connector.config?.server_config); - const isLive = isMCPBacked || LIVE_CONNECTOR_TYPES.has(connector.connector_type); - // Get connector-specific config component (MCP-backed connectors use a generic view) const ConnectorConfigComponent = useMemo(() => { if (isMCPBacked) return MCPServiceConfig; @@ -280,77 +276,64 @@ export const ConnectorEditView: FC = ({ /> )} - {/* Summary + vision toggles (Obsidian is plugin-push, non-indexable by design) */} - {showsAiToggles && !isLive && ( - <> - {/* AI Summary toggle */} - - - {/* Vision LLM toggle for file/attachment connectors */} - {supportsVisionLlm && ( - - )} - - {/* Date-range and periodic sync stay indexable-only */} - {connector.is_indexable && - connector.connector_type !== "GOOGLE_DRIVE_CONNECTOR" && - connector.connector_type !== "COMPOSIO_GOOGLE_DRIVE_CONNECTOR" && - connector.connector_type !== "DROPBOX_CONNECTOR" && - connector.connector_type !== "ONEDRIVE_CONNECTOR" && - connector.connector_type !== "WEBCRAWLER_CONNECTOR" && - connector.connector_type !== "GITHUB_CONNECTOR" && ( - - )} - - {connector.is_indexable && - (() => { - const isGoogleDrive = connector.connector_type === "GOOGLE_DRIVE_CONNECTOR"; - const isComposioGoogleDrive = - connector.connector_type === "COMPOSIO_GOOGLE_DRIVE_CONNECTOR"; - const requiresFolderSelection = isGoogleDrive || isComposioGoogleDrive; - const selectedFolders = - (connector.config?.selected_folders as - | Array<{ id: string; name: string }> - | undefined) || []; - const selectedFiles = - (connector.config?.selected_files as - | Array<{ id: string; name: string }> - | undefined) || []; - const hasItemsSelected = selectedFolders.length > 0 || selectedFiles.length > 0; - const isDisabled = requiresFolderSelection && !hasItemsSelected; - - return ( - - ); - })()} - + {/* Vision toggle (Obsidian is plugin-push, non-indexable by design) */} + {showsVisionToggle && ( + )} + {/* Date-range and periodic sync stay indexable-only */} + {connector.is_indexable && + connector.connector_type !== "GOOGLE_DRIVE_CONNECTOR" && + connector.connector_type !== "COMPOSIO_GOOGLE_DRIVE_CONNECTOR" && + connector.connector_type !== "DROPBOX_CONNECTOR" && + connector.connector_type !== "ONEDRIVE_CONNECTOR" && + connector.connector_type !== "WEBCRAWLER_CONNECTOR" && + connector.connector_type !== "GITHUB_CONNECTOR" && ( + + )} + + {connector.is_indexable && + (() => { + const isGoogleDrive = connector.connector_type === "GOOGLE_DRIVE_CONNECTOR"; + const isComposioGoogleDrive = + connector.connector_type === "COMPOSIO_GOOGLE_DRIVE_CONNECTOR"; + const requiresFolderSelection = isGoogleDrive || isComposioGoogleDrive; + const selectedFolders = + (connector.config?.selected_folders as Array<{ id: string; name: string }> | undefined) || + []; + const selectedFiles = + (connector.config?.selected_files as Array<{ id: string; name: string }> | undefined) || + []; + const hasItemsSelected = selectedFolders.length > 0 || selectedFiles.length > 0; + const isDisabled = requiresFolderSelection && !hasItemsSelected; + + return ( + + ); + })()} + {/* Info box - hidden for live connectors */} {connector.is_indexable && !isLive && ( diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/indexing-configuration-view.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/indexing-configuration-view.tsx index 74b4fad9f..218b3d329 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/indexing-configuration-view.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/indexing-configuration-view.tsx @@ -11,7 +11,6 @@ import { getConnectorTypeDisplay } from "@/lib/connectors/utils"; import { cn } from "@/lib/utils"; import { DateRangeSelector } from "../../components/date-range-selector"; import { PeriodicSyncConfig } from "../../components/periodic-sync-config"; -import { SummaryConfig } from "../../components/summary-config"; import { VisionLLMConfig } from "../../components/vision-llm-config"; import { type IndexingConfigState, @@ -35,7 +34,6 @@ interface IndexingConfigurationViewProps { endDate: Date | undefined; periodicEnabled: boolean; frequencyMinutes: string; - enableSummary: boolean; enableVisionLlm: boolean; isStartingIndexing: boolean; isFromOAuth?: boolean; @@ -43,7 +41,6 @@ interface IndexingConfigurationViewProps { onEndDateChange: (date: Date | undefined) => void; onPeriodicEnabledChange: (enabled: boolean) => void; onFrequencyChange: (frequency: string) => void; - onEnableSummaryChange: (enabled: boolean) => void; onEnableVisionLlmChange: (enabled: boolean) => void; onConfigChange?: (config: Record) => void; onStartIndexing: () => void; @@ -57,7 +54,6 @@ export const IndexingConfigurationView: FC = ({ endDate, periodicEnabled, frequencyMinutes, - enableSummary, enableVisionLlm, isStartingIndexing, isFromOAuth = false, @@ -65,7 +61,6 @@ export const IndexingConfigurationView: FC = ({ onEndDateChange, onPeriodicEnabledChange, onFrequencyChange, - onEnableSummaryChange, onEnableVisionLlmChange, onConfigChange, onStartIndexing, @@ -78,9 +73,11 @@ export const IndexingConfigurationView: FC = ({ () => (connector ? getConnectorConfigComponent(connector.connector_type) : null), [connector] ); - const showsAiToggles = - (connector?.is_indexable ?? false) || - connector?.connector_type === EnumConnectorName.OBSIDIAN_CONNECTOR; + const showsVisionToggle = + !isLive && + ((connector?.is_indexable ?? false) || + connector?.connector_type === EnumConnectorName.OBSIDIAN_CONNECTOR) && + VISION_LLM_CONNECTOR_TYPES.has(config.connectorType); const [isScrolled, setIsScrolled] = useState(false); const [hasMoreContent, setHasMoreContent] = useState(false); const scrollContainerRef = useRef(null); @@ -178,57 +175,46 @@ export const IndexingConfigurationView: FC = ({ )} - {/* Summary + vision toggles (Obsidian is plugin-push, non-indexable by design) */} - {showsAiToggles && !isLive && ( - <> - {/* AI Summary toggle */} - - - {/* Vision LLM toggle for file/attachment connectors */} - {VISION_LLM_CONNECTOR_TYPES.has(config.connectorType) && ( - - )} - - {/* Date-range and periodic sync stay indexable-only */} - {connector?.is_indexable && - config.connectorType !== "GOOGLE_DRIVE_CONNECTOR" && - config.connectorType !== "COMPOSIO_GOOGLE_DRIVE_CONNECTOR" && - config.connectorType !== "DROPBOX_CONNECTOR" && - config.connectorType !== "ONEDRIVE_CONNECTOR" && - config.connectorType !== "WEBCRAWLER_CONNECTOR" && - config.connectorType !== "GITHUB_CONNECTOR" && ( - - )} - - {connector?.is_indexable && - config.connectorType !== "GOOGLE_DRIVE_CONNECTOR" && - config.connectorType !== "COMPOSIO_GOOGLE_DRIVE_CONNECTOR" && - config.connectorType !== "DROPBOX_CONNECTOR" && - config.connectorType !== "ONEDRIVE_CONNECTOR" && ( - - )} - + {/* Vision toggle (Obsidian is plugin-push, non-indexable by design) */} + {showsVisionToggle && ( + )} + {/* Date-range and periodic sync stay indexable-only */} + {connector?.is_indexable && + config.connectorType !== "GOOGLE_DRIVE_CONNECTOR" && + config.connectorType !== "COMPOSIO_GOOGLE_DRIVE_CONNECTOR" && + config.connectorType !== "DROPBOX_CONNECTOR" && + config.connectorType !== "ONEDRIVE_CONNECTOR" && + config.connectorType !== "WEBCRAWLER_CONNECTOR" && + config.connectorType !== "GITHUB_CONNECTOR" && ( + + )} + + {connector?.is_indexable && + config.connectorType !== "GOOGLE_DRIVE_CONNECTOR" && + config.connectorType !== "COMPOSIO_GOOGLE_DRIVE_CONNECTOR" && + config.connectorType !== "DROPBOX_CONNECTOR" && + config.connectorType !== "ONEDRIVE_CONNECTOR" && ( + + )} + {/* Info box - hidden for live connectors */} {connector?.is_indexable && !isLive && ( diff --git a/surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts b/surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts index 25ab82e2e..aa9440a12 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts +++ b/surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts @@ -82,7 +82,6 @@ export const useConnectorDialog = () => { const [isStartingIndexing, setIsStartingIndexing] = useState(false); const [periodicEnabled, setPeriodicEnabled] = useState(false); const [frequencyMinutes, setFrequencyMinutes] = useState("1440"); - const [enableSummary, setEnableSummary] = useState(false); const [enableVisionLlm, setEnableVisionLlm] = useState(false); // Edit mode state @@ -418,7 +417,6 @@ export const useConnectorDialog = () => { periodic_indexing_enabled: false, indexing_frequency_minutes: null, next_scheduled_at: null, - enable_summary: false, enable_vision_llm: false, }, queryParams: { @@ -520,7 +518,6 @@ export const useConnectorDialog = () => { connector_type: connectorData.connector_type as EnumConnectorName, is_active: true, next_scheduled_at: connectorData.next_scheduled_at as string | null, - enable_summary: false, enable_vision_llm: false, }, queryParams: { @@ -657,8 +654,7 @@ export const useConnectorDialog = () => { setConnectorConfig(connector.config || {}); setPeriodicEnabled(false); setFrequencyMinutes("1440"); - setEnableSummary(connector.enable_summary ?? false); - setEnableVisionLlm(connector.enable_vision_llm ?? false); + setEnableVisionLlm(connector.enable_vision_llm ?? false); setStartDate(undefined); setEndDate(undefined); @@ -806,14 +802,13 @@ export const useConnectorDialog = () => { const startDateStr = startDate ? format(startDate, "yyyy-MM-dd") : undefined; const endDateStr = endDate ? format(endDate, "yyyy-MM-dd") : undefined; - // Update connector with summary, periodic sync settings, and config changes - if (enableSummary || enableVisionLlm || periodicEnabled || indexingConnectorConfig) { + // Update connector with vision, periodic sync settings, and config changes + if (enableVisionLlm || periodicEnabled || indexingConnectorConfig) { const frequency = periodicEnabled ? parseInt(frequencyMinutes, 10) : undefined; await updateConnector({ id: indexingConfig.connectorId, data: { - enable_summary: enableSummary, - enable_vision_llm: enableVisionLlm, + enable_vision_llm: enableVisionLlm, ...(periodicEnabled && { periodic_indexing_enabled: true, indexing_frequency_minutes: frequency, @@ -940,7 +935,6 @@ export const useConnectorDialog = () => { updateConnector, periodicEnabled, frequencyMinutes, - enableSummary, enableVisionLlm, indexingConnectorConfig, setIsOpen, @@ -1005,7 +999,6 @@ export const useConnectorDialog = () => { setConnectorName(connector.name); setPeriodicEnabled(!connector.is_indexable ? false : connector.periodic_indexing_enabled); setFrequencyMinutes(connector.indexing_frequency_minutes?.toString() || "1440"); - setEnableSummary(connector.enable_summary ?? false); setEnableVisionLlm(connector.enable_vision_llm ?? false); setStartDate(undefined); setEndDate(undefined); @@ -1084,7 +1077,6 @@ export const useConnectorDialog = () => { id: editingConnector.id, data: { name: connectorName || editingConnector.name, - enable_summary: enableSummary, enable_vision_llm: enableVisionLlm, periodic_indexing_enabled: !editingConnector.is_indexable ? false : periodicEnabled, indexing_frequency_minutes: !editingConnector.is_indexable ? null : frequency, @@ -1219,7 +1211,6 @@ export const useConnectorDialog = () => { updateConnector, periodicEnabled, frequencyMinutes, - enableSummary, enableVisionLlm, getFrequencyLabel, connectorConfig, @@ -1380,7 +1371,6 @@ export const useConnectorDialog = () => { setEndDate(undefined); setPeriodicEnabled(false); setFrequencyMinutes("1440"); - setEnableSummary(false); setEnableVisionLlm(false); } } @@ -1417,7 +1407,6 @@ export const useConnectorDialog = () => { isDisconnecting, periodicEnabled, frequencyMinutes, - enableSummary, enableVisionLlm, searchSpaceId, allConnectors, @@ -1432,7 +1421,6 @@ export const useConnectorDialog = () => { setEndDate, setPeriodicEnabled, setFrequencyMinutes, - setEnableSummary, setEnableVisionLlm, setConnectorName, diff --git a/surfsense_web/components/assistant-ui/document-upload-popup.tsx b/surfsense_web/components/assistant-ui/document-upload-popup.tsx index 477d7ee77..504f1d8d4 100644 --- a/surfsense_web/components/assistant-ui/document-upload-popup.tsx +++ b/surfsense_web/components/assistant-ui/document-upload-popup.tsx @@ -1,8 +1,6 @@ "use client"; import { useAtomValue } from "jotai"; -import { AlertTriangle } from "lucide-react"; -import { useRouter } from "next/navigation"; import { createContext, type FC, @@ -12,14 +10,8 @@ import { useRef, useState, } from "react"; -import { - globalNewLLMConfigsAtom, - llmPreferencesAtom, -} from "@/atoms/new-llm-config/new-llm-config-query.atoms"; import { activeSearchSpaceIdAtom } from "@/atoms/search-spaces/search-space-query.atoms"; import { DocumentUploadTab } from "@/components/sources/DocumentUploadTab"; -import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; -import { Button } from "@/components/ui/button"; import { Dialog, DialogContent, @@ -98,12 +90,7 @@ const DocumentUploadPopupContent: FC<{ isOpen: boolean; onOpenChange: (open: boolean) => void; }> = ({ isOpen, onOpenChange }) => { - const router = useRouter(); const searchSpaceId = useAtomValue(activeSearchSpaceIdAtom); - const { data: preferences = {}, isFetching: preferencesLoading } = - useAtomValue(llmPreferencesAtom); - const { data: globalConfigs = [], isFetching: globalConfigsLoading } = - useAtomValue(globalNewLLMConfigsAtom); if (!searchSpaceId) return null; @@ -111,22 +98,6 @@ const DocumentUploadPopupContent: FC<{ onOpenChange(false); }; - // Check if document summary LLM is properly configured - // - If ID is 0 (Auto mode), we need global configs to be available - // - If ID is positive (user config) or negative (specific global config), it's configured - // - If ID is null/undefined, it's not configured - const docSummaryLlmId = preferences.document_summary_llm_id; - const isAutoMode = docSummaryLlmId === 0; - const hasGlobalConfigs = globalConfigs.length > 0; - - const hasDocumentSummaryLLM = - docSummaryLlmId !== null && - docSummaryLlmId !== undefined && - // If it's Auto mode, we need global configs to actually be available - (!isAutoMode || hasGlobalConfigs); - - const isLoading = preferencesLoading || globalConfigsLoading; - return (
    - {!isLoading && !hasDocumentSummaryLLM ? ( -
    - - - LLM Configuration Required - -

    - {isAutoMode && !hasGlobalConfigs - ? "Auto mode requires a global LLM configuration. Please add one in Settings" - : "A Document Summary LLM is required to process uploads, configure one in Settings"} -

    - -
    -
    -
    - ) : ( - - )} +
    diff --git a/surfsense_web/components/assistant-ui/inline-citation.tsx b/surfsense_web/components/assistant-ui/inline-citation.tsx index 6a8f2e035..59a10739c 100644 --- a/surfsense_web/components/assistant-ui/inline-citation.tsx +++ b/surfsense_web/components/assistant-ui/inline-citation.tsx @@ -3,7 +3,7 @@ import { useSetAtom } from "jotai"; import { FileText } from "lucide-react"; import type { FC } from "react"; -import { useState } from "react"; +import { useId, useState } from "react"; import { openCitationPanelAtom } from "@/atoms/citation/citation-panel.atom"; import { useCitationMetadata } from "@/components/assistant-ui/citation-metadata-context"; import { CitationPanelContent } from "@/components/citation-panel/citation-panel"; @@ -120,12 +120,14 @@ interface UrlCitationProps { * page title and snippet (extracted deterministically from web_search tool results). */ export const UrlCitation: FC = ({ url }) => { + const reactId = useId(); + const citationInstanceId = `url-cite-${reactId.replace(/:/g, "")}`; const domain = tryGetHostname(url) ?? url; const meta = useCitationMetadata(url); return ( { const [actionQuery, setActionQuery] = useState(""); const [suggestionAnchorPoint, setSuggestionAnchorPoint] = useState(null); + const [isComposerInputEmpty, setIsComposerInputEmpty] = useState(true); const editorRef = useRef(null); const prevMentionedDocsRef = useRef>(new Map()); const documentPickerRef = useRef(null); @@ -536,6 +537,7 @@ const Composer: FC = () => { // short-circuit keeps pure-text keystrokes from churning the atom. const handleEditorChange = useCallback( (text: string, docs: MentionedDocument[]) => { + setIsComposerInputEmpty(text.trim().length === 0 && docs.length === 0); aui.composer().setText(text); setMentionedDocuments((prev) => { if (prev.length === docs.length) { @@ -651,6 +653,7 @@ const Composer: FC = () => { : action.prompt; editorRef.current?.setText(finalPrompt); aui.composer().setText(finalPrompt); + setIsComposerInputEmpty(false); setShowPromptPicker(false); setActionQuery(""); setSuggestionAnchorPoint(null); @@ -662,6 +665,7 @@ const Composer: FC = () => { (prompt: string) => { editorRef.current?.setText(prompt); aui.composer().setText(prompt); + setIsComposerInputEmpty(false); editorRef.current?.focus(); }, [aui] @@ -676,6 +680,7 @@ const Composer: FC = () => { : `${action.prompt}\n\n${clipboardInitialText}`; editorRef.current?.setText(finalPrompt); aui.composer().setText(finalPrompt); + setIsComposerInputEmpty(false); setShowPromptPicker(false); setActionQuery(""); setSuggestionAnchorPoint(null); @@ -755,6 +760,7 @@ const Composer: FC = () => { aui.composer().send(); editorRef.current?.clear(); + setIsComposerInputEmpty(true); setMentionedDocuments([]); }, [ showDocumentPopover, @@ -893,7 +899,7 @@ const Composer: FC = () => {
    @@ -904,7 +910,7 @@ const Composer: FC = () => { onDismiss={() => setClipboardInitialText(undefined)} /> )} -
    +
    { onDocumentRemove={handleDocumentRemove} onSubmit={handleSubmit} onKeyDown={handleKeyDown} - className="min-h-[24px] **:data-slate-placeholder:font-normal" + className="min-h-[48px] sm:min-h-[24px] **:data-slate-placeholder:font-normal" />
    @@ -926,7 +932,9 @@ const Composer: FC = () => { isThreadEmpty={isThreadEmpty} onVisibleChange={setConnectToolsTrayVisible} /> - {isThreadEmpty && } + {isThreadEmpty && isComposerInputEmpty ? ( + + ) : null}
    ); diff --git a/surfsense_web/components/homepage/hero-section.tsx b/surfsense_web/components/homepage/hero-section.tsx index 0641d4e4e..09cf316d8 100644 --- a/surfsense_web/components/homepage/hero-section.tsx +++ b/surfsense_web/components/homepage/hero-section.tsx @@ -351,9 +351,24 @@ function GetStartedButton() { } function DownloadButton() { - const { os, primary, alternatives } = usePrimaryDownload(); + const { os, primary, alternatives, isMobileOS } = usePrimaryDownload(); const fallbackUrl = GITHUB_RELEASES_URL; + const mobileDisabledLabel = "Desktop app unavailable on mobile"; + + if (isMobileOS) { + return ( + + ); + } if (!primary) { return ( diff --git a/surfsense_web/components/layout/providers/FreeLayoutDataProvider.tsx b/surfsense_web/components/layout/providers/FreeLayoutDataProvider.tsx index 1c5d7af6e..9e2a99f2a 100644 --- a/surfsense_web/components/layout/providers/FreeLayoutDataProvider.tsx +++ b/surfsense_web/components/layout/providers/FreeLayoutDataProvider.tsx @@ -3,7 +3,7 @@ import { Inbox, LibraryBig } from "lucide-react"; import { useRouter } from "next/navigation"; import type { ReactNode } from "react"; -import { Fragment, useCallback, useEffect, useMemo, useState } from "react"; +import { useCallback, useEffect, useMemo, useState } from "react"; import { useAnonymousMode } from "@/contexts/anonymous-mode"; import { useLoginGate } from "@/contexts/login-gate"; import { useAnnouncements } from "@/hooks/use-announcements"; @@ -110,15 +110,13 @@ export function FreeLayoutDataProvider({ children }: FreeLayoutDataProviderProps navItems={navItems} onNavItemClick={handleNavItemClick} chats={[]} - sharedChats={[]} activeChatId={null} onNewChat={resetChat} onChatSelect={handleChatSelect} onChatRename={gatedAction("rename chats")} onChatDelete={gatedAction("delete chats")} onChatArchive={gatedAction("archive chats")} - onViewAllSharedChats={gatedAction("view shared chats")} - onViewAllPrivateChats={gatedAction("view chat history")} + onViewAllChats={gatedAction("view chat history")} user={{ email: "Guest", name: "Guest", @@ -137,7 +135,7 @@ export function FreeLayoutDataProvider({ children }: FreeLayoutDataProviderProps onOpenChange: setIsDocsSidebarOpen, }} > - {children} + {children} ); } diff --git a/surfsense_web/components/layout/providers/LayoutDataProvider.tsx b/surfsense_web/components/layout/providers/LayoutDataProvider.tsx index 34fd15e3b..46f6ec8ae 100644 --- a/surfsense_web/components/layout/providers/LayoutDataProvider.tsx +++ b/surfsense_web/components/layout/providers/LayoutDataProvider.tsx @@ -1,6 +1,6 @@ "use client"; -import { useQuery, useQueryClient } from "@tanstack/react-query"; +import { useQuery } from "@tanstack/react-query"; import { useAtom, useAtomValue, useSetAtom } from "jotai"; import { AlertTriangle, Inbox, LibraryBig, Workflow } from "lucide-react"; import { useParams, usePathname, useRouter } from "next/navigation"; @@ -41,13 +41,15 @@ import { } from "@/components/ui/dialog"; import { Input } from "@/components/ui/input"; import { Spinner } from "@/components/ui/spinner"; +import { useActivateChatThread } from "@/hooks/use-activate-chat-thread"; import { useAnnouncements } from "@/hooks/use-announcements"; import { useInbox } from "@/hooks/use-inbox"; import { useIsMobile } from "@/hooks/use-mobile"; +import { useArchiveThread, useDeleteThread, useRenameThread } from "@/hooks/use-thread-mutations"; import { notificationsApiService } from "@/lib/apis/notifications-api.service"; import { searchSpacesApiService } from "@/lib/apis/search-spaces-api.service"; import { getLoginPath, logout } from "@/lib/auth-utils"; -import { deleteThread, fetchThreads, updateThread } from "@/lib/chat/thread-persistence"; +import { fetchThreads } from "@/lib/chat/thread-persistence"; import { resetUser, trackLogout } from "@/lib/posthog/events"; import { cacheKeys } from "@/lib/query-client/cache-keys"; import type { ChatItem, NavItem, SearchSpace } from "../types/layout.types"; @@ -77,7 +79,6 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid const router = useRouter(); const params = useParams(); const pathname = usePathname(); - const queryClient = useQueryClient(); const { theme, setTheme } = useTheme(); const isMobile = useIsMobile(); @@ -96,6 +97,10 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid const resetCurrentThread = useSetAtom(resetCurrentThreadAtom); const syncChatTab = useSetAtom(syncChatTabAtom); const removeChatTab = useSetAtom(removeChatTabAtom); + const { activateChatThread, prefetchChatThread } = useActivateChatThread(); + const { mutateAsync: archiveThread } = useArchiveThread(searchSpaceId); + const { mutateAsync: deleteThread } = useDeleteThread(searchSpaceId); + const { mutateAsync: renameThread } = useRenameThread(searchSpaceId); // Key used to force-remount the page component (e.g. after deleting the active chat // when the router is out of sync due to replaceState) @@ -121,7 +126,7 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid }); // Unified slide-out panel state (only one can be open at a time) - type SlideoutPanel = "inbox" | "shared" | "private" | null; + type SlideoutPanel = "inbox" | "chats" | null; const [activeSlideoutPanel, setActiveSlideoutPanel] = useState(null); const isInboxSidebarOpen = activeSlideoutPanel === "inbox"; @@ -301,37 +306,21 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid title: chatId ? (thread?.title ?? undefined) : "New Chat", chatUrl, searchSpaceId: Number(searchSpaceId), + ...(thread?.visibility !== undefined ? { visibility: thread.visibility } : {}), }); }, [currentChatId, searchSpaceId, threadsData?.threads, syncChatTab]); - // Transform and split chats into private and shared based on visibility - const { myChats, sharedChats } = useMemo(() => { - if (!threadsData?.threads) return { myChats: [], sharedChats: [] }; + const chats = useMemo(() => { + if (!threadsData?.threads) return []; - const privateChats: ChatItem[] = []; - const sharedChatsList: ChatItem[] = []; - - for (const thread of threadsData.threads) { - const chatItem: ChatItem = { - id: thread.id, - name: thread.title || `Chat ${thread.id}`, - url: `/dashboard/${searchSpaceId}/new-chat/${thread.id}`, - visibility: thread.visibility, - isOwnThread: thread.is_own_thread, - archived: thread.archived, - }; - - // Split based on visibility, not ownership: - // - PRIVATE chats go to "Private Chats" section - // - SEARCH_SPACE chats go to "Shared Chats" section - if (thread.visibility === "SEARCH_SPACE") { - sharedChatsList.push(chatItem); - } else { - privateChats.push(chatItem); - } - } - - return { myChats: privateChats, sharedChats: sharedChatsList }; + return threadsData.threads.map((thread) => ({ + id: thread.id, + name: thread.title || `Chat ${thread.id}`, + url: `/dashboard/${searchSpaceId}/new-chat/${thread.id}`, + visibility: thread.visibility, + isOwnThread: thread.is_own_thread, + archived: thread.archived, + })); }, [threadsData, searchSpaceId]); // Navigation items @@ -478,12 +467,34 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid const handleTabSwitch = useCallback( (tab: Tab) => { if (tab.type === "chat") { - const url = tab.chatUrl || `/dashboard/${searchSpaceId}/new-chat`; - router.push(url); + activateChatThread({ + id: tab.chatId ?? null, + title: tab.title, + url: tab.chatUrl, + searchSpaceId: tab.searchSpaceId ?? searchSpaceId, + ...(tab.visibility !== undefined ? { visibility: tab.visibility } : {}), + ...(tab.hasComments !== undefined ? { hasComments: tab.hasComments } : {}), + }); } // Document tabs are handled in-place by LayoutShell — no navigation needed }, - [router, searchSpaceId] + [activateChatThread, searchSpaceId] + ); + + const handleTabPrefetch = useCallback( + (tab: Tab) => { + if (tab.type === "chat") { + prefetchChatThread(tab.chatId); + } + }, + [prefetchChatThread] + ); + + const handleChatPrefetch = useCallback( + (chat: ChatItem) => { + prefetchChatThread(chat.id); + }, + [prefetchChatThread] ); const handleNavItemClick = useCallback( @@ -535,9 +546,15 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid const handleChatSelect = useCallback( (chat: ChatItem) => { - router.push(chat.url); + activateChatThread({ + id: chat.id, + title: chat.name, + url: chat.url, + searchSpaceId, + ...(chat.visibility !== undefined ? { visibility: chat.visibility } : {}), + }); }, - [router] + [activateChatThread, searchSpaceId] ); const handleChatDelete = useCallback((chat: ChatItem) => { @@ -559,18 +576,14 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid : tSidebar("chat_unarchived") || "Chat restored"; try { - await updateThread(chat.id, { archived: newArchivedState }); + await archiveThread({ threadId: chat.id, archived: newArchivedState }); toast.success(successMessage); - // Invalidate queries to refresh UI (React Query will only refetch active queries) - queryClient.invalidateQueries({ queryKey: ["threads", searchSpaceId] }); - queryClient.invalidateQueries({ queryKey: ["all-threads", searchSpaceId] }); - queryClient.invalidateQueries({ queryKey: ["search-threads", searchSpaceId] }); } catch (error) { console.error("Error archiving thread:", error); toast.error(tSidebar("error_archiving_chat") || "Failed to archive chat"); } }, - [queryClient, searchSpaceId, tSidebar] + [archiveThread, tSidebar] ); const handleSettings = useCallback(() => { @@ -599,12 +612,8 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid } }, [router]); - const handleViewAllSharedChats = useCallback(() => { - setActiveSlideoutPanel((prev) => (prev === "shared" ? null : "shared")); - }, []); - - const handleViewAllPrivateChats = useCallback(() => { - setActiveSlideoutPanel((prev) => (prev === "private" ? null : "private")); + const handleViewAllChats = useCallback(() => { + setActiveSlideoutPanel((prev) => (prev === "chats" ? null : "chats")); }, []); // Delete handlers @@ -612,13 +621,21 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid if (!chatToDelete) return; setIsDeletingChat(true); try { - await deleteThread(chatToDelete.id); + await deleteThread({ threadId: chatToDelete.id }); const fallbackTab = removeChatTab(chatToDelete.id); - queryClient.invalidateQueries({ queryKey: ["threads", searchSpaceId] }); if (currentChatId === chatToDelete.id) { resetCurrentThread(); if (fallbackTab?.type === "chat" && fallbackTab.chatUrl) { - router.push(fallbackTab.chatUrl); + activateChatThread({ + id: fallbackTab.chatId ?? null, + title: fallbackTab.title, + url: fallbackTab.chatUrl, + searchSpaceId: fallbackTab.searchSpaceId ?? searchSpaceId, + ...(fallbackTab.visibility !== undefined ? { visibility: fallbackTab.visibility } : {}), + ...(fallbackTab.hasComments !== undefined + ? { hasComments: fallbackTab.hasComments } + : {}), + }); } else { const isOutOfSync = currentThreadState.id !== null && !params?.chat_id; if (isOutOfSync) { @@ -638,7 +655,7 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid } }, [ chatToDelete, - queryClient, + deleteThread, searchSpaceId, resetCurrentThread, currentChatId, @@ -646,6 +663,7 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid params?.chat_id, router, removeChatTab, + activateChatThread, ]); // Rename handler @@ -653,11 +671,12 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid if (!chatToRename || !newChatTitle.trim()) return; setIsRenamingChat(true); try { - await updateThread(chatToRename.id, { title: newChatTitle.trim() }); + await renameThread({ + threadId: chatToRename.id, + title: newChatTitle.trim(), + previousTitle: chatToRename.name, + }); toast.success(tSidebar("chat_renamed") || "Chat renamed"); - queryClient.invalidateQueries({ queryKey: ["threads", searchSpaceId] }); - queryClient.invalidateQueries({ queryKey: ["all-threads", searchSpaceId] }); - queryClient.invalidateQueries({ queryKey: ["search-threads", searchSpaceId] }); } catch (error) { console.error("Error renaming thread:", error); toast.error(tSidebar("error_renaming_chat") || "Failed to rename chat"); @@ -667,7 +686,7 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid setChatToRename(null); setNewChatTitle(""); } - }, [chatToRename, newChatTitle, queryClient, searchSpaceId, tSidebar]); + }, [chatToRename, newChatTitle, renameThread, tSidebar]); // Detect if we're on the chat page (needs overflow-hidden for chat's own scroll) const isChatPage = pathname?.includes("/new-chat") ?? false; @@ -695,16 +714,15 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid searchSpace={activeSearchSpace} navItems={navItems} onNavItemClick={handleNavItemClick} - chats={myChats} - sharedChats={sharedChats} + chats={chats} activeChatId={currentChatId} onNewChat={handleNewChat} onChatSelect={handleChatSelect} + onChatPrefetch={handleChatPrefetch} onChatRename={handleChatRename} onChatDelete={handleChatDelete} onChatArchive={handleChatArchive} - onViewAllSharedChats={handleViewAllSharedChats} - onViewAllPrivateChats={handleViewAllPrivateChats} + onViewAllChats={handleViewAllChats} user={{ email: user?.email || "", name: user?.display_name || user?.email?.split("@")[0], @@ -727,7 +745,7 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid } workspacePanelContentClassName={ isAutomationsPage - ? "max-w-none" + ? "max-w-none select-none" : isUserSettingsPage || isSearchSpaceSettingsPage || isTeamPage ? "max-w-5xl" : undefined @@ -759,10 +777,7 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid markAllAsRead: statusInbox.markAllAsRead, }, }} - allSharedChatsPanel={{ - searchSpaceId, - }} - allPrivateChatsPanel={{ + allChatsPanel={{ searchSpaceId, }} documentsPanel={{ @@ -770,6 +785,7 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid onOpenChange: setIsDocumentsSidebarOpen, }} onTabSwitch={handleTabSwitch} + onTabPrefetch={handleTabPrefetch} > {children} @@ -841,7 +857,12 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid {tSidebar("rename") || "Rename"} - {isRenamingChat && } + {isRenamingChat && ( + + )} diff --git a/surfsense_web/components/layout/types/layout.types.ts b/surfsense_web/components/layout/types/layout.types.ts index 720aaecf1..1bb0a089e 100644 --- a/surfsense_web/components/layout/types/layout.types.ts +++ b/surfsense_web/components/layout/types/layout.types.ts @@ -70,8 +70,7 @@ export interface ChatsSectionProps { activeChatId?: number | null; onChatSelect: (chat: ChatItem) => void; onChatDelete?: (chat: ChatItem) => void; - onViewAllSharedChats?: () => void; - onViewAllPrivateChats?: () => void; + onViewAllChats?: () => void; searchSpaceId?: string; } @@ -96,13 +95,11 @@ export interface SidebarProps { searchSpaceId?: string; navItems: NavItem[]; chats: ChatItem[]; - sharedChats?: ChatItem[]; activeChatId?: number | null; onNewChat: () => void; onChatSelect: (chat: ChatItem) => void; onChatDelete?: (chat: ChatItem) => void; - onViewAllSharedChats?: () => void; - onViewAllPrivateChats?: () => void; + onViewAllChats?: () => void; user: User; theme?: string; onSettings?: () => void; diff --git a/surfsense_web/components/layout/ui/header/Header.tsx b/surfsense_web/components/layout/ui/header/Header.tsx index c6ccfddc6..79839622d 100644 --- a/surfsense_web/components/layout/ui/header/Header.tsx +++ b/surfsense_web/components/layout/ui/header/Header.tsx @@ -8,7 +8,7 @@ import { activeTabAtom } from "@/atoms/tabs/tabs.atom"; import { ActionLogButton } from "@/components/agent-action-log/action-log-button"; import { ChatHeader } from "@/components/new-chat/chat-header"; import { ChatShareButton } from "@/components/new-chat/chat-share-button"; -import type { ChatVisibility, ThreadRecord } from "@/lib/chat/thread-persistence"; +import type { ThreadRecord } from "@/lib/chat/thread-persistence"; interface HeaderProps { mobileMenuTrigger?: React.ReactNode; @@ -26,6 +26,14 @@ export function Header({ mobileMenuTrigger }: HeaderProps) { const currentThreadState = useAtomValue(currentThreadAtom); const hasThread = isChatPage && !isDocumentTab && currentThreadState.id !== null; + const activeSearchSpaceId = searchSpaceId ? Number(searchSpaceId) : null; + const canRenderShareButton = + hasThread && + currentThreadState.id !== null && + currentThreadState.visibility !== null && + currentThreadState.searchSpaceId !== null && + activeSearchSpaceId !== null && + currentThreadState.searchSpaceId === activeSearchSpaceId; // Free chat pages have their own header with model selector; only render mobile trigger if (isFreePage) { @@ -37,21 +45,24 @@ export function Header({ mobileMenuTrigger }: HeaderProps) { ); } - const threadForButton: ThreadRecord | null = - hasThread && currentThreadState.id !== null - ? { - id: currentThreadState.id, - visibility: currentThreadState.visibility ?? "PRIVATE", - created_by_id: null, - search_space_id: 0, - title: "", - archived: false, - created_at: "", - updated_at: "", - } - : null; - - const handleVisibilityChange = (_visibility: ChatVisibility) => {}; + let threadForButton: ThreadRecord | null = null; + if ( + canRenderShareButton && + currentThreadState.id !== null && + currentThreadState.visibility !== null && + currentThreadState.searchSpaceId !== null + ) { + threadForButton = { + id: currentThreadState.id, + visibility: currentThreadState.visibility, + created_by_id: null, + search_space_id: currentThreadState.searchSpaceId, + title: "", + archived: false, + created_at: "", + updated_at: "", + }; + } return (
    @@ -66,9 +77,7 @@ export function Header({ mobileMenuTrigger }: HeaderProps) { {/* Right side - Actions */}
    {hasThread && } - {hasThread && ( - - )} + {threadForButton && }
    ); diff --git a/surfsense_web/components/layout/ui/shell/LayoutShell.tsx b/surfsense_web/components/layout/ui/shell/LayoutShell.tsx index 91d85cc1e..1c076a254 100644 --- a/surfsense_web/components/layout/ui/shell/LayoutShell.tsx +++ b/surfsense_web/components/layout/ui/shell/LayoutShell.tsx @@ -27,8 +27,7 @@ import { RightPanelToggleButton, } from "../right-panel/RightPanel"; import { - AllPrivateChatsSidebarContent, - AllSharedChatsSidebarContent, + AllChatsSidebarContent, DocumentsSidebar, InboxSidebarContent, MobileSidebar, @@ -94,7 +93,7 @@ interface TabDataSource { markAllAsRead: () => Promise; } -export type ActiveSlideoutPanel = "inbox" | "shared" | "private" | null; +export type ActiveSlideoutPanel = "inbox" | "chats" | null; // Inbox-related props — per-tab data sources with independent loading/pagination interface InboxProps { @@ -115,15 +114,14 @@ interface LayoutShellProps { navItems: NavItem[]; onNavItemClick?: (item: NavItem) => void; chats: ChatItem[]; - sharedChats?: ChatItem[]; activeChatId?: number | null; onNewChat: () => void; onChatSelect: (chat: ChatItem) => void; + onChatPrefetch?: (chat: ChatItem) => void; onChatRename?: (chat: ChatItem) => void; onChatDelete?: (chat: ChatItem) => void; onChatArchive?: (chat: ChatItem) => void; - onViewAllSharedChats?: () => void; - onViewAllPrivateChats?: () => void; + onViewAllChats?: () => void; user: User; onSettings?: () => void; onManageMembers?: () => void; @@ -148,10 +146,7 @@ interface LayoutShellProps { inbox?: InboxProps; isLoadingChats?: boolean; // All chats panel props - allSharedChatsPanel?: { - searchSpaceId: string; - }; - allPrivateChatsPanel?: { + allChatsPanel?: { searchSpaceId: string; }; documentsPanel?: { @@ -159,11 +154,13 @@ interface LayoutShellProps { onOpenChange: (open: boolean) => void; }; onTabSwitch?: (tab: Tab) => void; + onTabPrefetch?: (tab: Tab) => void; } function MainContentPanel({ isChatPage, onTabSwitch, + onTabPrefetch, onNewChat, showRightPanelExpandButton = true, showTopBorder = false, @@ -171,6 +168,7 @@ function MainContentPanel({ }: { isChatPage: boolean; onTabSwitch?: (tab: Tab) => void; + onTabPrefetch?: (tab: Tab) => void; onNewChat?: () => void; showRightPanelExpandButton?: boolean; showTopBorder?: boolean; @@ -185,6 +183,7 @@ function MainContentPanel({ > : null} className="min-w-0" @@ -226,15 +225,14 @@ export function LayoutShell({ navItems, onNavItemClick, chats, - sharedChats, activeChatId, onNewChat, onChatSelect, + onChatPrefetch, onChatRename, onChatDelete, onChatArchive, - onViewAllSharedChats, - onViewAllPrivateChats, + onViewAllChats, user, onSettings, onManageMembers, @@ -256,10 +254,10 @@ export function LayoutShell({ onSlideoutPanelChange, inbox, isLoadingChats = false, - allSharedChatsPanel, - allPrivateChatsPanel, + allChatsPanel, documentsPanel, onTabSwitch, + onTabPrefetch, }: LayoutShellProps) { const isMobile = useIsMobile(); const electronAPI = useElectronAPI(); @@ -288,13 +286,7 @@ export function LayoutShell({ const anySlideOutOpen = activeSlideoutPanel !== null; const panelAriaLabel = - activeSlideoutPanel === "inbox" - ? "Inbox" - : activeSlideoutPanel === "shared" - ? "Shared Chats" - : activeSlideoutPanel === "private" - ? "Private Chats" - : "Panel"; + activeSlideoutPanel === "inbox" ? "Inbox" : activeSlideoutPanel === "chats" ? "Chats" : "Panel"; // Mobile layout if (isMobile) { @@ -317,17 +309,15 @@ export function LayoutShell({ navItems={navItems} onNavItemClick={onNavItemClick} chats={chats} - sharedChats={sharedChats} activeChatId={activeChatId} onNewChat={onNewChat} onChatSelect={onChatSelect} + onChatPrefetch={onChatPrefetch} onChatRename={onChatRename} onChatDelete={onChatDelete} onChatArchive={onChatArchive} - onViewAllSharedChats={onViewAllSharedChats} - onViewAllPrivateChats={onViewAllPrivateChats} - isSharedChatsPanelOpen={activeSlideoutPanel === "shared"} - isPrivateChatsPanelOpen={activeSlideoutPanel === "private"} + onViewAllChats={onViewAllChats} + isChatsPanelOpen={activeSlideoutPanel === "chats"} user={user} onSettings={onSettings} onManageMembers={onManageMembers} @@ -379,34 +369,18 @@ export function LayoutShell({ /> )} - {activeSlideoutPanel === "shared" && allSharedChatsPanel && ( + {activeSlideoutPanel === "chats" && allChatsPanel && ( - closeSlideout(open)} - searchSpaceId={allSharedChatsPanel.searchSpaceId} - onCloseMobileSidebar={() => setMobileMenuOpen(false)} - /> - - )} - {activeSlideoutPanel === "private" && allPrivateChatsPanel && ( - - closeSlideout(open)} - searchSpaceId={allPrivateChatsPanel.searchSpaceId} + searchSpaceId={allChatsPanel.searchSpaceId} onCloseMobileSidebar={() => setMobileMenuOpen(false)} /> @@ -478,17 +452,15 @@ export function LayoutShell({ navItems={navItems} onNavItemClick={onNavItemClick} chats={chats} - sharedChats={sharedChats} activeChatId={activeChatId} onNewChat={onNewChat} onChatSelect={onChatSelect} + onChatPrefetch={onChatPrefetch} onChatRename={onChatRename} onChatDelete={onChatDelete} onChatArchive={onChatArchive} - onViewAllSharedChats={onViewAllSharedChats} - onViewAllPrivateChats={onViewAllPrivateChats} - isSharedChatsPanelOpen={activeSlideoutPanel === "shared"} - isPrivateChatsPanelOpen={activeSlideoutPanel === "private"} + onViewAllChats={onViewAllChats} + isChatsPanelOpen={activeSlideoutPanel === "chats"} user={user} onSettings={onSettings} onManageMembers={onManageMembers} @@ -554,33 +526,18 @@ export function LayoutShell({ /> )} - {activeSlideoutPanel === "shared" && allSharedChatsPanel && ( + {activeSlideoutPanel === "chats" && allChatsPanel && ( - closeSlideout(open)} - searchSpaceId={allSharedChatsPanel.searchSpaceId} - /> - - )} - {activeSlideoutPanel === "private" && allPrivateChatsPanel && ( - - closeSlideout(open)} - searchSpaceId={allPrivateChatsPanel.searchSpaceId} + searchSpaceId={allChatsPanel.searchSpaceId} /> )} @@ -603,6 +560,7 @@ export function LayoutShell({ void; searchSpaceId: string; onCloseMobileSidebar?: () => void; } -interface AllPrivateChatsSidebarProps extends AllPrivateChatsSidebarContentProps { +interface AllChatsSidebarProps extends AllChatsSidebarContentProps { open: boolean; } -export function AllPrivateChatsSidebarContent({ +export function AllChatsSidebarContent({ onOpenChange, searchSpaceId, onCloseMobileSidebar, -}: AllPrivateChatsSidebarContentProps) { +}: AllChatsSidebarContentProps) { const t = useTranslations("sidebar"); const router = useRouter(); const params = useParams(); const queryClient = useQueryClient(); const isMobile = useIsMobile(); const removeChatTab = useSetAtom(removeChatTabAtom); + const { activateChatThread, prefetchChatThread } = useActivateChatThread(); + const { mutateAsync: deleteThread } = useDeleteThread(searchSpaceId); + const { mutateAsync: archiveThread } = useArchiveThread(searchSpaceId); + const { mutateAsync: renameThread } = useRenameThread(searchSpaceId); const currentChatId = Array.isArray(params.chat_id) ? Number(params.chat_id[0]) @@ -122,57 +124,66 @@ export function AllPrivateChatsSidebarContent({ enabled: !!searchSpaceId && isSearchMode, }); - // Filter to only private chats (PRIVATE visibility or no visibility set) const { activeChats, archivedChats } = useMemo(() => { if (isSearchMode) { - const privateSearchResults = (searchData ?? []).filter( - (thread) => thread.visibility !== "SEARCH_SPACE" - ); return { - activeChats: privateSearchResults.filter((t) => !t.archived), - archivedChats: privateSearchResults.filter((t) => t.archived), + activeChats: (searchData ?? []).filter((t) => !t.archived), + archivedChats: (searchData ?? []).filter((t) => t.archived), }; } if (!threadsData) return { activeChats: [], archivedChats: [] }; - const activePrivate = threadsData.threads.filter( - (thread) => thread.visibility !== "SEARCH_SPACE" - ); - const archivedPrivate = threadsData.archived_threads.filter( - (thread) => thread.visibility !== "SEARCH_SPACE" - ); - - return { activeChats: activePrivate, archivedChats: archivedPrivate }; + return { + activeChats: threadsData.threads, + archivedChats: threadsData.archived_threads, + }; }, [threadsData, searchData, isSearchMode]); const threads = showArchived ? archivedChats : activeChats; const handleThreadClick = useCallback( - (threadId: number) => { - router.push(`/dashboard/${searchSpaceId}/new-chat/${threadId}`); + (thread: ThreadListItem) => { + activateChatThread({ + id: thread.id, + title: thread.title || "New Chat", + searchSpaceId, + visibility: thread.visibility, + }); onOpenChange(false); onCloseMobileSidebar?.(); }, - [router, onOpenChange, searchSpaceId, onCloseMobileSidebar] + [activateChatThread, onOpenChange, searchSpaceId, onCloseMobileSidebar] ); const handleDeleteThread = useCallback( async (threadId: number) => { setDeletingThreadId(threadId); try { - await deleteThread(threadId); + await deleteThread({ threadId }); const fallbackTab = removeChatTab(threadId); toast.success(t("chat_deleted") || "Chat deleted successfully"); - queryClient.invalidateQueries({ queryKey: ["all-threads", searchSpaceId] }); - queryClient.invalidateQueries({ queryKey: ["search-threads", searchSpaceId] }); - queryClient.invalidateQueries({ queryKey: ["threads", searchSpaceId] }); if (currentChatId === threadId) { onOpenChange(false); setTimeout(() => { - if (fallbackTab?.type === "chat" && fallbackTab.chatUrl) { - router.push(fallbackTab.chatUrl); + if ( + fallbackTab?.type === "chat" && + fallbackTab.chatUrl && + fallbackTab.chatId !== undefined + ) { + activateChatThread({ + id: fallbackTab.chatId ?? null, + title: fallbackTab.title, + url: fallbackTab.chatUrl, + searchSpaceId: fallbackTab.searchSpaceId ?? searchSpaceId, + ...(fallbackTab.visibility !== undefined + ? { visibility: fallbackTab.visibility } + : {}), + ...(fallbackTab.hasComments !== undefined + ? { hasComments: fallbackTab.hasComments } + : {}), + }); return; } router.push(`/dashboard/${searchSpaceId}/new-chat`); @@ -185,22 +196,28 @@ export function AllPrivateChatsSidebarContent({ setDeletingThreadId(null); } }, - [queryClient, searchSpaceId, t, currentChatId, router, onOpenChange, removeChatTab] + [ + activateChatThread, + deleteThread, + t, + currentChatId, + router, + onOpenChange, + removeChatTab, + searchSpaceId, + ] ); const handleToggleArchive = useCallback( async (threadId: number, currentlyArchived: boolean) => { setArchivingThreadId(threadId); try { - await updateThread(threadId, { archived: !currentlyArchived }); + await archiveThread({ threadId, archived: !currentlyArchived }); toast.success( currentlyArchived ? t("chat_unarchived") || "Chat restored" : t("chat_archived") || "Chat archived" ); - queryClient.invalidateQueries({ queryKey: ["all-threads", searchSpaceId] }); - queryClient.invalidateQueries({ queryKey: ["search-threads", searchSpaceId] }); - queryClient.invalidateQueries({ queryKey: ["threads", searchSpaceId] }); } catch (error) { console.error("Error archiving thread:", error); toast.error(t("error_archiving_chat") || "Failed to archive chat"); @@ -208,7 +225,7 @@ export function AllPrivateChatsSidebarContent({ setArchivingThreadId(null); } }, - [queryClient, searchSpaceId, t] + [archiveThread, t] ); const handleStartRename = useCallback((threadId: number, title: string) => { @@ -221,14 +238,12 @@ export function AllPrivateChatsSidebarContent({ if (!renamingThread || !newTitle.trim()) return; setIsRenaming(true); try { - await updateThread(renamingThread.id, { title: newTitle.trim() }); - toast.success(t("chat_renamed") || "Chat renamed"); - queryClient.invalidateQueries({ queryKey: ["all-threads", searchSpaceId] }); - queryClient.invalidateQueries({ queryKey: ["search-threads", searchSpaceId] }); - queryClient.invalidateQueries({ queryKey: ["threads", searchSpaceId] }); - queryClient.invalidateQueries({ - queryKey: ["threads", searchSpaceId, "detail", String(renamingThread.id)], + await renameThread({ + threadId: renamingThread.id, + title: newTitle.trim(), + previousTitle: renamingThread.title, }); + toast.success(t("chat_renamed") || "Chat renamed"); } catch (error) { console.error("Error renaming thread:", error); toast.error(t("error_renaming_chat") || "Failed to rename chat"); @@ -238,7 +253,7 @@ export function AllPrivateChatsSidebarContent({ setRenamingThread(null); setNewTitle(""); } - }, [renamingThread, newTitle, queryClient, searchSpaceId, t]); + }, [renamingThread, newTitle, renameThread, t]); const handleClearSearch = useCallback(() => { setSearchQuery(""); @@ -265,7 +280,7 @@ export function AllPrivateChatsSidebarContent({ {t("close") || "Close"} )} -

    {t("chats") || "Private Chats"}

    +

    {t("chats") || "Chats"}

    @@ -353,8 +368,10 @@ export function AllPrivateChatsSidebarContent({ variant="ghost" onClick={() => { if (wasLongPress()) return; - handleThreadClick(thread.id); + handleThreadClick(thread); }} + onMouseEnter={() => prefetchChatThread(thread.id)} + onFocus={() => prefetchChatThread(thread.id)} onTouchStart={() => { pendingThreadIdRef.current = thread.id; longPressHandlers.onTouchStart(); @@ -366,11 +383,12 @@ export function AllPrivateChatsSidebarContent({ "h-auto w-full justify-start gap-2 overflow-hidden px-2 py-1.5 text-left font-normal", "group-hover/item:bg-accent group-hover/item:text-accent-foreground", "focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring", + thread.visibility === "SEARCH_SPACE" && "pr-9", isActive && "bg-accent text-accent-foreground", isBusy && "opacity-50 pointer-events-none" )} > - {thread.title || "New Chat"} + {thread.title || "New Chat"} ) : ( @@ -378,17 +396,22 @@ export function AllPrivateChatsSidebarContent({ @@ -407,64 +430,83 @@ export function AllPrivateChatsSidebarContent({ : "bg-gradient-to-l from-sidebar from-60% to-transparent group-hover/item:from-accent", isMobile ? "opacity-0" - : openDropdownId === thread.id + : thread.visibility === "SEARCH_SPACE" || openDropdownId === thread.id ? "opacity-100" : "opacity-0 group-hover/item:opacity-100" )} > - setOpenDropdownId(isOpen ? thread.id : null)} - > - - - - - {!thread.archived && ( - handleStartRename(thread.id, thread.title || "New Chat")} + /> + ) : null} + setOpenDropdownId(isOpen ? thread.id : null)} + > + + + + + {!thread.archived && ( + + handleStartRename(thread.id, thread.title || "New Chat") + } + > + + {t("rename") || "Rename"} + )} - - handleDeleteThread(thread.id)}> - - {t("delete") || "Delete"} - - - + handleToggleArchive(thread.id, thread.archived)} + disabled={isArchiving} + > + {thread.archived ? ( + <> + + {t("unarchive") || "Restore"} + + ) : ( + <> + + {t("archive") || "Archive"} + + )} + + handleDeleteThread(thread.id)}> + + {t("delete") || "Delete"} + + + +
    ); @@ -486,7 +528,7 @@ export function AllPrivateChatsSidebarContent({

    {showArchived ? t("no_archived_chats") || "No archived chats" - : t("no_chats") || "No private chats"} + : t("no_chats") || "No chats"}

    {!showArchived && (

    @@ -527,16 +569,17 @@ export function AllPrivateChatsSidebarContent({ @@ -545,21 +588,17 @@ export function AllPrivateChatsSidebarContent({ ); } -export function AllPrivateChatsSidebar({ +export function AllChatsSidebar({ open, onOpenChange, searchSpaceId, onCloseMobileSidebar, -}: AllPrivateChatsSidebarProps) { +}: AllChatsSidebarProps) { const t = useTranslations("sidebar"); return ( - - + void; - searchSpaceId: string; - onCloseMobileSidebar?: () => void; -} - -interface AllSharedChatsSidebarProps extends AllSharedChatsSidebarContentProps { - open: boolean; -} - -export function AllSharedChatsSidebarContent({ - onOpenChange, - searchSpaceId, - onCloseMobileSidebar, -}: AllSharedChatsSidebarContentProps) { - const t = useTranslations("sidebar"); - const router = useRouter(); - const params = useParams(); - const queryClient = useQueryClient(); - const isMobile = useIsMobile(); - const removeChatTab = useSetAtom(removeChatTabAtom); - - const currentChatId = Array.isArray(params.chat_id) - ? Number(params.chat_id[0]) - : params.chat_id - ? Number(params.chat_id) - : null; - const [deletingThreadId, setDeletingThreadId] = useState(null); - const [archivingThreadId, setArchivingThreadId] = useState(null); - const [searchQuery, setSearchQuery] = useState(""); - const [showArchived, setShowArchived] = useState(false); - const [openDropdownId, setOpenDropdownId] = useState(null); - const [showRenameDialog, setShowRenameDialog] = useState(false); - const [renamingThread, setRenamingThread] = useState<{ id: number; title: string } | null>(null); - const [newTitle, setNewTitle] = useState(""); - const [isRenaming, setIsRenaming] = useState(false); - const debouncedSearchQuery = useDebouncedValue(searchQuery, 300); - - const pendingThreadIdRef = useRef(null); - const { handlers: longPressHandlers, wasLongPress } = useLongPress( - useCallback(() => { - if (pendingThreadIdRef.current !== null) { - setOpenDropdownId(pendingThreadIdRef.current); - } - }, []) - ); - - const isSearchMode = !!debouncedSearchQuery.trim(); - - const { - data: threadsData, - error: threadsError, - isLoading: isLoadingThreads, - } = useQuery({ - queryKey: ["all-threads", searchSpaceId], - queryFn: () => fetchThreads(Number(searchSpaceId)), - enabled: !!searchSpaceId && !isSearchMode, - }); - - const { - data: searchData, - error: searchError, - isLoading: isLoadingSearch, - } = useQuery({ - queryKey: ["search-threads", searchSpaceId, debouncedSearchQuery], - queryFn: () => searchThreads(Number(searchSpaceId), debouncedSearchQuery.trim()), - enabled: !!searchSpaceId && isSearchMode, - }); - - // Filter to only shared chats (SEARCH_SPACE visibility) - const { activeChats, archivedChats } = useMemo(() => { - if (isSearchMode) { - const sharedSearchResults = (searchData ?? []).filter( - (thread) => thread.visibility === "SEARCH_SPACE" - ); - return { - activeChats: sharedSearchResults.filter((t) => !t.archived), - archivedChats: sharedSearchResults.filter((t) => t.archived), - }; - } - - if (!threadsData) return { activeChats: [], archivedChats: [] }; - - const activeShared = threadsData.threads.filter( - (thread) => thread.visibility === "SEARCH_SPACE" - ); - const archivedShared = threadsData.archived_threads.filter( - (thread) => thread.visibility === "SEARCH_SPACE" - ); - - return { activeChats: activeShared, archivedChats: archivedShared }; - }, [threadsData, searchData, isSearchMode]); - - const threads = showArchived ? archivedChats : activeChats; - - const handleThreadClick = useCallback( - (threadId: number) => { - router.push(`/dashboard/${searchSpaceId}/new-chat/${threadId}`); - onOpenChange(false); - onCloseMobileSidebar?.(); - }, - [router, onOpenChange, searchSpaceId, onCloseMobileSidebar] - ); - - const handleDeleteThread = useCallback( - async (threadId: number) => { - setDeletingThreadId(threadId); - try { - await deleteThread(threadId); - const fallbackTab = removeChatTab(threadId); - toast.success(t("chat_deleted") || "Chat deleted successfully"); - queryClient.invalidateQueries({ queryKey: ["all-threads", searchSpaceId] }); - queryClient.invalidateQueries({ queryKey: ["search-threads", searchSpaceId] }); - queryClient.invalidateQueries({ queryKey: ["threads", searchSpaceId] }); - - if (currentChatId === threadId) { - onOpenChange(false); - setTimeout(() => { - if (fallbackTab?.type === "chat" && fallbackTab.chatUrl) { - router.push(fallbackTab.chatUrl); - return; - } - router.push(`/dashboard/${searchSpaceId}/new-chat`); - }, 250); - } - } catch (error) { - console.error("Error deleting thread:", error); - toast.error(t("error_deleting_chat") || "Failed to delete chat"); - } finally { - setDeletingThreadId(null); - } - }, - [queryClient, searchSpaceId, t, currentChatId, router, onOpenChange, removeChatTab] - ); - - const handleToggleArchive = useCallback( - async (threadId: number, currentlyArchived: boolean) => { - setArchivingThreadId(threadId); - try { - await updateThread(threadId, { archived: !currentlyArchived }); - toast.success( - currentlyArchived - ? t("chat_unarchived") || "Chat restored" - : t("chat_archived") || "Chat archived" - ); - queryClient.invalidateQueries({ queryKey: ["all-threads", searchSpaceId] }); - queryClient.invalidateQueries({ queryKey: ["search-threads", searchSpaceId] }); - queryClient.invalidateQueries({ queryKey: ["threads", searchSpaceId] }); - } catch (error) { - console.error("Error archiving thread:", error); - toast.error(t("error_archiving_chat") || "Failed to archive chat"); - } finally { - setArchivingThreadId(null); - } - }, - [queryClient, searchSpaceId, t] - ); - - const handleStartRename = useCallback((threadId: number, title: string) => { - setRenamingThread({ id: threadId, title }); - setNewTitle(title); - setShowRenameDialog(true); - }, []); - - const handleConfirmRename = useCallback(async () => { - if (!renamingThread || !newTitle.trim()) return; - setIsRenaming(true); - try { - await updateThread(renamingThread.id, { title: newTitle.trim() }); - toast.success(t("chat_renamed") || "Chat renamed"); - queryClient.invalidateQueries({ queryKey: ["all-threads", searchSpaceId] }); - queryClient.invalidateQueries({ queryKey: ["search-threads", searchSpaceId] }); - queryClient.invalidateQueries({ queryKey: ["threads", searchSpaceId] }); - queryClient.invalidateQueries({ - queryKey: ["threads", searchSpaceId, "detail", String(renamingThread.id)], - }); - } catch (error) { - console.error("Error renaming thread:", error); - toast.error(t("error_renaming_chat") || "Failed to rename chat"); - } finally { - setIsRenaming(false); - setShowRenameDialog(false); - setRenamingThread(null); - setNewTitle(""); - } - }, [renamingThread, newTitle, queryClient, searchSpaceId, t]); - - const handleClearSearch = useCallback(() => { - setSearchQuery(""); - }, []); - - const isLoading = isSearchMode ? isLoadingSearch : isLoadingThreads; - const error = isSearchMode ? searchError : threadsError; - - const activeCount = activeChats.length; - const archivedCount = archivedChats.length; - - return ( - <> -

    -
    - {isMobile && ( - - )} -

    {t("shared_chats") || "Shared Chats"}

    -
    - -
    - - setSearchQuery(e.target.value)} - className="h-8 border-0 bg-muted pl-8 pr-7 text-sm shadow-none" - /> - {searchQuery && ( - - )} -
    -
    - - {!isSearchMode && ( - setShowArchived(value === "archived")} - className="shrink-0 mx-3 mt-1.5" - > - - - - - Active - - {activeCount} - - - - - - - Archived - - {archivedCount} - - - - - - )} - -
    - {isLoading ? ( -
    - {[75, 90, 55, 80, 65, 85].map((titleWidth) => ( -
    - - -
    - ))} -
    - ) : error ? ( -
    - {t("error_loading_chats") || "Error loading chats"} -
    - ) : threads.length > 0 ? ( -
    - {threads.map((thread) => { - const isDeleting = deletingThreadId === thread.id; - const isArchiving = archivingThreadId === thread.id; - const isBusy = isDeleting || isArchiving; - const isActive = currentChatId === thread.id; - - return ( -
    - {isMobile ? ( - - ) : ( - - - - - -

    - {t("updated") || "Updated"}: {formatThreadTimestamp(thread.updatedAt)} -

    -
    -
    - )} - -
    - setOpenDropdownId(isOpen ? thread.id : null)} - > - - - - - {!thread.archived && ( - handleStartRename(thread.id, thread.title || "New Chat")} - > - - {t("rename") || "Rename"} - - )} - handleToggleArchive(thread.id, thread.archived)} - disabled={isArchiving} - > - {thread.archived ? ( - <> - - {t("unarchive") || "Restore"} - - ) : ( - <> - - {t("archive") || "Archive"} - - )} - - handleDeleteThread(thread.id)}> - - {t("delete") || "Delete"} - - - -
    -
    - ); - })} -
    - ) : isSearchMode ? ( -
    - -

    - {t("no_chats_found") || "No chats found"} -

    -

    - {t("try_different_search") || "Try a different search term"} -

    -
    - ) : ( -
    - -

    - {showArchived - ? t("no_archived_chats") || "No archived chats" - : t("no_shared_chats") || "No shared chats"} -

    - {!showArchived && ( -

    - Share a chat to collaborate with your team -

    - )} -
    - )} -
    - - - - - {t("rename_chat") || "Rename Chat"} - - - {t("rename_chat_description") || "Enter a new name for this conversation."} - - - setNewTitle(e.target.value)} - placeholder={t("chat_title_placeholder") || "Chat title"} - onKeyDown={(e) => { - if (e.key === "Enter" && !isRenaming && newTitle.trim()) { - handleConfirmRename(); - } - }} - /> - - - - - - - - ); -} - -export function AllSharedChatsSidebar({ - open, - onOpenChange, - searchSpaceId, - onCloseMobileSidebar, -}: AllSharedChatsSidebarProps) { - const t = useTranslations("sidebar"); - - return ( - - - - ); -} diff --git a/surfsense_web/components/layout/ui/sidebar/ChatListItem.tsx b/surfsense_web/components/layout/ui/sidebar/ChatListItem.tsx index ea4f946c2..c854225a2 100644 --- a/surfsense_web/components/layout/ui/sidebar/ChatListItem.tsx +++ b/surfsense_web/components/layout/ui/sidebar/ChatListItem.tsx @@ -18,10 +18,12 @@ import { cn } from "@/lib/utils"; interface ChatListItemProps { name: string; isActive?: boolean; + isShared?: boolean; archived?: boolean; dropdownOpen?: boolean; onDropdownOpenChange?: (open: boolean) => void; onClick?: () => void; + onPrefetch?: () => void; onRename?: () => void; onArchive?: () => void; onDelete?: () => void; @@ -34,6 +36,7 @@ export function ChatListItem({ dropdownOpen: controlledOpen, onDropdownOpenChange, onClick, + onPrefetch, onRename, onArchive, onDelete, @@ -60,6 +63,8 @@ export function ChatListItem({ type="button" variant="ghost" onClick={handleClick} + onMouseEnter={onPrefetch} + onFocus={onPrefetch} {...(isMobile ? longPressHandlers : {})} className={cn( "h-auto w-full justify-start gap-2 overflow-hidden px-2 py-1.5 text-left font-normal", @@ -68,7 +73,7 @@ export function ChatListItem({ isActive && "bg-accent text-accent-foreground" )} > - {animatedName} + {animatedName} {/* Actions dropdown - trigger hidden on mobile, long-press opens it instead */} diff --git a/surfsense_web/components/layout/ui/sidebar/DocumentsSidebar.tsx b/surfsense_web/components/layout/ui/sidebar/DocumentsSidebar.tsx index a90d6b32e..6c6668319 100644 --- a/surfsense_web/components/layout/ui/sidebar/DocumentsSidebar.tsx +++ b/surfsense_web/components/layout/ui/sidebar/DocumentsSidebar.tsx @@ -619,7 +619,6 @@ function AuthenticatedDocumentsSidebarBase({ searchSpaceId, excludePatterns: matched.excludePatterns ?? DEFAULT_EXCLUDE_PATTERNS, fileExtensions: matched.fileExtensions ?? Array.from(getSupportedExtensionsSet()), - enableSummary: false, rootFolderId: folder.id, }); toast.success(`Re-scan complete: ${matched.name}`); diff --git a/surfsense_web/components/layout/ui/sidebar/MobileSidebar.tsx b/surfsense_web/components/layout/ui/sidebar/MobileSidebar.tsx index 83d423ace..89a01d0c7 100644 --- a/surfsense_web/components/layout/ui/sidebar/MobileSidebar.tsx +++ b/surfsense_web/components/layout/ui/sidebar/MobileSidebar.tsx @@ -19,17 +19,15 @@ interface MobileSidebarProps { navItems: NavItem[]; onNavItemClick?: (item: NavItem) => void; chats: ChatItem[]; - sharedChats?: ChatItem[]; activeChatId?: number | null; onNewChat: () => void; onChatSelect: (chat: ChatItem) => void; + onChatPrefetch?: (chat: ChatItem) => void; onChatRename?: (chat: ChatItem) => void; onChatDelete?: (chat: ChatItem) => void; onChatArchive?: (chat: ChatItem) => void; - onViewAllSharedChats?: () => void; - onViewAllPrivateChats?: () => void; - isSharedChatsPanelOpen?: boolean; - isPrivateChatsPanelOpen?: boolean; + onViewAllChats?: () => void; + isChatsPanelOpen?: boolean; user: User; onSettings?: () => void; onManageMembers?: () => void; @@ -69,17 +67,15 @@ export function MobileSidebar({ navItems, onNavItemClick, chats, - sharedChats, activeChatId, onNewChat, onChatSelect, + onChatPrefetch, onChatRename, onChatDelete, onChatArchive, - onViewAllSharedChats, - onViewAllPrivateChats, - isSharedChatsPanelOpen = false, - isPrivateChatsPanelOpen = false, + onViewAllChats, + isChatsPanelOpen = false, user, onSettings, onManageMembers, @@ -152,34 +148,25 @@ export function MobileSidebar({ navItems={navItems} onNavItemClick={handleNavItemClick} chats={chats} - sharedChats={sharedChats} activeChatId={activeChatId} onNewChat={() => { onNewChat(); onOpenChange(false); }} onChatSelect={handleChatSelect} + onChatPrefetch={onChatPrefetch} onChatRename={onChatRename} onChatDelete={onChatDelete} onChatArchive={onChatArchive} - onViewAllSharedChats={ - onViewAllSharedChats + onViewAllChats={ + onViewAllChats ? () => { onOpenChange(false); - onViewAllSharedChats(); + onViewAllChats(); } : undefined } - onViewAllPrivateChats={ - onViewAllPrivateChats - ? () => { - onOpenChange(false); - onViewAllPrivateChats(); - } - : undefined - } - isSharedChatsPanelOpen={isSharedChatsPanelOpen} - isPrivateChatsPanelOpen={isPrivateChatsPanelOpen} + isChatsPanelOpen={isChatsPanelOpen} user={user} onSettings={ onSettings diff --git a/surfsense_web/components/layout/ui/sidebar/Sidebar.tsx b/surfsense_web/components/layout/ui/sidebar/Sidebar.tsx index 805f8bfd3..6a4785d98 100644 --- a/surfsense_web/components/layout/ui/sidebar/Sidebar.tsx +++ b/surfsense_web/components/layout/ui/sidebar/Sidebar.tsx @@ -67,17 +67,15 @@ interface SidebarProps { navItems: NavItem[]; onNavItemClick?: (item: NavItem) => void; chats: ChatItem[]; - sharedChats?: ChatItem[]; activeChatId?: number | null; onNewChat: () => void; onChatSelect: (chat: ChatItem) => void; + onChatPrefetch?: (chat: ChatItem) => void; onChatRename?: (chat: ChatItem) => void; onChatDelete?: (chat: ChatItem) => void; onChatArchive?: (chat: ChatItem) => void; - onViewAllSharedChats?: () => void; - onViewAllPrivateChats?: () => void; - isSharedChatsPanelOpen?: boolean; - isPrivateChatsPanelOpen?: boolean; + onViewAllChats?: () => void; + isChatsPanelOpen?: boolean; user: User; onSettings?: () => void; onManageMembers?: () => void; @@ -106,17 +104,15 @@ export function Sidebar({ navItems, onNavItemClick, chats, - sharedChats = [], activeChatId, onNewChat, onChatSelect, + onChatPrefetch, onChatRename, onChatDelete, onChatArchive, - onViewAllSharedChats, - onViewAllPrivateChats, - isSharedChatsPanelOpen = false, - isPrivateChatsPanelOpen = false, + onViewAllChats, + isChatsPanelOpen = false, user, onSettings, onManageMembers, @@ -264,73 +260,20 @@ export function Sidebar({
    ) : (
    - {/* Shared Chats Section - takes only space needed, max 50% */} - {!disableTooltips && isSharedChatsPanelOpen ? t("hide") : t("show_all")} - - ) : undefined - } - > - {isLoadingChats ? ( - - ) : sharedChats.length > 0 ? ( -
    -
    4 ? "pb-2" : ""}`} - > - {sharedChats.slice(0, 20).map((chat) => ( - setOpenDropdownChatId(open ? chat.id : null)} - onClick={() => onChatSelect(chat)} - onRename={() => onChatRename?.(chat)} - onArchive={() => onChatArchive?.(chat)} - onDelete={() => onChatDelete?.(chat)} - /> - ))} -
    - {/* Gradient fade indicator when more than 4 items */} - {sharedChats.length > 4 && ( -
    - )} -
    - ) : ( -

    {t("no_shared_chats")}

    - )} - - - {/* Private Chats Section - fills remaining space */} - - {!disableTooltips && isPrivateChatsPanelOpen ? t("hide") : t("show_all")} + {!disableTooltips && isChatsPanelOpen ? t("hide") : t("show_all")} ) : undefined } @@ -347,10 +290,12 @@ export function Sidebar({ key={chat.id} name={chat.name} isActive={chat.id === activeChatId} + isShared={chat.visibility === "SEARCH_SPACE"} archived={chat.archived} dropdownOpen={openDropdownChatId === chat.id} onDropdownOpenChange={(open) => setOpenDropdownChatId(open ? chat.id : null)} onClick={() => onChatSelect(chat)} + onPrefetch={() => onChatPrefetch?.(chat)} onRename={() => onChatRename?.(chat)} onArchive={() => onChatArchive?.(chat)} onDelete={() => onChatDelete?.(chat)} diff --git a/surfsense_web/components/layout/ui/sidebar/SidebarSection.tsx b/surfsense_web/components/layout/ui/sidebar/SidebarSection.tsx index 99162dddf..c041dec86 100644 --- a/surfsense_web/components/layout/ui/sidebar/SidebarSection.tsx +++ b/surfsense_web/components/layout/ui/sidebar/SidebarSection.tsx @@ -39,12 +39,12 @@ export function SidebarSection({ className )} > -
    +
    {title} diff --git a/surfsense_web/components/layout/ui/sidebar/SidebarUserProfile.tsx b/surfsense_web/components/layout/ui/sidebar/SidebarUserProfile.tsx index bc3b36efd..ea93ce4d0 100644 --- a/surfsense_web/components/layout/ui/sidebar/SidebarUserProfile.tsx +++ b/surfsense_web/components/layout/ui/sidebar/SidebarUserProfile.tsx @@ -139,14 +139,14 @@ export function SidebarUserProfile({ const { locale, setLocale } = useLocaleContext(); const { isDesktop } = usePlatform(); const isDesktopViewport = useMediaQuery("(min-width: 768px)"); - const { os, primary } = usePrimaryDownload(); + const { os, primary, isMobileOS } = usePrimaryDownload(); const [isLoggingOut, setIsLoggingOut] = useState(false); const bgColor = getUserAvatarColor(user.email); const initials = getUserInitials(user.email); const displayName = user.name || user.email.split("@")[0]; const downloadUrl = primary?.url ?? GITHUB_RELEASES_URL; const downloadLabel = t("download_for_os", { os }); - const showDownloadCta = !isDesktop && isDesktopViewport; + const showDownloadCta = !isDesktop && !isMobileOS && isDesktopViewport; const handleLanguageChange = (newLocale: "en" | "es" | "pt" | "hi" | "zh") => { setLocale(newLocale); @@ -221,18 +221,15 @@ export function SidebarUserProfile({ - -
    - -
    -

    {displayName}

    -

    {user.email}

    -
    + +
    + {/*

    {displayName}

    */} +

    + {user.email} +

    - - {t("user_settings")} @@ -327,14 +324,14 @@ export function SidebarUserProfile({ ))} -

    +

    v{APP_VERSION}

    - {!isDesktop && ( + {!isDesktop && !isMobileOS && ( @@ -406,18 +403,15 @@ export function SidebarUserProfile({ - -
    - -
    -

    {displayName}

    -

    {user.email}

    -
    + +
    +

    {displayName}

    +

    + {user.email} +

    - - {t("user_settings")} @@ -512,7 +506,7 @@ export function SidebarUserProfile({ ))} -

    +

    v{APP_VERSION}

    diff --git a/surfsense_web/components/layout/ui/sidebar/index.ts b/surfsense_web/components/layout/ui/sidebar/index.ts index d72f86c8a..e25149b06 100644 --- a/surfsense_web/components/layout/ui/sidebar/index.ts +++ b/surfsense_web/components/layout/ui/sidebar/index.ts @@ -1,5 +1,4 @@ -export { AllPrivateChatsSidebar, AllPrivateChatsSidebarContent } from "./AllPrivateChatsSidebar"; -export { AllSharedChatsSidebar, AllSharedChatsSidebarContent } from "./AllSharedChatsSidebar"; +export { AllChatsSidebar, AllChatsSidebarContent } from "./AllChatsSidebar"; export { ChatListItem } from "./ChatListItem"; export { DocumentsSidebar } from "./DocumentsSidebar"; export { InboxSidebar, InboxSidebarContent } from "./InboxSidebar"; diff --git a/surfsense_web/components/layout/ui/tabs/TabBar.tsx b/surfsense_web/components/layout/ui/tabs/TabBar.tsx index c5fb91f4d..869c9cee2 100644 --- a/surfsense_web/components/layout/ui/tabs/TabBar.tsx +++ b/surfsense_web/components/layout/ui/tabs/TabBar.tsx @@ -15,6 +15,7 @@ import { cn } from "@/lib/utils"; interface TabBarProps { onTabSwitch?: (tab: Tab) => void; + onTabPrefetch?: (tab: Tab) => void; onNewChat?: () => void; leftActions?: React.ReactNode; rightActions?: React.ReactNode; @@ -36,6 +37,7 @@ function nextTabListScrollLeft(input: { export function TabBar({ onTabSwitch, + onTabPrefetch, onNewChat, leftActions, rightActions, @@ -71,6 +73,15 @@ export function TabBar({ [activeTabId, switchTab, onTabSwitch] ); + const handleTabPrefetch = useCallback( + (tab: Tab) => { + if (tab.type === "chat") { + onTabPrefetch?.(tab); + } + }, + [onTabPrefetch] + ); + const handleTabClose = useCallback( (e: React.MouseEvent, tabId: string) => { e.stopPropagation(); @@ -195,7 +206,11 @@ export function TabBar({ type="button" variant="ghost" onClick={() => handleTabClick(tab)} - onMouseEnter={() => setHoveredTabIndex(index)} + onMouseEnter={() => { + setHoveredTabIndex(index); + handleTabPrefetch(tab); + }} + onFocus={() => handleTabPrefetch(tab)} onMouseLeave={() => setHoveredTabIndex(null)} className={cn( "h-full w-full justify-start overflow-hidden px-3 text-left text-[13px] font-medium transition-colors duration-150", diff --git a/surfsense_web/components/new-chat/chat-example-prompts.tsx b/surfsense_web/components/new-chat/chat-example-prompts.tsx index 95d7a0eaa..f969aa61b 100644 --- a/surfsense_web/components/new-chat/chat-example-prompts.tsx +++ b/surfsense_web/components/new-chat/chat-example-prompts.tsx @@ -1,10 +1,17 @@ "use client"; -import { CornerDownLeft, Lightbulb } from "lucide-react"; -import { memo, useCallback } from "react"; +import { + FilePlus2, + Search, + Settings2, + type LucideIcon, + WandSparkles, + Workflow, + X, +} from "lucide-react"; +import { memo, useCallback, useState } from "react"; import { Button } from "@/components/ui/button"; import { ScrollArea } from "@/components/ui/scroll-area"; -import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { CHAT_EXAMPLE_CATEGORIES } from "@/lib/chat/example-prompts"; interface ChatExamplePromptsProps { @@ -12,6 +19,13 @@ interface ChatExamplePromptsProps { onSelect: (prompt: string) => void; } +const CATEGORY_ICONS: Record = { + search: Search, + create: FilePlus2, + automate: Workflow, + tools: Settings2, +}; + const ExamplePromptButton = memo(function ExamplePromptButton({ prompt, onSelect, @@ -26,50 +40,72 @@ const ExamplePromptButton = memo(function ExamplePromptButton({ type="button" variant="ghost" onClick={handleClick} - className="h-auto w-full items-start justify-start gap-2.5 whitespace-normal rounded-md border bg-background px-3 py-2 text-left font-normal text-muted-foreground hover:bg-accent hover:text-accent-foreground" + className="h-auto w-full items-start justify-start whitespace-normal rounded-lg bg-transparent px-2.5 py-1.5 text-left font-normal text-muted-foreground shadow-none hover:bg-foreground/10 hover:text-foreground sm:rounded-xl sm:px-3 sm:py-2" > -