From 8d50f90060f8e53c4a5f2ddda88bed2198981938 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Mon, 27 Apr 2026 14:04:50 -0700 Subject: [PATCH 1/8] chore: linting --- .../app/agents/new_chat/chat_deepagent.py | 12 +- .../agents/new_chat/middleware/__init__.py | 6 +- .../agents/new_chat/middleware/file_intent.py | 9 +- .../agents/new_chat/middleware/filesystem.py | 36 +++- .../new_chat/middleware/knowledge_search.py | 2 +- .../middleware/local_folder_backend.py | 44 +++-- .../multi_root_local_folder_backend.py | 28 +++- .../new_chat/tools/connected_accounts.py | 12 +- .../agents/new_chat/tools/discord/_auth.py | 3 +- .../new_chat/tools/discord/list_channels.py | 32 +++- .../new_chat/tools/discord/read_messages.py | 32 +++- .../new_chat/tools/discord/send_message.py | 35 +++- .../agents/new_chat/tools/gmail/read_email.py | 21 ++- .../new_chat/tools/gmail/search_emails.py | 45 +++-- .../tools/google_calendar/search_events.py | 54 ++++-- .../app/agents/new_chat/tools/hitl.py | 4 +- .../app/agents/new_chat/tools/luma/_auth.py | 3 +- .../new_chat/tools/luma/create_event.py | 21 ++- .../agents/new_chat/tools/luma/list_events.py | 37 ++-- .../agents/new_chat/tools/luma/read_event.py | 16 +- .../app/agents/new_chat/tools/mcp_client.py | 6 +- .../app/agents/new_chat/tools/mcp_tool.py | 158 +++++++++++------- .../app/agents/new_chat/tools/registry.py | 13 +- .../app/agents/new_chat/tools/teams/_auth.py | 3 +- .../new_chat/tools/teams/list_channels.py | 33 +++- .../new_chat/tools/teams/read_messages.py | 32 ++-- .../new_chat/tools/teams/send_message.py | 24 ++- .../agents/new_chat/tools/tool_response.py | 5 +- .../app/connectors/exceptions.py | 1 - surfsense_backend/app/routes/__init__.py | 4 +- .../app/routes/mcp_oauth_route.py | 130 ++++++++++---- .../app/routes/new_chat_routes.py | 2 +- .../app/routes/oauth_connector_base.py | 29 ++-- .../routes/search_source_connectors_routes.py | 4 +- .../app/services/mcp_oauth/discovery.py | 4 +- .../app/services/mcp_oauth/registry.py | 62 ++++--- .../app/services/obsidian_plugin_indexer.py | 9 +- .../app/tasks/chat/stream_new_chat.py | 41 ++--- surfsense_backend/app/utils/async_retry.py | 9 +- .../app/utils/connector_naming.py | 5 +- .../test_obsidian_plugin_routes.py | 16 +- .../middleware/test_file_intent_middleware.py | 10 +- .../test_filesystem_verification.py | 4 +- .../unit/test_obsidian_plugin_indexer.py | 7 +- .../unit/test_stream_new_chat_contract.py | 1 - .../new-chat/[[...chat_id]]/page.tsx | 23 +-- .../components/DesktopShortcutsContent.tsx | 50 +++--- surfsense_web/app/desktop/login/page.tsx | 11 +- .../assistant-ui/connector-popup.tsx | 50 +++--- .../components/mcp-connect-form.tsx | 14 +- .../components/mcp-config.tsx | 14 +- .../components/teams-config.tsx | 6 +- .../views/connector-edit-view.tsx | 12 +- .../views/indexing-configuration-view.tsx | 5 +- .../tabs/active-connectors-tab.tsx | 6 +- .../views/connector-accounts-list-view.tsx | 129 +++++++------- .../components/assistant-ui/markdown-text.tsx | 7 +- .../components/editor-panel/editor-panel.tsx | 141 +++++++++------- .../editor/plugins/fixed-toolbar-kit.tsx | 3 +- .../components/editor/source-code-editor.tsx | 2 +- .../layout/ui/right-panel/RightPanel.tsx | 8 +- .../ui/sidebar/DesktopLocalTabContent.tsx | 6 +- .../layout/ui/sidebar/DocumentsSidebar.tsx | 62 ++++--- .../ui/sidebar/LocalFilesystemBrowser.tsx | 109 ++++++------ .../layout/ui/tabs/DocumentTabContent.tsx | 4 +- .../components/new-chat/model-selector.tsx | 10 +- .../components/report-panel/report-panel.tsx | 3 +- .../settings/agent-model-manager.tsx | 10 +- .../components/settings/roles-manager.tsx | 32 +++- .../settings/user-settings-dialog.tsx | 17 +- .../tool-ui/generic-hitl-approval.tsx | 4 +- .../tool-ui/google-calendar/create-event.tsx | 9 +- surfsense_web/contracts/enums/toolIcons.tsx | 2 +- surfsense_web/types/window.d.ts | 15 +- 74 files changed, 1135 insertions(+), 693 deletions(-) diff --git a/surfsense_backend/app/agents/new_chat/chat_deepagent.py b/surfsense_backend/app/agents/new_chat/chat_deepagent.py index 73a39ccbf..ddf87cf2a 100644 --- a/surfsense_backend/app/agents/new_chat/chat_deepagent.py +++ b/surfsense_backend/app/agents/new_chat/chat_deepagent.py @@ -50,7 +50,10 @@ from app.agents.new_chat.system_prompt import ( build_configurable_system_prompt, build_surfsense_system_prompt, ) -from app.agents.new_chat.tools.registry import build_tools_async, get_connector_gated_tools +from app.agents.new_chat.tools.registry import ( + build_tools_async, + get_connector_gated_tools, +) from app.db import ChatVisibility from app.services.connector_service import ConnectorService from app.utils.perf import get_perf_logger @@ -294,9 +297,7 @@ async def create_surfsense_deep_agent( } modified_disabled_tools = list(disabled_tools) if disabled_tools else [] - modified_disabled_tools.extend( - get_connector_gated_tools(available_connectors) - ) + modified_disabled_tools.extend(get_connector_gated_tools(available_connectors)) # Remove direct KB search tool; we now pre-seed a scoped filesystem via middleware. if "search_knowledge_base" not in modified_disabled_tools: @@ -328,7 +329,8 @@ async def create_surfsense_deep_agent( meta = getattr(t, "metadata", None) or {} if meta.get("mcp_is_generic") and meta.get("mcp_connector_name"): _mcp_connector_tools.setdefault( - meta["mcp_connector_name"], [], + meta["mcp_connector_name"], + [], ).append(t.name) if _mcp_connector_tools: diff --git a/surfsense_backend/app/agents/new_chat/middleware/__init__.py b/surfsense_backend/app/agents/new_chat/middleware/__init__.py index 5a24b2f9e..6e4542e1a 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/__init__.py +++ b/surfsense_backend/app/agents/new_chat/middleware/__init__.py @@ -3,12 +3,12 @@ from app.agents.new_chat.middleware.dedup_tool_calls import ( DedupHITLToolCallsMiddleware, ) -from app.agents.new_chat.middleware.filesystem import ( - SurfSenseFilesystemMiddleware, -) from app.agents.new_chat.middleware.file_intent import ( FileIntentMiddleware, ) +from app.agents.new_chat.middleware.filesystem import ( + SurfSenseFilesystemMiddleware, +) from app.agents.new_chat.middleware.knowledge_search import ( KnowledgeBaseSearchMiddleware, ) diff --git a/surfsense_backend/app/agents/new_chat/middleware/file_intent.py b/surfsense_backend/app/agents/new_chat/middleware/file_intent.py index 4bf5dcfe4..05cb230ce 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/file_intent.py +++ b/surfsense_backend/app/agents/new_chat/middleware/file_intent.py @@ -213,7 +213,9 @@ def _build_classifier_prompt(*, recent_conversation: str, user_text: str) -> str ) -def _build_recent_conversation(messages: list[BaseMessage], *, max_messages: int = 6) -> str: +def _build_recent_conversation( + messages: list[BaseMessage], *, max_messages: int = 6 +) -> str: rows: list[str] = [] for msg in messages[-max_messages:]: role = "user" if isinstance(msg, HumanMessage) else "assistant" @@ -246,7 +248,9 @@ class FileIntentMiddleware(AgentMiddleware): # type: ignore[type-arg] [HumanMessage(content=prompt)], config={"tags": ["surfsense:internal"]}, ) - payload = json.loads(_extract_json_payload(_extract_text_from_message(response))) + payload = json.loads( + _extract_json_payload(_extract_text_from_message(response)) + ) plan = FileIntentPlan.model_validate(payload) return plan except (json.JSONDecodeError, ValidationError, ValueError) as exc: @@ -317,4 +321,3 @@ class FileIntentMiddleware(AgentMiddleware): # type: ignore[type-arg] insert_at = max(len(new_messages) - 1, 0) new_messages.insert(insert_at, contract_msg) return {"messages": new_messages, "file_operation_contract": contract} - diff --git a/surfsense_backend/app/agents/new_chat/middleware/filesystem.py b/surfsense_backend/app/agents/new_chat/middleware/filesystem.py index 8dfa89ef2..cb50693f1 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/filesystem.py +++ b/surfsense_backend/app/agents/new_chat/middleware/filesystem.py @@ -877,7 +877,9 @@ class SurfSenseFilesystemMiddleware(FilesystemMiddleware): suggested_path = contract.get("suggested_path") if isinstance(suggested_path, str) and suggested_path.strip(): normalized_suggested = self._normalize_absolute_path(suggested_path) - suggested_mount = self._extract_mount_from_path(normalized_suggested, mounts) + suggested_mount = self._extract_mount_from_path( + normalized_suggested, mounts + ) matching_mounts = [ mount @@ -1071,14 +1073,18 @@ class SurfSenseFilesystemMiddleware(FilesystemMiddleware): ] = False, ) -> Command | str: if self._filesystem_mode != FilesystemMode.DESKTOP_LOCAL_FOLDER: - return "Error: move_file is only available in desktop local-folder mode." + return ( + "Error: move_file is only available in desktop local-folder mode." + ) if not source_path.strip() or not destination_path.strip(): return "Error: source_path and destination_path are required." resolved_backend = self._get_backend(runtime) source_target = self._resolve_move_target_path(source_path, runtime) - destination_target = self._resolve_move_target_path(destination_path, runtime) + destination_target = self._resolve_move_target_path( + destination_path, runtime + ) try: validated_source = validate_path(source_target) validated_destination = validate_path(destination_target) @@ -1106,7 +1112,9 @@ class SurfSenseFilesystemMiddleware(FilesystemMiddleware): ], } ) - return f"Moved '{validated_source}' to '{res.path or validated_destination}'" + return ( + f"Moved '{validated_source}' to '{res.path or validated_destination}'" + ) async def async_move_file( source_path: Annotated[ @@ -1125,14 +1133,18 @@ class SurfSenseFilesystemMiddleware(FilesystemMiddleware): ] = False, ) -> Command | str: if self._filesystem_mode != FilesystemMode.DESKTOP_LOCAL_FOLDER: - return "Error: move_file is only available in desktop local-folder mode." + return ( + "Error: move_file is only available in desktop local-folder mode." + ) if not source_path.strip() or not destination_path.strip(): return "Error: source_path and destination_path are required." resolved_backend = self._get_backend(runtime) source_target = self._resolve_move_target_path(source_path, runtime) - destination_target = self._resolve_move_target_path(destination_path, runtime) + destination_target = self._resolve_move_target_path( + destination_path, runtime + ) try: validated_source = validate_path(source_target) validated_destination = validate_path(destination_target) @@ -1160,7 +1172,9 @@ class SurfSenseFilesystemMiddleware(FilesystemMiddleware): ], } ) - return f"Moved '{validated_source}' to '{res.path or validated_destination}'" + return ( + f"Moved '{validated_source}' to '{res.path or validated_destination}'" + ) return StructuredTool.from_function( name="move_file", @@ -1201,7 +1215,9 @@ class SurfSenseFilesystemMiddleware(FilesystemMiddleware): ] = True, ) -> str: if self._filesystem_mode != FilesystemMode.DESKTOP_LOCAL_FOLDER: - return "Error: list_tree is only available in desktop local-folder mode." + return ( + "Error: list_tree is only available in desktop local-folder mode." + ) if max_depth < 0: return "Error: max_depth must be >= 0." if page_size < 1: @@ -1253,7 +1269,9 @@ class SurfSenseFilesystemMiddleware(FilesystemMiddleware): ] = True, ) -> str: if self._filesystem_mode != FilesystemMode.DESKTOP_LOCAL_FOLDER: - return "Error: list_tree is only available in desktop local-folder mode." + return ( + "Error: list_tree is only available in desktop local-folder mode." + ) if max_depth < 0: return "Error: max_depth must be >= 0." if page_size < 1: diff --git a/surfsense_backend/app/agents/new_chat/middleware/knowledge_search.py b/surfsense_backend/app/agents/new_chat/middleware/knowledge_search.py index 51378a013..6df317aaa 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/knowledge_search.py +++ b/surfsense_backend/app/agents/new_chat/middleware/knowledge_search.py @@ -27,8 +27,8 @@ from pydantic import BaseModel, Field, ValidationError from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession -from app.agents.new_chat.utils import parse_date_or_datetime, resolve_date_range from app.agents.new_chat.filesystem_selection import FilesystemMode +from app.agents.new_chat.utils import parse_date_or_datetime, resolve_date_range from app.db import ( NATIVE_TO_LEGACY_DOCTYPE, Chunk, diff --git a/surfsense_backend/app/agents/new_chat/middleware/local_folder_backend.py b/surfsense_backend/app/agents/new_chat/middleware/local_folder_backend.py index 0cee3e007..565fcb48b 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/local_folder_backend.py +++ b/surfsense_backend/app/agents/new_chat/middleware/local_folder_backend.py @@ -120,7 +120,9 @@ class LocalFolderBackend: if not target.exists() or not target.is_dir(): return [] infos: list[FileInfo] = [] - for child in sorted(target.iterdir(), key=lambda p: (not p.is_dir(), p.name.lower())): + for child in sorted( + target.iterdir(), key=lambda p: (not p.is_dir(), p.name.lower()) + ): infos.append( FileInfo( path=self._to_virtual(child, self._root), @@ -317,7 +319,9 @@ class LocalFolderBackend: return WriteResult(error="Error: source and destination paths are the same") with self._acquire_path_locks(source_path, destination_path): if not source.exists(): - return WriteResult(error=f"Error: source path '{source_path}' not found") + return WriteResult( + error=f"Error: source path '{source_path}' not found" + ) if destination.exists(): if not overwrite: return WriteResult( @@ -339,8 +343,12 @@ class LocalFolderBackend: else: source.rename(destination) except OSError as exc: - return WriteResult(error=f"Error: failed to move '{source_path}': {exc}") - return WriteResult(path=self._to_virtual(destination, self._root), files_update=None) + return WriteResult( + error=f"Error: failed to move '{source_path}': {exc}" + ) + return WriteResult( + path=self._to_virtual(destination, self._root), files_update=None + ) async def amove( self, @@ -368,12 +376,16 @@ class LocalFolderBackend: if not path.exists() or not path.is_file(): return EditResult(error=f"Error: File '{file_path}' not found") content = path.read_text(encoding="utf-8", errors="replace") - result = perform_string_replacement(content, old_string, new_string, replace_all) + result = perform_string_replacement( + content, old_string, new_string, replace_all + ) if isinstance(result, str): return EditResult(error=result) updated_content, occurrences = result self._write_text_atomic(path, updated_content) - return EditResult(path=file_path, files_update=None, occurrences=int(occurrences)) + return EditResult( + path=file_path, files_update=None, occurrences=int(occurrences) + ) async def aedit( self, @@ -447,7 +459,9 @@ class LocalFolderBackend: matches: list[GrepMatch] = [] for file_path in self._iter_candidate_files(path, glob): try: - lines = file_path.read_text(encoding="utf-8", errors="replace").splitlines() + lines = file_path.read_text( + encoding="utf-8", errors="replace" + ).splitlines() except Exception: continue for idx, line in enumerate(lines, start=1): @@ -481,12 +495,18 @@ class LocalFolderBackend: FileUploadResponse(path=virtual_path, error=_FILE_NOT_FOUND) ) except IsADirectoryError: - responses.append(FileUploadResponse(path=virtual_path, error=_IS_DIRECTORY)) + responses.append( + FileUploadResponse(path=virtual_path, error=_IS_DIRECTORY) + ) except Exception: - responses.append(FileUploadResponse(path=virtual_path, error=_INVALID_PATH)) + responses.append( + FileUploadResponse(path=virtual_path, error=_INVALID_PATH) + ) return responses - async def aupload_files(self, files: list[tuple[str, bytes]]) -> list[FileUploadResponse]: + async def aupload_files( + self, files: list[tuple[str, bytes]] + ) -> list[FileUploadResponse]: return await asyncio.to_thread(self.upload_files, files) def download_files(self, paths: list[str]) -> list[FileDownloadResponse]: @@ -515,7 +535,9 @@ class LocalFolderBackend: ) except Exception: responses.append( - FileDownloadResponse(path=virtual_path, content=None, error=_INVALID_PATH) + FileDownloadResponse( + path=virtual_path, content=None, error=_INVALID_PATH + ) ) return responses diff --git a/surfsense_backend/app/agents/new_chat/middleware/multi_root_local_folder_backend.py b/surfsense_backend/app/agents/new_chat/middleware/multi_root_local_folder_backend.py index 82914f9ce..93eabe6ff 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/multi_root_local_folder_backend.py +++ b/surfsense_backend/app/agents/new_chat/middleware/multi_root_local_folder_backend.py @@ -127,7 +127,9 @@ class MultiRootLocalFolderBackend: mount, local_path = self._split_mount_path(path) except ValueError: return [] - return self._transform_infos(mount, self._mount_to_backend[mount].ls_info(local_path)) + return self._transform_infos( + mount, self._mount_to_backend[mount].ls_info(local_path) + ) async def als_info(self, path: str) -> list[FileInfo]: return await asyncio.to_thread(self.ls_info, path) @@ -355,7 +357,9 @@ class MultiRootLocalFolderBackend: all_matches.extend( [ GrepMatch( - path=self._prefix_mount_path(mount, self._get_str(match, "path")), + path=self._prefix_mount_path( + mount, self._get_str(match, "path") + ), line=self._get_int(match, "line"), text=self._get_str(match, "text"), ) @@ -394,7 +398,9 @@ class MultiRootLocalFolderBackend: try: mount, local_path = self._split_mount_path(virtual_path) except ValueError: - invalid.append(FileUploadResponse(path=virtual_path, error=_INVALID_PATH)) + invalid.append( + FileUploadResponse(path=virtual_path, error=_INVALID_PATH) + ) continue grouped.setdefault(mount, []).append((local_path, content)) @@ -404,7 +410,9 @@ class MultiRootLocalFolderBackend: responses.extend( [ FileUploadResponse( - path=self._prefix_mount_path(mount, self._get_str(item, "path")), + path=self._prefix_mount_path( + mount, self._get_str(item, "path") + ), error=self._get_str(item, "error") or None, ) for item in result @@ -412,7 +420,9 @@ class MultiRootLocalFolderBackend: ) return responses - async def aupload_files(self, files: list[tuple[str, bytes]]) -> list[FileUploadResponse]: + async def aupload_files( + self, files: list[tuple[str, bytes]] + ) -> list[FileUploadResponse]: return await asyncio.to_thread(self.upload_files, files) def download_files(self, paths: list[str]) -> list[FileDownloadResponse]: @@ -423,7 +433,9 @@ class MultiRootLocalFolderBackend: mount, local_path = self._split_mount_path(virtual_path) except ValueError: invalid.append( - FileDownloadResponse(path=virtual_path, content=None, error=_INVALID_PATH) + FileDownloadResponse( + path=virtual_path, content=None, error=_INVALID_PATH + ) ) continue grouped.setdefault(mount, []).append(local_path) @@ -434,7 +446,9 @@ class MultiRootLocalFolderBackend: responses.extend( [ FileDownloadResponse( - path=self._prefix_mount_path(mount, self._get_str(item, "path")), + path=self._prefix_mount_path( + mount, self._get_str(item, "path") + ), content=self._get_value(item, "content"), error=self._get_str(item, "error") or None, ) diff --git a/surfsense_backend/app/agents/new_chat/tools/connected_accounts.py b/surfsense_backend/app/agents/new_chat/tools/connected_accounts.py index e0b1978e1..5675a42e6 100644 --- a/surfsense_backend/app/agents/new_chat/tools/connected_accounts.py +++ b/surfsense_backend/app/agents/new_chat/tools/connected_accounts.py @@ -57,7 +57,11 @@ def create_get_connected_accounts_tool( async def _run(service: str) -> list[dict[str, Any]]: svc_cfg = MCP_SERVICES.get(service) if not svc_cfg: - return [{"error": f"Unknown service '{service}'. Valid: {', '.join(sorted(MCP_SERVICES.keys()))}"}] + return [ + { + "error": f"Unknown service '{service}'. Valid: {', '.join(sorted(MCP_SERVICES.keys()))}" + } + ] try: connector_type = SearchSourceConnectorType(svc_cfg.connector_type) @@ -74,7 +78,11 @@ def create_get_connected_accounts_tool( connectors = result.scalars().all() if not connectors: - return [{"error": f"No {svc_cfg.name} accounts connected. Ask the user to connect one in settings."}] + return [ + { + "error": f"No {svc_cfg.name} accounts connected. Ask the user to connect one in settings." + } + ] is_multi = len(connectors) > 1 diff --git a/surfsense_backend/app/agents/new_chat/tools/discord/_auth.py b/surfsense_backend/app/agents/new_chat/tools/discord/_auth.py index 1f51e3660..c345f8a5e 100644 --- a/surfsense_backend/app/agents/new_chat/tools/discord/_auth.py +++ b/surfsense_backend/app/agents/new_chat/tools/discord/_auth.py @@ -19,7 +19,8 @@ async def get_discord_connector( select(SearchSourceConnector).filter( SearchSourceConnector.search_space_id == search_space_id, SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type == SearchSourceConnectorType.DISCORD_CONNECTOR, + SearchSourceConnector.connector_type + == SearchSourceConnectorType.DISCORD_CONNECTOR, ) ) return result.scalars().first() diff --git a/surfsense_backend/app/agents/new_chat/tools/discord/list_channels.py b/surfsense_backend/app/agents/new_chat/tools/discord/list_channels.py index a33b88aa0..3cc99ac17 100644 --- a/surfsense_backend/app/agents/new_chat/tools/discord/list_channels.py +++ b/surfsense_backend/app/agents/new_chat/tools/discord/list_channels.py @@ -23,16 +23,24 @@ def create_list_discord_channels_tool( Dictionary with status and a list of channels (id, name). """ if db_session is None or search_space_id is None or user_id is None: - return {"status": "error", "message": "Discord tool not properly configured."} + return { + "status": "error", + "message": "Discord tool not properly configured.", + } try: - connector = await get_discord_connector(db_session, search_space_id, user_id) + connector = await get_discord_connector( + db_session, search_space_id, user_id + ) if not connector: return {"status": "error", "message": "No Discord connector found."} guild_id = get_guild_id(connector) if not guild_id: - return {"status": "error", "message": "No guild ID in Discord connector config."} + return { + "status": "error", + "message": "No guild ID in Discord connector config.", + } token = get_bot_token(connector) @@ -44,9 +52,16 @@ def create_list_discord_channels_tool( ) if resp.status_code == 401: - return {"status": "auth_error", "message": "Discord bot token is invalid.", "connector_type": "discord"} + return { + "status": "auth_error", + "message": "Discord bot token is invalid.", + "connector_type": "discord", + } if resp.status_code != 200: - return {"status": "error", "message": f"Discord API error: {resp.status_code}"} + return { + "status": "error", + "message": f"Discord API error: {resp.status_code}", + } # Type 0 = text channel channels = [ @@ -54,7 +69,12 @@ def create_list_discord_channels_tool( for ch in resp.json() if ch.get("type") == 0 ] - return {"status": "success", "guild_id": guild_id, "channels": channels, "total": len(channels)} + return { + "status": "success", + "guild_id": guild_id, + "channels": channels, + "total": len(channels), + } except Exception as e: from langgraph.errors import GraphInterrupt diff --git a/surfsense_backend/app/agents/new_chat/tools/discord/read_messages.py b/surfsense_backend/app/agents/new_chat/tools/discord/read_messages.py index 852a9297b..d8bf989a1 100644 --- a/surfsense_backend/app/agents/new_chat/tools/discord/read_messages.py +++ b/surfsense_backend/app/agents/new_chat/tools/discord/read_messages.py @@ -31,12 +31,17 @@ def create_read_discord_messages_tool( id, author, content, timestamp. """ if db_session is None or search_space_id is None or user_id is None: - return {"status": "error", "message": "Discord tool not properly configured."} + return { + "status": "error", + "message": "Discord tool not properly configured.", + } limit = min(limit, 50) try: - connector = await get_discord_connector(db_session, search_space_id, user_id) + connector = await get_discord_connector( + db_session, search_space_id, user_id + ) if not connector: return {"status": "error", "message": "No Discord connector found."} @@ -51,11 +56,21 @@ def create_read_discord_messages_tool( ) if resp.status_code == 401: - return {"status": "auth_error", "message": "Discord bot token is invalid.", "connector_type": "discord"} + return { + "status": "auth_error", + "message": "Discord bot token is invalid.", + "connector_type": "discord", + } if resp.status_code == 403: - return {"status": "error", "message": "Bot lacks permission to read this channel."} + return { + "status": "error", + "message": "Bot lacks permission to read this channel.", + } if resp.status_code != 200: - return {"status": "error", "message": f"Discord API error: {resp.status_code}"} + return { + "status": "error", + "message": f"Discord API error: {resp.status_code}", + } messages = [ { @@ -67,7 +82,12 @@ def create_read_discord_messages_tool( for m in resp.json() ] - return {"status": "success", "channel_id": channel_id, "messages": messages, "total": len(messages)} + return { + "status": "success", + "channel_id": channel_id, + "messages": messages, + "total": len(messages), + } except Exception as e: from langgraph.errors import GraphInterrupt diff --git a/surfsense_backend/app/agents/new_chat/tools/discord/send_message.py b/surfsense_backend/app/agents/new_chat/tools/discord/send_message.py index be4e6fdb2..236cd017a 100644 --- a/surfsense_backend/app/agents/new_chat/tools/discord/send_message.py +++ b/surfsense_backend/app/agents/new_chat/tools/discord/send_message.py @@ -35,13 +35,21 @@ def create_send_discord_message_tool( - If status is "rejected", the user explicitly declined. Do NOT retry. """ if db_session is None or search_space_id is None or user_id is None: - return {"status": "error", "message": "Discord tool not properly configured."} + return { + "status": "error", + "message": "Discord tool not properly configured.", + } if len(content) > 2000: - return {"status": "error", "message": "Message exceeds Discord's 2000-character limit."} + return { + "status": "error", + "message": "Message exceeds Discord's 2000-character limit.", + } try: - connector = await get_discord_connector(db_session, search_space_id, user_id) + connector = await get_discord_connector( + db_session, search_space_id, user_id + ) if not connector: return {"status": "error", "message": "No Discord connector found."} @@ -53,7 +61,10 @@ def create_send_discord_message_tool( ) if result.rejected: - return {"status": "rejected", "message": "User declined. Message was not sent."} + return { + "status": "rejected", + "message": "User declined. Message was not sent.", + } final_content = result.params.get("content", content) final_channel = result.params.get("channel_id", channel_id) @@ -72,11 +83,21 @@ def create_send_discord_message_tool( ) if resp.status_code == 401: - return {"status": "auth_error", "message": "Discord bot token is invalid.", "connector_type": "discord"} + return { + "status": "auth_error", + "message": "Discord bot token is invalid.", + "connector_type": "discord", + } if resp.status_code == 403: - return {"status": "error", "message": "Bot lacks permission to send messages in this channel."} + return { + "status": "error", + "message": "Bot lacks permission to send messages in this channel.", + } if resp.status_code not in (200, 201): - return {"status": "error", "message": f"Discord API error: {resp.status_code}"} + return { + "status": "error", + "message": f"Discord API error: {resp.status_code}", + } msg_data = resp.json() return { diff --git a/surfsense_backend/app/agents/new_chat/tools/gmail/read_email.py b/surfsense_backend/app/agents/new_chat/tools/gmail/read_email.py index 9071f129a..deec1627c 100644 --- a/surfsense_backend/app/agents/new_chat/tools/gmail/read_email.py +++ b/surfsense_backend/app/agents/new_chat/tools/gmail/read_email.py @@ -65,12 +65,22 @@ def create_read_gmail_email_tool( detail, error = await gmail.get_message_details(message_id) if error: - if "re-authenticate" in error.lower() or "authentication failed" in error.lower(): - return {"status": "auth_error", "message": error, "connector_type": "gmail"} + if ( + "re-authenticate" in error.lower() + or "authentication failed" in error.lower() + ): + return { + "status": "auth_error", + "message": error, + "connector_type": "gmail", + } return {"status": "error", "message": error} if not detail: - return {"status": "not_found", "message": f"Email with ID '{message_id}' not found."} + return { + "status": "not_found", + "message": f"Email with ID '{message_id}' not found.", + } content = gmail.format_message_to_markdown(detail) @@ -82,6 +92,9 @@ def create_read_gmail_email_tool( if isinstance(e, GraphInterrupt): raise logger.error("Error reading Gmail email: %s", e, exc_info=True) - return {"status": "error", "message": "Failed to read email. Please try again."} + return { + "status": "error", + "message": "Failed to read email. Please try again.", + } return read_gmail_email diff --git a/surfsense_backend/app/agents/new_chat/tools/gmail/search_emails.py b/surfsense_backend/app/agents/new_chat/tools/gmail/search_emails.py index de43f03d0..2e363609e 100644 --- a/surfsense_backend/app/agents/new_chat/tools/gmail/search_emails.py +++ b/surfsense_backend/app/agents/new_chat/tools/gmail/search_emails.py @@ -125,12 +125,24 @@ def create_search_gmail_tool( max_results=max_results, query=query ) if error: - if "re-authenticate" in error.lower() or "authentication failed" in error.lower(): - return {"status": "auth_error", "message": error, "connector_type": "gmail"} + if ( + "re-authenticate" in error.lower() + or "authentication failed" in error.lower() + ): + return { + "status": "auth_error", + "message": error, + "connector_type": "gmail", + } return {"status": "error", "message": error} if not messages_list: - return {"status": "success", "emails": [], "total": 0, "message": "No emails found."} + return { + "status": "success", + "emails": [], + "total": 0, + "message": "No emails found.", + } emails = [] for msg in messages_list: @@ -141,16 +153,18 @@ def create_search_gmail_tool( h["name"].lower(): h["value"] for h in detail.get("payload", {}).get("headers", []) } - emails.append({ - "message_id": detail.get("id"), - "thread_id": detail.get("threadId"), - "subject": headers.get("subject", "No Subject"), - "from": headers.get("from", "Unknown"), - "to": headers.get("to", ""), - "date": headers.get("date", ""), - "snippet": detail.get("snippet", ""), - "labels": detail.get("labelIds", []), - }) + emails.append( + { + "message_id": detail.get("id"), + "thread_id": detail.get("threadId"), + "subject": headers.get("subject", "No Subject"), + "from": headers.get("from", "Unknown"), + "to": headers.get("to", ""), + "date": headers.get("date", ""), + "snippet": detail.get("snippet", ""), + "labels": detail.get("labelIds", []), + } + ) return {"status": "success", "emails": emails, "total": len(emails)} @@ -160,6 +174,9 @@ def create_search_gmail_tool( if isinstance(e, GraphInterrupt): raise logger.error("Error searching Gmail: %s", e, exc_info=True) - return {"status": "error", "message": "Failed to search Gmail. Please try again."} + return { + "status": "error", + "message": "Failed to search Gmail. Please try again.", + } return search_gmail diff --git a/surfsense_backend/app/agents/new_chat/tools/google_calendar/search_events.py b/surfsense_backend/app/agents/new_chat/tools/google_calendar/search_events.py index a622b0efa..dc6adb822 100644 --- a/surfsense_backend/app/agents/new_chat/tools/google_calendar/search_events.py +++ b/surfsense_backend/app/agents/new_chat/tools/google_calendar/search_events.py @@ -39,7 +39,10 @@ def create_search_calendar_events_tool( event_id, summary, start, end, location, attendees. """ if db_session is None or search_space_id is None or user_id is None: - return {"status": "error", "message": "Calendar tool not properly configured."} + return { + "status": "error", + "message": "Calendar tool not properly configured.", + } max_results = min(max_results, 50) @@ -76,10 +79,22 @@ def create_search_calendar_events_tool( ) if error: - if "re-authenticate" in error.lower() or "authentication failed" in error.lower(): - return {"status": "auth_error", "message": error, "connector_type": "google_calendar"} + if ( + "re-authenticate" in error.lower() + or "authentication failed" in error.lower() + ): + return { + "status": "auth_error", + "message": error, + "connector_type": "google_calendar", + } if "no events found" in error.lower(): - return {"status": "success", "events": [], "total": 0, "message": error} + return { + "status": "success", + "events": [], + "total": 0, + "message": error, + } return {"status": "error", "message": error} events = [] @@ -87,19 +102,19 @@ def create_search_calendar_events_tool( start = ev.get("start", {}) end = ev.get("end", {}) attendees_raw = ev.get("attendees", []) - events.append({ - "event_id": ev.get("id"), - "summary": ev.get("summary", "No Title"), - "start": start.get("dateTime") or start.get("date", ""), - "end": end.get("dateTime") or end.get("date", ""), - "location": ev.get("location", ""), - "description": ev.get("description", ""), - "html_link": ev.get("htmlLink", ""), - "attendees": [ - a.get("email", "") for a in attendees_raw[:10] - ], - "status": ev.get("status", ""), - }) + events.append( + { + "event_id": ev.get("id"), + "summary": ev.get("summary", "No Title"), + "start": start.get("dateTime") or start.get("date", ""), + "end": end.get("dateTime") or end.get("date", ""), + "location": ev.get("location", ""), + "description": ev.get("description", ""), + "html_link": ev.get("htmlLink", ""), + "attendees": [a.get("email", "") for a in attendees_raw[:10]], + "status": ev.get("status", ""), + } + ) return {"status": "success", "events": events, "total": len(events)} @@ -109,6 +124,9 @@ def create_search_calendar_events_tool( if isinstance(e, GraphInterrupt): raise logger.error("Error searching calendar events: %s", e, exc_info=True) - return {"status": "error", "message": "Failed to search calendar events. Please try again."} + return { + "status": "error", + "message": "Failed to search calendar events. Please try again.", + } return search_calendar_events diff --git a/surfsense_backend/app/agents/new_chat/tools/hitl.py b/surfsense_backend/app/agents/new_chat/tools/hitl.py index 89f02abf6..8480e57b1 100644 --- a/surfsense_backend/app/agents/new_chat/tools/hitl.py +++ b/surfsense_backend/app/agents/new_chat/tools/hitl.py @@ -130,7 +130,9 @@ def request_approval( try: decision_type, edited_params = _parse_decision(approval) except ValueError: - logger.warning("No approval decision received for %s — rejecting for safety", tool_name) + logger.warning( + "No approval decision received for %s — rejecting for safety", tool_name + ) return HITLResult(rejected=True, decision_type="error", params=params) logger.info("User decision for %s: %s", tool_name, decision_type) diff --git a/surfsense_backend/app/agents/new_chat/tools/luma/_auth.py b/surfsense_backend/app/agents/new_chat/tools/luma/_auth.py index 1d88161d6..37deb1525 100644 --- a/surfsense_backend/app/agents/new_chat/tools/luma/_auth.py +++ b/surfsense_backend/app/agents/new_chat/tools/luma/_auth.py @@ -17,7 +17,8 @@ async def get_luma_connector( select(SearchSourceConnector).filter( SearchSourceConnector.search_space_id == search_space_id, SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type == SearchSourceConnectorType.LUMA_CONNECTOR, + SearchSourceConnector.connector_type + == SearchSourceConnectorType.LUMA_CONNECTOR, ) ) return result.scalars().first() diff --git a/surfsense_backend/app/agents/new_chat/tools/luma/create_event.py b/surfsense_backend/app/agents/new_chat/tools/luma/create_event.py index 2217d29e6..0a24a988f 100644 --- a/surfsense_backend/app/agents/new_chat/tools/luma/create_event.py +++ b/surfsense_backend/app/agents/new_chat/tools/luma/create_event.py @@ -62,7 +62,10 @@ def create_create_luma_event_tool( ) if result.rejected: - return {"status": "rejected", "message": "User declined. Event was not created."} + return { + "status": "rejected", + "message": "User declined. Event was not created.", + } final_name = result.params.get("name", name) final_start = result.params.get("start_at", start_at) @@ -90,11 +93,21 @@ def create_create_luma_event_tool( ) if resp.status_code == 401: - return {"status": "auth_error", "message": "Luma API key is invalid.", "connector_type": "luma"} + return { + "status": "auth_error", + "message": "Luma API key is invalid.", + "connector_type": "luma", + } if resp.status_code == 403: - return {"status": "error", "message": "Luma Plus subscription required to create events via API."} + return { + "status": "error", + "message": "Luma Plus subscription required to create events via API.", + } if resp.status_code not in (200, 201): - return {"status": "error", "message": f"Luma API error: {resp.status_code} — {resp.text[:200]}"} + return { + "status": "error", + "message": f"Luma API error: {resp.status_code} — {resp.text[:200]}", + } data = resp.json() event_id = data.get("api_id") or data.get("event", {}).get("api_id") diff --git a/surfsense_backend/app/agents/new_chat/tools/luma/list_events.py b/surfsense_backend/app/agents/new_chat/tools/luma/list_events.py index cd4721758..aec5ad220 100644 --- a/surfsense_backend/app/agents/new_chat/tools/luma/list_events.py +++ b/surfsense_backend/app/agents/new_chat/tools/luma/list_events.py @@ -46,7 +46,9 @@ def create_list_luma_events_tool( async with httpx.AsyncClient(timeout=20.0) as client: while len(all_entries) < max_results: - params: dict[str, Any] = {"limit": min(100, max_results - len(all_entries))} + params: dict[str, Any] = { + "limit": min(100, max_results - len(all_entries)) + } if cursor: params["cursor"] = cursor @@ -57,9 +59,16 @@ def create_list_luma_events_tool( ) if resp.status_code == 401: - return {"status": "auth_error", "message": "Luma API key is invalid.", "connector_type": "luma"} + return { + "status": "auth_error", + "message": "Luma API key is invalid.", + "connector_type": "luma", + } if resp.status_code != 200: - return {"status": "error", "message": f"Luma API error: {resp.status_code}"} + return { + "status": "error", + "message": f"Luma API error: {resp.status_code}", + } data = resp.json() entries = data.get("entries", []) @@ -76,16 +85,18 @@ def create_list_luma_events_tool( for entry in all_entries[:max_results]: ev = entry.get("event", {}) geo = ev.get("geo_info", {}) - events.append({ - "event_id": entry.get("api_id"), - "name": ev.get("name", "Untitled"), - "start_at": ev.get("start_at", ""), - "end_at": ev.get("end_at", ""), - "timezone": ev.get("timezone", ""), - "location": geo.get("name", ""), - "url": ev.get("url", ""), - "visibility": ev.get("visibility", ""), - }) + events.append( + { + "event_id": entry.get("api_id"), + "name": ev.get("name", "Untitled"), + "start_at": ev.get("start_at", ""), + "end_at": ev.get("end_at", ""), + "timezone": ev.get("timezone", ""), + "location": geo.get("name", ""), + "url": ev.get("url", ""), + "visibility": ev.get("visibility", ""), + } + ) return {"status": "success", "events": events, "total": len(events)} diff --git a/surfsense_backend/app/agents/new_chat/tools/luma/read_event.py b/surfsense_backend/app/agents/new_chat/tools/luma/read_event.py index eb3ac55c6..b37a9d617 100644 --- a/surfsense_backend/app/agents/new_chat/tools/luma/read_event.py +++ b/surfsense_backend/app/agents/new_chat/tools/luma/read_event.py @@ -44,11 +44,21 @@ def create_read_luma_event_tool( ) if resp.status_code == 401: - return {"status": "auth_error", "message": "Luma API key is invalid.", "connector_type": "luma"} + return { + "status": "auth_error", + "message": "Luma API key is invalid.", + "connector_type": "luma", + } if resp.status_code == 404: - return {"status": "not_found", "message": f"Event '{event_id}' not found."} + return { + "status": "not_found", + "message": f"Event '{event_id}' not found.", + } if resp.status_code != 200: - return {"status": "error", "message": f"Luma API error: {resp.status_code}"} + return { + "status": "error", + "message": f"Luma API error: {resp.status_code}", + } data = resp.json() ev = data.get("event", data) diff --git a/surfsense_backend/app/agents/new_chat/tools/mcp_client.py b/surfsense_backend/app/agents/new_chat/tools/mcp_client.py index b46ddbcc5..e28ac8bda 100644 --- a/surfsense_backend/app/agents/new_chat/tools/mcp_client.py +++ b/surfsense_backend/app/agents/new_chat/tools/mcp_client.py @@ -220,10 +220,8 @@ class MCPClient: logger.info("MCP tool '%s' succeeded: %s", tool_name, result_str[:200]) return result_str - except asyncio.TimeoutError: - logger.error( - "MCP tool '%s' timed out after %.0fs", tool_name, timeout - ) + except TimeoutError: + logger.error("MCP tool '%s' timed out after %.0fs", tool_name, timeout) return f"Error: MCP tool '{tool_name}' timed out after {timeout:.0f}s" except RuntimeError as e: if "Invalid structured content" in str(e): diff --git a/surfsense_backend/app/agents/new_chat/tools/mcp_tool.py b/surfsense_backend/app/agents/new_chat/tools/mcp_tool.py index dfee24516..5b96ab374 100644 --- a/surfsense_backend/app/agents/new_chat/tools/mcp_tool.py +++ b/surfsense_backend/app/agents/new_chat/tools/mcp_tool.py @@ -35,7 +35,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.agents.new_chat.tools.hitl import request_approval from app.agents.new_chat.tools.mcp_client import MCPClient -from app.db import SearchSourceConnector, SearchSourceConnectorType +from app.db import SearchSourceConnector from app.services.mcp_oauth.registry import MCP_SERVICES, get_service_by_connector_type logger = logging.getLogger(__name__) @@ -105,13 +105,15 @@ def _create_dynamic_input_model_from_schema( description=( "Arguments to pass to this tool as a JSON object. " "Infer sensible key names from the tool name and description " - "(e.g. {\"search\": \"my query\"} for a search tool)." + '(e.g. {"search": "my query"} for a search tool).' ), ), ) model_name = f"{tool_name.replace(' ', '').replace('-', '_')}Input" - model = create_model(model_name, __config__=ConfigDict(extra="allow"), **field_definitions) + model = create_model( + model_name, __config__=ConfigDict(extra="allow"), **field_definitions + ) return model @@ -187,16 +189,23 @@ async def _create_mcp_tool_from_definition_stdio( except Exception as e: last_error = e if attempt < _TOOL_CALL_MAX_RETRIES - 1: - delay = _TOOL_CALL_RETRY_DELAY * (2 ** attempt) + delay = _TOOL_CALL_RETRY_DELAY * (2**attempt) logger.warning( "MCP tool '%s' failed (attempt %d/%d): %s. Retrying in %.1fs...", - tool_name, attempt + 1, _TOOL_CALL_MAX_RETRIES, e, delay, + tool_name, + attempt + 1, + _TOOL_CALL_MAX_RETRIES, + e, + delay, ) await asyncio.sleep(delay) else: logger.error( "MCP tool '%s' failed after %d attempts: %s", - tool_name, _TOOL_CALL_MAX_RETRIES, e, exc_info=True, + tool_name, + _TOOL_CALL_MAX_RETRIES, + e, + exc_info=True, ) return f"Error: MCP tool '{tool_name}' failed after {_TOOL_CALL_MAX_RETRIES} attempts: {last_error!s}" @@ -318,17 +327,22 @@ async def _create_mcp_tool_from_definition_http( try: result_str = await _do_mcp_call(headers, call_kwargs) - logger.debug("MCP HTTP tool '%s' succeeded (len=%d)", exposed_name, len(result_str)) + logger.debug( + "MCP HTTP tool '%s' succeeded (len=%d)", exposed_name, len(result_str) + ) return result_str except Exception as first_err: if not _is_auth_error(first_err) or connector_id is None: - logger.exception("MCP HTTP tool '%s' execution failed: %s", exposed_name, first_err) + logger.exception( + "MCP HTTP tool '%s' execution failed: %s", exposed_name, first_err + ) return f"Error: MCP HTTP tool '{exposed_name}' execution failed: {first_err!s}" logger.warning( "MCP HTTP tool '%s' got 401 — attempting token refresh for connector %s", - exposed_name, connector_id, + exposed_name, + connector_id, ) fresh_headers = await _force_refresh_and_get_headers(connector_id) if fresh_headers is None: @@ -348,7 +362,8 @@ async def _create_mcp_tool_from_definition_http( except Exception as retry_err: logger.exception( "MCP HTTP tool '%s' still failing after token refresh: %s", - exposed_name, retry_err, + exposed_name, + retry_err, ) if _is_auth_error(retry_err): await _mark_connector_auth_expired(connector_id) @@ -393,7 +408,8 @@ async def _load_stdio_mcp_tools( if not command or not isinstance(command, str): logger.warning( "MCP connector %d (name: '%s') missing or invalid command field, skipping", - connector_id, connector_name, + connector_id, + connector_name, ) return tools @@ -401,7 +417,8 @@ async def _load_stdio_mcp_tools( if not isinstance(args, list): logger.warning( "MCP connector %d (name: '%s') has invalid args field (must be list), skipping", - connector_id, connector_name, + connector_id, + connector_name, ) return tools @@ -409,7 +426,8 @@ async def _load_stdio_mcp_tools( if not isinstance(env, dict): logger.warning( "MCP connector %d (name: '%s') has invalid env field (must be dict), skipping", - connector_id, connector_name, + connector_id, + connector_name, ) return tools @@ -420,7 +438,9 @@ async def _load_stdio_mcp_tools( logger.info( "Discovered %d tools from stdio MCP server '%s' (connector %d)", - len(tool_definitions), command, connector_id, + len(tool_definitions), + command, + connector_id, ) for tool_def in tool_definitions: @@ -436,7 +456,9 @@ async def _load_stdio_mcp_tools( except Exception as e: logger.exception( "Failed to create tool '%s' from connector %d: %s", - tool_def.get("name"), connector_id, e, + tool_def.get("name"), + connector_id, + e, ) return tools @@ -468,7 +490,8 @@ async def _load_http_mcp_tools( if not url or not isinstance(url, str): logger.warning( "MCP connector %d (name: '%s') missing or invalid url field, skipping", - connector_id, connector_name, + connector_id, + connector_name, ) return tools @@ -476,7 +499,8 @@ async def _load_http_mcp_tools( if not isinstance(headers, dict): logger.warning( "MCP connector %d (name: '%s') has invalid headers field (must be dict), skipping", - connector_id, connector_name, + connector_id, + connector_name, ) return tools @@ -507,7 +531,9 @@ async def _load_http_mcp_tools( if not _is_auth_error(first_err) or connector_id is None: logger.exception( "Failed to connect to HTTP MCP server at '%s' (connector %d): %s", - url, connector_id, first_err, + url, + connector_id, + first_err, ) return tools @@ -534,7 +560,8 @@ async def _load_http_mcp_tools( except Exception as retry_err: logger.exception( "HTTP MCP discovery for connector %d still failing after refresh: %s", - connector_id, retry_err, + connector_id, + retry_err, ) if _is_auth_error(retry_err): await _mark_connector_auth_expired(connector_id) @@ -543,17 +570,20 @@ async def _load_http_mcp_tools( total_discovered = len(tool_definitions) if allowed_set: - tool_definitions = [ - td for td in tool_definitions if td["name"] in allowed_set - ] + tool_definitions = [td for td in tool_definitions if td["name"] in allowed_set] logger.info( "HTTP MCP server '%s' (connector %d): %d/%d tools after allowlist filter", - url, connector_id, len(tool_definitions), total_discovered, + url, + connector_id, + len(tool_definitions), + total_discovered, ) else: logger.info( "Discovered %d tools from HTTP MCP server '%s' (connector %d) — no allowlist, loading all", - total_discovered, url, connector_id, + total_discovered, + url, + connector_id, ) for tool_def in tool_definitions: @@ -573,7 +603,9 @@ async def _load_http_mcp_tools( except Exception as e: logger.exception( "Failed to create HTTP tool '%s' from connector %d: %s", - tool_def.get("name"), connector_id, e, + tool_def.get("name"), + connector_id, + e, ) return tools @@ -628,7 +660,7 @@ def _inject_oauth_headers( async def _refresh_connector_token( session: AsyncSession, - connector: "SearchSourceConnector", + connector: SearchSourceConnector, ) -> str | None: """Refresh the OAuth token for an MCP connector and persist the result. @@ -692,12 +724,8 @@ async def _refresh_connector_token( updated_oauth = dict(mcp_oauth) updated_oauth["access_token"] = enc.encrypt_token(new_access) if token_json.get("refresh_token"): - updated_oauth["refresh_token"] = enc.encrypt_token( - token_json["refresh_token"] - ) - updated_oauth["expires_at"] = ( - new_expires_at.isoformat() if new_expires_at else None - ) + updated_oauth["refresh_token"] = enc.encrypt_token(token_json["refresh_token"]) + updated_oauth["expires_at"] = new_expires_at.isoformat() if new_expires_at else None updated_cfg = {**cfg, "mcp_oauth": updated_oauth} updated_cfg.pop("auth_expired", None) @@ -713,7 +741,7 @@ async def _refresh_connector_token( async def _maybe_refresh_mcp_oauth_token( session: AsyncSession, - connector: "SearchSourceConnector", + connector: SearchSourceConnector, cfg: dict[str, Any], server_config: dict[str, Any], ) -> dict[str, Any]: @@ -731,10 +759,11 @@ async def _maybe_refresh_mcp_oauth_token( try: expires_at = datetime.fromisoformat(expires_at_str) if expires_at.tzinfo is None: - from datetime import timezone - expires_at = expires_at.replace(tzinfo=timezone.utc) + expires_at = expires_at.replace(tzinfo=UTC) - if datetime.now(UTC) < expires_at - timedelta(seconds=_TOKEN_REFRESH_BUFFER_SECONDS): + if datetime.now(UTC) < expires_at - timedelta( + seconds=_TOKEN_REFRESH_BUFFER_SECONDS + ): return server_config except (ValueError, TypeError): return server_config @@ -744,7 +773,9 @@ async def _maybe_refresh_mcp_oauth_token( if not new_access: return server_config - logger.info("Proactively refreshed MCP OAuth token for connector %s", connector.id) + logger.info( + "Proactively refreshed MCP OAuth token for connector %s", connector.id + ) refreshed_config = dict(server_config) refreshed_config["headers"] = { @@ -920,7 +951,7 @@ async def load_mcp_tools( result = await session.execute( select(SearchSourceConnector).filter( SearchSourceConnector.search_space_id == search_space_id, - cast(SearchSourceConnector.config, JSONB).has_key("server_config"), # noqa: W601 + cast(SearchSourceConnector.config, JSONB).has_key("server_config"), ), ) @@ -956,13 +987,17 @@ async def load_mcp_tools( if not server_config or not isinstance(server_config, dict): logger.warning( "MCP connector %d (name: '%s') has invalid or missing server_config, skipping", - connector.id, connector.name, + connector.id, + connector.name, ) continue if cfg.get("mcp_oauth"): server_config = await _maybe_refresh_mcp_oauth_token( - session, connector, cfg, server_config, + session, + connector, + cfg, + server_config, ) cfg = connector.config or {} server_config = _inject_oauth_headers(cfg, server_config) @@ -995,22 +1030,25 @@ async def load_mcp_tools( if service_key: tool_name_prefix = f"{service_key}_{connector.id}" - discovery_tasks.append({ - "connector_id": connector.id, - "connector_name": connector.name, - "server_config": server_config, - "trusted_tools": trusted_tools, - "allowed_tools": allowed_tools, - "readonly_tools": readonly_tools, - "tool_name_prefix": tool_name_prefix, - "transport": server_config.get("transport", "stdio"), - "is_generic_mcp": svc_cfg is None, - }) + discovery_tasks.append( + { + "connector_id": connector.id, + "connector_name": connector.name, + "server_config": server_config, + "trusted_tools": trusted_tools, + "allowed_tools": allowed_tools, + "readonly_tools": readonly_tools, + "tool_name_prefix": tool_name_prefix, + "transport": server_config.get("transport", "stdio"), + "is_generic_mcp": svc_cfg is None, + } + ) except Exception as e: logger.exception( "Failed to prepare MCP connector %d: %s", - connector.id, e, + connector.id, + e, ) async def _discover_one(task: dict[str, Any]) -> list[StructuredTool]: @@ -1039,23 +1077,23 @@ async def load_mcp_tools( ), timeout=_MCP_DISCOVERY_TIMEOUT_SECONDS, ) - except asyncio.TimeoutError: + except TimeoutError: logger.error( "MCP connector %d timed out after %ds during discovery", - task["connector_id"], _MCP_DISCOVERY_TIMEOUT_SECONDS, + task["connector_id"], + _MCP_DISCOVERY_TIMEOUT_SECONDS, ) return [] except Exception as e: logger.exception( "Failed to load tools from MCP connector %d: %s", - task["connector_id"], e, + task["connector_id"], + e, ) return [] results = await asyncio.gather(*[_discover_one(t) for t in discovery_tasks]) - tools: list[StructuredTool] = [ - tool for sublist in results for tool in sublist - ] + tools: list[StructuredTool] = [tool for sublist in results for tool in sublist] _mcp_tools_cache[search_space_id] = (now, tools) @@ -1063,7 +1101,9 @@ async def load_mcp_tools( oldest_key = min(_mcp_tools_cache, key=lambda k: _mcp_tools_cache[k][0]) del _mcp_tools_cache[oldest_key] - logger.info("Loaded %d MCP tools for search space %d", len(tools), search_space_id) + logger.info( + "Loaded %d MCP tools for search space %d", len(tools), search_space_id + ) return tools except Exception as e: diff --git a/surfsense_backend/app/agents/new_chat/tools/registry.py b/surfsense_backend/app/agents/new_chat/tools/registry.py index 85c89b114..3ac8677b9 100644 --- a/surfsense_backend/app/agents/new_chat/tools/registry.py +++ b/surfsense_backend/app/agents/new_chat/tools/registry.py @@ -50,6 +50,7 @@ from .confluence import ( create_delete_confluence_page_tool, create_update_confluence_page_tool, ) +from .connected_accounts import create_get_connected_accounts_tool from .discord import ( create_list_discord_channels_tool, create_read_discord_messages_tool, @@ -78,7 +79,6 @@ from .google_drive import ( create_create_google_drive_file_tool, create_delete_google_drive_file_tool, ) -from .connected_accounts import create_get_connected_accounts_tool from .luma import ( create_create_luma_event_tool, create_list_luma_events_tool, @@ -675,10 +675,7 @@ def get_connector_gated_tools( available_connectors: list[str] | None, ) -> list[str]: """Return tool names to disable""" - if available_connectors is None: - available = set() - else: - available = set(available_connectors) + available = set() if available_connectors is None else set(available_connectors) disabled: list[str] = [] for tool_def in BUILTIN_TOOLS: @@ -829,14 +826,16 @@ async def build_tools_async( tools.extend(mcp_tools) logging.info( "Registered %d MCP tools: %s", - len(mcp_tools), [t.name for t in mcp_tools], + len(mcp_tools), + [t.name for t in mcp_tools], ) except Exception as e: logging.exception("Failed to load MCP tools: %s", e) logging.info( "Total tools for agent: %d — %s", - len(tools), [t.name for t in tools], + len(tools), + [t.name for t in tools], ) return tools diff --git a/surfsense_backend/app/agents/new_chat/tools/teams/_auth.py b/surfsense_backend/app/agents/new_chat/tools/teams/_auth.py index f24f5502e..4345bb476 100644 --- a/surfsense_backend/app/agents/new_chat/tools/teams/_auth.py +++ b/surfsense_backend/app/agents/new_chat/tools/teams/_auth.py @@ -17,7 +17,8 @@ async def get_teams_connector( select(SearchSourceConnector).filter( SearchSourceConnector.search_space_id == search_space_id, SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type == SearchSourceConnectorType.TEAMS_CONNECTOR, + SearchSourceConnector.connector_type + == SearchSourceConnectorType.TEAMS_CONNECTOR, ) ) return result.scalars().first() diff --git a/surfsense_backend/app/agents/new_chat/tools/teams/list_channels.py b/surfsense_backend/app/agents/new_chat/tools/teams/list_channels.py index a676595c1..d7b000853 100644 --- a/surfsense_backend/app/agents/new_chat/tools/teams/list_channels.py +++ b/surfsense_backend/app/agents/new_chat/tools/teams/list_channels.py @@ -35,12 +35,21 @@ def create_list_teams_channels_tool( headers = {"Authorization": f"Bearer {token}"} async with httpx.AsyncClient(timeout=20.0) as client: - teams_resp = await client.get(f"{GRAPH_API}/me/joinedTeams", headers=headers) + teams_resp = await client.get( + f"{GRAPH_API}/me/joinedTeams", headers=headers + ) if teams_resp.status_code == 401: - return {"status": "auth_error", "message": "Teams token expired. Please re-authenticate.", "connector_type": "teams"} + return { + "status": "auth_error", + "message": "Teams token expired. Please re-authenticate.", + "connector_type": "teams", + } if teams_resp.status_code != 200: - return {"status": "error", "message": f"Graph API error: {teams_resp.status_code}"} + return { + "status": "error", + "message": f"Graph API error: {teams_resp.status_code}", + } teams_data = teams_resp.json().get("value", []) result_teams = [] @@ -58,13 +67,19 @@ def create_list_teams_channels_tool( {"id": ch["id"], "name": ch.get("displayName", "")} for ch in ch_resp.json().get("value", []) ] - result_teams.append({ - "team_id": team_id, - "team_name": team.get("displayName", ""), - "channels": channels, - }) + result_teams.append( + { + "team_id": team_id, + "team_name": team.get("displayName", ""), + "channels": channels, + } + ) - return {"status": "success", "teams": result_teams, "total_teams": len(result_teams)} + return { + "status": "success", + "teams": result_teams, + "total_teams": len(result_teams), + } except Exception as e: from langgraph.errors import GraphInterrupt diff --git a/surfsense_backend/app/agents/new_chat/tools/teams/read_messages.py b/surfsense_backend/app/agents/new_chat/tools/teams/read_messages.py index 90896cb95..d24a7e4d3 100644 --- a/surfsense_backend/app/agents/new_chat/tools/teams/read_messages.py +++ b/surfsense_backend/app/agents/new_chat/tools/teams/read_messages.py @@ -52,11 +52,21 @@ def create_read_teams_messages_tool( ) if resp.status_code == 401: - return {"status": "auth_error", "message": "Teams token expired. Please re-authenticate.", "connector_type": "teams"} + return { + "status": "auth_error", + "message": "Teams token expired. Please re-authenticate.", + "connector_type": "teams", + } if resp.status_code == 403: - return {"status": "error", "message": "Insufficient permissions to read this channel."} + return { + "status": "error", + "message": "Insufficient permissions to read this channel.", + } if resp.status_code != 200: - return {"status": "error", "message": f"Graph API error: {resp.status_code}"} + return { + "status": "error", + "message": f"Graph API error: {resp.status_code}", + } raw_msgs = resp.json().get("value", []) messages = [] @@ -64,13 +74,15 @@ def create_read_teams_messages_tool( sender = m.get("from", {}) user_info = sender.get("user", {}) if sender else {} body = m.get("body", {}) - messages.append({ - "id": m.get("id"), - "sender": user_info.get("displayName", "Unknown"), - "content": body.get("content", ""), - "content_type": body.get("contentType", "text"), - "timestamp": m.get("createdDateTime", ""), - }) + messages.append( + { + "id": m.get("id"), + "sender": user_info.get("displayName", "Unknown"), + "content": body.get("content", ""), + "content_type": body.get("contentType", "text"), + "timestamp": m.get("createdDateTime", ""), + } + ) return { "status": "success", diff --git a/surfsense_backend/app/agents/new_chat/tools/teams/send_message.py b/surfsense_backend/app/agents/new_chat/tools/teams/send_message.py index ba3a515d9..fd8d00870 100644 --- a/surfsense_backend/app/agents/new_chat/tools/teams/send_message.py +++ b/surfsense_backend/app/agents/new_chat/tools/teams/send_message.py @@ -50,12 +50,19 @@ def create_send_teams_message_tool( result = request_approval( action_type="teams_send_message", tool_name="send_teams_message", - params={"team_id": team_id, "channel_id": channel_id, "content": content}, + params={ + "team_id": team_id, + "channel_id": channel_id, + "content": content, + }, context={"connector_id": connector.id}, ) if result.rejected: - return {"status": "rejected", "message": "User declined. Message was not sent."} + return { + "status": "rejected", + "message": "User declined. Message was not sent.", + } final_content = result.params.get("content", content) final_team = result.params.get("team_id", team_id) @@ -74,20 +81,27 @@ def create_send_teams_message_tool( ) if resp.status_code == 401: - return {"status": "auth_error", "message": "Teams token expired. Please re-authenticate.", "connector_type": "teams"} + return { + "status": "auth_error", + "message": "Teams token expired. Please re-authenticate.", + "connector_type": "teams", + } if resp.status_code == 403: return { "status": "insufficient_permissions", "message": "Missing ChannelMessage.Send permission. Please re-authenticate with updated scopes.", } if resp.status_code not in (200, 201): - return {"status": "error", "message": f"Graph API error: {resp.status_code} — {resp.text[:200]}"} + return { + "status": "error", + "message": f"Graph API error: {resp.status_code} — {resp.text[:200]}", + } msg_data = resp.json() return { "status": "success", "message_id": msg_data.get("id"), - "message": f"Message sent to Teams channel.", + "message": "Message sent to Teams channel.", } except Exception as e: diff --git a/surfsense_backend/app/agents/new_chat/tools/tool_response.py b/surfsense_backend/app/agents/new_chat/tools/tool_response.py index 5fb1864b7..8644ada5c 100644 --- a/surfsense_backend/app/agents/new_chat/tools/tool_response.py +++ b/surfsense_backend/app/agents/new_chat/tools/tool_response.py @@ -6,7 +6,6 @@ from typing import Any class ToolResponse: - @staticmethod def success(message: str, **data: Any) -> dict[str, Any]: return {"status": "success", "message": message, **data} @@ -31,9 +30,7 @@ class ToolResponse: return {"status": "rejected", "message": message} @staticmethod - def not_found( - resource: str, identifier: str, **data: Any - ) -> dict[str, Any]: + def not_found(resource: str, identifier: str, **data: Any) -> dict[str, Any]: return { "status": "not_found", "error": f"{resource} '{identifier}' was not found.", diff --git a/surfsense_backend/app/connectors/exceptions.py b/surfsense_backend/app/connectors/exceptions.py index 32a1e7bdc..027adbb87 100644 --- a/surfsense_backend/app/connectors/exceptions.py +++ b/surfsense_backend/app/connectors/exceptions.py @@ -13,7 +13,6 @@ from typing import Any class ConnectorError(Exception): - def __init__( self, message: str, diff --git a/surfsense_backend/app/routes/__init__.py b/surfsense_backend/app/routes/__init__.py index 8df930f30..de4e05423 100644 --- a/surfsense_backend/app/routes/__init__.py +++ b/surfsense_backend/app/routes/__init__.py @@ -98,7 +98,9 @@ router.include_router(logs_router) router.include_router(circleback_webhook_router) # Circleback meeting webhooks router.include_router(surfsense_docs_router) # Surfsense documentation for citations router.include_router(notifications_router) # Notifications with Zero sync -router.include_router(mcp_oauth_router) # MCP OAuth 2.1 for Linear, Jira, ClickUp, Slack, Airtable +router.include_router( + mcp_oauth_router +) # MCP OAuth 2.1 for Linear, Jira, ClickUp, Slack, Airtable router.include_router(composio_router) # Composio OAuth and toolkit management router.include_router(public_chat_router) # Public chat sharing and cloning router.include_router(incentive_tasks_router) # Incentive tasks for earning free pages diff --git a/surfsense_backend/app/routes/mcp_oauth_route.py b/surfsense_backend/app/routes/mcp_oauth_route.py index e14be83d0..1abc1f1ec 100644 --- a/surfsense_backend/app/routes/mcp_oauth_route.py +++ b/surfsense_backend/app/routes/mcp_oauth_route.py @@ -29,7 +29,11 @@ from app.db import ( ) from app.users import current_active_user from app.utils.connector_naming import generate_unique_connector_name -from app.utils.oauth_security import OAuthStateManager, TokenEncryption, generate_pkce_pair +from app.utils.oauth_security import ( + OAuthStateManager, + TokenEncryption, + generate_pkce_pair, +) logger = logging.getLogger(__name__) @@ -37,7 +41,9 @@ router = APIRouter() async def _fetch_account_metadata( - service_key: str, access_token: str, token_json: dict[str, Any], + service_key: str, + access_token: str, + token_json: dict[str, Any], ) -> dict[str, Any]: """Fetch display-friendly account metadata after a successful token exchange. @@ -86,7 +92,8 @@ async def _fetch_account_metadata( meta["display_name"] = whoami.get("email", "Airtable") else: logger.warning( - "Airtable whoami API returned %d (non-blocking)", resp.status_code, + "Airtable whoami API returned %d (non-blocking)", + resp.status_code, ) except Exception: @@ -98,6 +105,7 @@ async def _fetch_account_metadata( return meta + _state_manager: OAuthStateManager | None = None _token_encryption: TokenEncryption | None = None @@ -151,6 +159,7 @@ def _frontend_redirect( # /add — start MCP OAuth flow # --------------------------------------------------------------------------- + @router.get("/auth/mcp/{service}/connector/add") async def connect_mcp_service( service: str, @@ -170,9 +179,12 @@ async def connect_mcp_service( ) metadata = await discover_oauth_metadata( - svc.mcp_url, origin_override=svc.oauth_discovery_origin, + svc.mcp_url, + origin_override=svc.oauth_discovery_origin, + ) + auth_endpoint = svc.auth_endpoint_override or metadata.get( + "authorization_endpoint" ) - auth_endpoint = svc.auth_endpoint_override or metadata.get("authorization_endpoint") token_endpoint = svc.token_endpoint_override or metadata.get("token_endpoint") registration_endpoint = metadata.get("registration_endpoint") @@ -236,7 +248,9 @@ async def connect_mcp_service( logger.info( "Generated %s MCP OAuth URL for user %s, space %s", - svc.name, user.id, space_id, + svc.name, + user.id, + space_id, ) return {"auth_url": auth_url} @@ -245,7 +259,8 @@ async def connect_mcp_service( except Exception as e: logger.error("Failed to initiate %s MCP OAuth: %s", service, e, exc_info=True) raise HTTPException( - status_code=500, detail=f"Failed to initiate {service} MCP OAuth.", + status_code=500, + detail=f"Failed to initiate {service} MCP OAuth.", ) from e @@ -253,6 +268,7 @@ async def connect_mcp_service( # /callback — handle OAuth redirect # --------------------------------------------------------------------------- + @router.get("/auth/mcp/{service}/connector/callback") async def mcp_oauth_callback( service: str, @@ -271,7 +287,9 @@ async def mcp_oauth_callback( except Exception: pass return _frontend_redirect( - space_id, error=f"{service}_mcp_oauth_denied", service=service, + space_id, + error=f"{service}_mcp_oauth_denied", + service=service, ) if not code: @@ -337,9 +355,7 @@ async def mcp_oauth_callback( expires_at = None if expires_in: - expires_at = datetime.now(UTC) + timedelta( - seconds=int(expires_in) - ) + expires_at = datetime.now(UTC) + timedelta(seconds=int(expires_in)) connector_config = { "server_config": { @@ -349,10 +365,14 @@ async def mcp_oauth_callback( "mcp_service": svc_key, "mcp_oauth": { "client_id": client_id, - "client_secret": enc.encrypt_token(client_secret) if client_secret else "", + "client_secret": enc.encrypt_token(client_secret) + if client_secret + else "", "token_endpoint": token_endpoint, "access_token": enc.encrypt_token(access_token), - "refresh_token": enc.encrypt_token(refresh_token) if refresh_token else None, + "refresh_token": enc.encrypt_token(refresh_token) + if refresh_token + else None, "expires_at": expires_at.isoformat() if expires_at else None, "scope": scope, }, @@ -361,15 +381,27 @@ async def mcp_oauth_callback( account_meta = await _fetch_account_metadata(svc_key, access_token, token_json) if account_meta: - _SAFE_META_KEYS = {"display_name", "team_id", "team_name", "user_id", "user_email", - "workspace_id", "workspace_name", "organization_name", - "organization_url_key", "cloud_id", "site_name", "base_url"} + safe_meta_keys = { + "display_name", + "team_id", + "team_name", + "user_id", + "user_email", + "workspace_id", + "workspace_name", + "organization_name", + "organization_url_key", + "cloud_id", + "site_name", + "base_url", + } for k, v in account_meta.items(): - if k in _SAFE_META_KEYS: + if k in safe_meta_keys: connector_config[k] = v logger.info( "Stored account metadata for %s: display_name=%s", - svc_key, account_meta.get("display_name", ""), + svc_key, + account_meta.get("display_name", ""), ) # ---- Re-auth path ---- @@ -400,15 +432,24 @@ async def mcp_oauth_callback( logger.info( "Re-authenticated %s MCP connector %s for user %s", - svc.name, db_connector.id, user_id, + svc.name, + db_connector.id, + user_id, ) reauth_return_url = data.get("return_url") - if reauth_return_url and reauth_return_url.startswith("/") and not reauth_return_url.startswith("//"): + if ( + reauth_return_url + and reauth_return_url.startswith("/") + and not reauth_return_url.startswith("//") + ): return RedirectResponse( url=f"{config.NEXT_FRONTEND_URL}{reauth_return_url}" ) return _frontend_redirect( - space_id, success=True, connector_id=db_connector.id, service=service, + space_id, + success=True, + connector_id=db_connector.id, + service=service, ) # ---- New connector path ---- @@ -436,24 +477,34 @@ async def mcp_oauth_callback( except IntegrityError as e: await session.rollback() raise HTTPException( - status_code=409, detail="A connector for this service already exists.", + status_code=409, + detail="A connector for this service already exists.", ) from e _invalidate_cache(space_id) logger.info( "Created %s MCP connector %s for user %s in space %s", - svc.name, new_connector.id, user_id, space_id, + svc.name, + new_connector.id, + user_id, + space_id, ) return _frontend_redirect( - space_id, success=True, connector_id=new_connector.id, service=service, + space_id, + success=True, + connector_id=new_connector.id, + service=service, ) except HTTPException: raise except Exception as e: logger.error( - "Failed to complete %s MCP OAuth: %s", service, e, exc_info=True, + "Failed to complete %s MCP OAuth: %s", + service, + e, + exc_info=True, ) raise HTTPException( status_code=500, @@ -465,6 +516,7 @@ async def mcp_oauth_callback( # /reauth — re-authenticate an existing MCP connector # --------------------------------------------------------------------------- + @router.get("/auth/mcp/{service}/connector/reauth") async def reauth_mcp_service( service: str, @@ -491,7 +543,8 @@ async def reauth_mcp_service( ) if not result.scalars().first(): raise HTTPException( - status_code=404, detail="Connector not found or access denied", + status_code=404, + detail="Connector not found or access denied", ) try: @@ -501,9 +554,12 @@ async def reauth_mcp_service( ) metadata = await discover_oauth_metadata( - svc.mcp_url, origin_override=svc.oauth_discovery_origin, + svc.mcp_url, + origin_override=svc.oauth_discovery_origin, + ) + auth_endpoint = svc.auth_endpoint_override or metadata.get( + "authorization_endpoint" ) - auth_endpoint = svc.auth_endpoint_override or metadata.get("authorization_endpoint") token_endpoint = svc.token_endpoint_override or metadata.get("token_endpoint") registration_endpoint = metadata.get("registration_endpoint") @@ -545,7 +601,9 @@ async def reauth_mcp_service( "service": service, "code_verifier": verifier, "mcp_client_id": client_id, - "mcp_client_secret": enc.encrypt_token(client_secret) if client_secret else "", + "mcp_client_secret": enc.encrypt_token(client_secret) + if client_secret + else "", "mcp_token_endpoint": token_endpoint, "mcp_url": svc.mcp_url, "connector_id": connector_id, @@ -554,7 +612,9 @@ async def reauth_mcp_service( extra["return_url"] = return_url state = _get_state_manager().generate_secure_state( - space_id, user.id, **extra, + space_id, + user.id, + **extra, ) auth_params: dict[str, str] = { @@ -572,7 +632,9 @@ async def reauth_mcp_service( logger.info( "Initiating %s MCP re-auth for user %s, connector %s", - svc.name, user.id, connector_id, + svc.name, + user.id, + connector_id, ) return {"auth_url": auth_url} @@ -580,7 +642,10 @@ async def reauth_mcp_service( raise except Exception as e: logger.error( - "Failed to initiate %s MCP re-auth: %s", service, e, exc_info=True, + "Failed to initiate %s MCP re-auth: %s", + service, + e, + exc_info=True, ) raise HTTPException( status_code=500, @@ -592,6 +657,7 @@ async def reauth_mcp_service( # Helpers # --------------------------------------------------------------------------- + def _invalidate_cache(space_id: int) -> None: try: from app.agents.new_chat.tools.mcp_tool import invalidate_mcp_tools_cache diff --git a/surfsense_backend/app/routes/new_chat_routes.py b/surfsense_backend/app/routes/new_chat_routes.py index 85a8658ec..091e87737 100644 --- a/surfsense_backend/app/routes/new_chat_routes.py +++ b/surfsense_backend/app/routes/new_chat_routes.py @@ -24,9 +24,9 @@ from sqlalchemy.orm import selectinload from app.agents.new_chat.filesystem_selection import ( ClientPlatform, - LocalFilesystemMount, FilesystemMode, FilesystemSelection, + LocalFilesystemMount, ) from app.config import config from app.db import ( diff --git a/surfsense_backend/app/routes/oauth_connector_base.py b/surfsense_backend/app/routes/oauth_connector_base.py index 0638e8f34..5b75d8519 100644 --- a/surfsense_backend/app/routes/oauth_connector_base.py +++ b/surfsense_backend/app/routes/oauth_connector_base.py @@ -9,6 +9,7 @@ Call ``build_router()`` to get a FastAPI ``APIRouter`` with ``/connector/add``, from __future__ import annotations import base64 +import contextlib import logging from datetime import UTC, datetime, timedelta from typing import Any @@ -41,7 +42,6 @@ logger = logging.getLogger(__name__) class OAuthConnectorRoute: - def __init__( self, *, @@ -244,10 +244,8 @@ class OAuthConnectorRoute: if resp.status_code != 200: detail = resp.text - try: + with contextlib.suppress(Exception): detail = resp.json().get("error_description", detail) - except Exception: - pass raise HTTPException( status_code=400, detail=f"Token exchange failed: {detail}" ) @@ -430,7 +428,11 @@ class OAuthConnectorRoute: state_mgr = oauth._get_state_manager() extra: dict[str, Any] = {"connector_id": connector_id} - if return_url and return_url.startswith("/") and not return_url.startswith("//"): + if ( + return_url + and return_url.startswith("/") + and not return_url.startswith("//") + ): extra["return_url"] = return_url auth_params: dict[str, str] = { @@ -450,9 +452,7 @@ class OAuthConnectorRoute: auth_params.update(oauth.extra_auth_params) - state_encoded = state_mgr.generate_secure_state( - space_id, user.id, **extra - ) + state_encoded = state_mgr.generate_secure_state(space_id, user.id, **extra) auth_params["state"] = state_encoded auth_url = f"{oauth.authorize_url}?{urlencode(auth_params)}" @@ -489,9 +489,7 @@ class OAuthConnectorRoute: status_code=400, detail="Missing authorization code" ) if not state: - raise HTTPException( - status_code=400, detail="Missing state parameter" - ) + raise HTTPException(status_code=400, detail="Missing state parameter") state_mgr = oauth._get_state_manager() try: @@ -552,7 +550,11 @@ class OAuthConnectorRoute: db_connector.id, user_id, ) - if reauth_return_url and reauth_return_url.startswith("/") and not reauth_return_url.startswith("//"): + if ( + reauth_return_url + and reauth_return_url.startswith("/") + and not reauth_return_url.startswith("//") + ): return RedirectResponse( url=f"{config.NEXT_FRONTEND_URL}{reauth_return_url}" ) @@ -603,7 +605,8 @@ class OAuthConnectorRoute: except IntegrityError as e: await session.rollback() raise HTTPException( - status_code=409, detail="A connector for this service already exists." + status_code=409, + detail="A connector for this service already exists.", ) from e logger.info( diff --git a/surfsense_backend/app/routes/search_source_connectors_routes.py b/surfsense_backend/app/routes/search_source_connectors_routes.py index d42a7fa1a..9037d275a 100644 --- a/surfsense_backend/app/routes/search_source_connectors_routes.py +++ b/surfsense_backend/app/routes/search_source_connectors_routes.py @@ -3092,7 +3092,7 @@ async def trust_mcp_tool( select(SearchSourceConnector).filter( SearchSourceConnector.id == connector_id, SearchSourceConnector.user_id == user.id, - cast(SearchSourceConnector.config, PG_JSONB).has_key("server_config"), # noqa: W601 + cast(SearchSourceConnector.config, PG_JSONB).has_key("server_config"), ) ) connector = result.scalars().first() @@ -3147,7 +3147,7 @@ async def untrust_mcp_tool( select(SearchSourceConnector).filter( SearchSourceConnector.id == connector_id, SearchSourceConnector.user_id == user.id, - cast(SearchSourceConnector.config, PG_JSONB).has_key("server_config"), # noqa: W601 + cast(SearchSourceConnector.config, PG_JSONB).has_key("server_config"), ) ) connector = result.scalars().first() diff --git a/surfsense_backend/app/services/mcp_oauth/discovery.py b/surfsense_backend/app/services/mcp_oauth/discovery.py index b0f3fef2a..dc21443bc 100644 --- a/surfsense_backend/app/services/mcp_oauth/discovery.py +++ b/surfsense_backend/app/services/mcp_oauth/discovery.py @@ -55,7 +55,9 @@ async def register_client( async with httpx.AsyncClient(follow_redirects=True) as client: resp = await client.post( - registration_endpoint, json=payload, timeout=timeout, + registration_endpoint, + json=payload, + timeout=timeout, ) resp.raise_for_status() return resp.json() diff --git a/surfsense_backend/app/services/mcp_oauth/registry.py b/surfsense_backend/app/services/mcp_oauth/registry.py index 49bc74d3d..835d70184 100644 --- a/surfsense_backend/app/services/mcp_oauth/registry.py +++ b/surfsense_backend/app/services/mcp_oauth/registry.py @@ -70,12 +70,14 @@ MCP_SERVICES: dict[str, MCPServiceConfig] = { "createJiraIssue", "editJiraIssue", ], - readonly_tools=frozenset({ - "getAccessibleAtlassianResources", - "searchJiraIssuesUsingJql", - "getVisibleJiraProjects", - "getJiraProjectIssueTypesMetadata", - }), + readonly_tools=frozenset( + { + "getAccessibleAtlassianResources", + "searchJiraIssuesUsingJql", + "getVisibleJiraProjects", + "getJiraProjectIssueTypesMetadata", + } + ), account_metadata_keys=["cloud_id", "site_name", "base_url"], ), "clickup": MCPServiceConfig( @@ -99,15 +101,23 @@ MCP_SERVICES: dict[str, MCPServiceConfig] = { auth_endpoint_override="https://slack.com/oauth/v2_user/authorize", token_endpoint_override="https://slack.com/api/oauth.v2.user.access", scopes=[ - "search:read.public", "search:read.private", "search:read.mpim", "search:read.im", - "channels:history", "groups:history", "mpim:history", "im:history", + "search:read.public", + "search:read.private", + "search:read.mpim", + "search:read.im", + "channels:history", + "groups:history", + "mpim:history", + "im:history", ], allowed_tools=[ "slack_search_channels", "slack_read_channel", "slack_read_thread", ], - readonly_tools=frozenset({"slack_search_channels", "slack_read_channel", "slack_read_thread"}), + readonly_tools=frozenset( + {"slack_search_channels", "slack_read_channel", "slack_read_thread"} + ), # TODO: oauth.v2.user.access only returns team.id, not team.name. # To populate team_name, either add "team:read" scope and call # GET /api/team.info during OAuth callback, or switch to oauth.v2.access. @@ -127,7 +137,9 @@ MCP_SERVICES: dict[str, MCPServiceConfig] = { "list_tables_for_base", "list_records_for_table", ], - readonly_tools=frozenset({"list_bases", "list_tables_for_base", "list_records_for_table"}), + readonly_tools=frozenset( + {"list_bases", "list_tables_for_base", "list_records_for_table"} + ), account_metadata_keys=["user_id", "user_email"], ), } @@ -136,20 +148,22 @@ _CONNECTOR_TYPE_TO_SERVICE: dict[str, MCPServiceConfig] = { svc.connector_type: svc for svc in MCP_SERVICES.values() } -LIVE_CONNECTOR_TYPES: frozenset[SearchSourceConnectorType] = frozenset({ - SearchSourceConnectorType.SLACK_CONNECTOR, - SearchSourceConnectorType.TEAMS_CONNECTOR, - SearchSourceConnectorType.LINEAR_CONNECTOR, - SearchSourceConnectorType.JIRA_CONNECTOR, - SearchSourceConnectorType.CLICKUP_CONNECTOR, - SearchSourceConnectorType.GOOGLE_CALENDAR_CONNECTOR, - SearchSourceConnectorType.COMPOSIO_GOOGLE_CALENDAR_CONNECTOR, - SearchSourceConnectorType.AIRTABLE_CONNECTOR, - SearchSourceConnectorType.GOOGLE_GMAIL_CONNECTOR, - SearchSourceConnectorType.COMPOSIO_GMAIL_CONNECTOR, - SearchSourceConnectorType.DISCORD_CONNECTOR, - SearchSourceConnectorType.LUMA_CONNECTOR, -}) +LIVE_CONNECTOR_TYPES: frozenset[SearchSourceConnectorType] = frozenset( + { + SearchSourceConnectorType.SLACK_CONNECTOR, + SearchSourceConnectorType.TEAMS_CONNECTOR, + SearchSourceConnectorType.LINEAR_CONNECTOR, + SearchSourceConnectorType.JIRA_CONNECTOR, + SearchSourceConnectorType.CLICKUP_CONNECTOR, + SearchSourceConnectorType.GOOGLE_CALENDAR_CONNECTOR, + SearchSourceConnectorType.COMPOSIO_GOOGLE_CALENDAR_CONNECTOR, + SearchSourceConnectorType.AIRTABLE_CONNECTOR, + SearchSourceConnectorType.GOOGLE_GMAIL_CONNECTOR, + SearchSourceConnectorType.COMPOSIO_GMAIL_CONNECTOR, + SearchSourceConnectorType.DISCORD_CONNECTOR, + SearchSourceConnectorType.LUMA_CONNECTOR, + } +) def get_service(key: str) -> MCPServiceConfig | None: diff --git a/surfsense_backend/app/services/obsidian_plugin_indexer.py b/surfsense_backend/app/services/obsidian_plugin_indexer.py index 8fbdad269..0fc4f30f4 100644 --- a/surfsense_backend/app/services/obsidian_plugin_indexer.py +++ b/surfsense_backend/app/services/obsidian_plugin_indexer.py @@ -156,7 +156,9 @@ async def _extract_binary_attachment_markdown( try: raw_bytes = base64.b64decode(payload.binary_base64, validate=True) except Exception: - logger.warning("obsidian attachment payload had invalid base64: %s", payload.path) + logger.warning( + "obsidian attachment payload had invalid base64: %s", payload.path + ) return "", {"attachment_extraction_status": "invalid_binary_payload"} suffix = f".{payload.extension.lstrip('.')}" @@ -180,7 +182,10 @@ async def _extract_binary_attachment_markdown( return result.markdown_content, metadata except Exception as exc: logger.warning( - "obsidian attachment ETL failed for %s: %s", payload.path, exc, exc_info=True + "obsidian attachment ETL failed for %s: %s", + payload.path, + exc, + exc_info=True, ) return "", { "attachment_extraction_status": "etl_failed", diff --git a/surfsense_backend/app/tasks/chat/stream_new_chat.py b/surfsense_backend/app/tasks/chat/stream_new_chat.py index 5a6117808..7239c57a5 100644 --- a/surfsense_backend/app/tasks/chat/stream_new_chat.py +++ b/surfsense_backend/app/tasks/chat/stream_new_chat.py @@ -31,7 +31,6 @@ from sqlalchemy.orm import selectinload from app.agents.new_chat.chat_deepagent import create_surfsense_deep_agent from app.agents.new_chat.checkpointer import get_checkpointer from app.agents.new_chat.filesystem_selection import FilesystemSelection -from app.config import config from app.agents.new_chat.llm_config import ( AgentConfig, create_chat_litellm_from_agent_config, @@ -182,9 +181,9 @@ def _tool_output_has_error(tool_output: Any) -> bool: if tool_output.get("error"): return True result = tool_output.get("result") - if isinstance(result, str) and result.strip().lower().startswith("error:"): - return True - return False + return bool( + isinstance(result, str) and result.strip().lower().startswith("error:") + ) if isinstance(tool_output, str): return tool_output.strip().lower().startswith("error:") return False @@ -230,7 +229,9 @@ def _log_file_contract(stage: str, result: StreamResult, **extra: Any) -> None: "stage": stage, "request_id": result.request_id or "unknown", "turn_id": result.turn_id or "unknown", - "chat_id": result.turn_id.split(":", 1)[0] if ":" in result.turn_id else "unknown", + "chat_id": result.turn_id.split(":", 1)[0] + if ":" in result.turn_id + else "unknown", "filesystem_mode": result.filesystem_mode, "client_platform": result.client_platform, "intent_detected": result.intent_detected, @@ -242,7 +243,9 @@ def _log_file_contract(stage: str, result: StreamResult, **extra: Any) -> None: "commit_gate_reason": result.commit_gate_reason or None, } payload.update(extra) - _perf_log.info("[file_operation_contract] %s", json.dumps(payload, ensure_ascii=False)) + _perf_log.info( + "[file_operation_contract] %s", json.dumps(payload, ensure_ascii=False) + ) async def _stream_agent_events( @@ -1289,7 +1292,8 @@ async def _stream_agent_events( result.intent_detected = intent_value if ( isinstance(intent_value, str) - and intent_value in ( + and intent_value + in ( "chat_only", "file_write", "file_read", @@ -1308,18 +1312,17 @@ async def _stream_agent_events( result.commit_gate_passed, result.commit_gate_reason = ( _evaluate_file_contract_outcome(result) ) - if not result.commit_gate_passed: - if _contract_enforcement_active(result): - gate_notice = ( - "I could not complete the requested file write because no successful " - "write_file/edit_file operation was confirmed." - ) - gate_text_id = streaming_service.generate_text_id() - yield streaming_service.format_text_start(gate_text_id) - yield streaming_service.format_text_delta(gate_text_id, gate_notice) - yield streaming_service.format_text_end(gate_text_id) - yield streaming_service.format_terminal_info(gate_notice, "error") - accumulated_text = gate_notice + if not result.commit_gate_passed and _contract_enforcement_active(result): + gate_notice = ( + "I could not complete the requested file write because no successful " + "write_file/edit_file operation was confirmed." + ) + gate_text_id = streaming_service.generate_text_id() + yield streaming_service.format_text_start(gate_text_id) + yield streaming_service.format_text_delta(gate_text_id, gate_notice) + yield streaming_service.format_text_end(gate_text_id) + yield streaming_service.format_terminal_info(gate_notice, "error") + accumulated_text = gate_notice else: result.commit_gate_passed = True result.commit_gate_reason = "" diff --git a/surfsense_backend/app/utils/async_retry.py b/surfsense_backend/app/utils/async_retry.py index c3bdd5386..a56f6550a 100644 --- a/surfsense_backend/app/utils/async_retry.py +++ b/surfsense_backend/app/utils/async_retry.py @@ -2,6 +2,7 @@ from __future__ import annotations +import contextlib import logging from collections.abc import Callable from typing import TypeVar @@ -32,9 +33,7 @@ F = TypeVar("F", bound=Callable) def _is_retryable(exc: BaseException) -> bool: if isinstance(exc, ConnectorError): return exc.retryable - if isinstance(exc, (httpx.TimeoutException, httpx.ConnectError)): - return True - return False + return bool(isinstance(exc, (httpx.TimeoutException, httpx.ConnectError))) def build_retry( @@ -86,10 +85,8 @@ def raise_for_status( retry_after_raw = response.headers.get("Retry-After") retry_after: float | None = None if retry_after_raw: - try: + with contextlib.suppress(ValueError, TypeError): retry_after = float(retry_after_raw) - except (ValueError, TypeError): - pass raise ConnectorRateLimitError( f"{service} rate limited (429)", service=service, diff --git a/surfsense_backend/app/utils/connector_naming.py b/surfsense_backend/app/utils/connector_naming.py index 889bf1464..99c8243a5 100644 --- a/surfsense_backend/app/utils/connector_naming.py +++ b/surfsense_backend/app/utils/connector_naming.py @@ -233,7 +233,10 @@ async def generate_unique_connector_name( if identifier: name = f"{base} - {identifier}" return await ensure_unique_connector_name( - session, name, search_space_id, user_id, + session, + name, + search_space_id, + user_id, ) count = await count_connectors_of_type( diff --git a/surfsense_backend/tests/integration/test_obsidian_plugin_routes.py b/surfsense_backend/tests/integration/test_obsidian_plugin_routes.py index 41779a570..22f6c6de5 100644 --- a/surfsense_backend/tests/integration/test_obsidian_plugin_routes.py +++ b/surfsense_backend/tests/integration/test_obsidian_plugin_routes.py @@ -499,7 +499,9 @@ class TestWireContractSmoke: "app.routes.obsidian_plugin_routes.upsert_note", new=AsyncMock(return_value=fake_doc), ) as upsert_mock, - patch("app.routes.obsidian_plugin_routes._queue_obsidian_attachment") as queue_mock, + patch( + "app.routes.obsidian_plugin_routes._queue_obsidian_attachment" + ) as queue_mock, ): sync_resp = await obsidian_sync( SyncBatchRequest( @@ -548,7 +550,9 @@ class TestWireContractSmoke: "app.routes.obsidian_plugin_routes.upsert_note", new=AsyncMock(return_value=fake_doc), ), - patch("app.routes.obsidian_plugin_routes._queue_obsidian_attachment") as queue_mock, + patch( + "app.routes.obsidian_plugin_routes._queue_obsidian_attachment" + ) as queue_mock, ): sync_resp = await obsidian_sync( SyncBatchRequest( @@ -600,7 +604,9 @@ class TestWireContractSmoke: "app.routes.obsidian_plugin_routes.upsert_note", new=AsyncMock(return_value=fake_doc), ), - patch("app.routes.obsidian_plugin_routes._queue_obsidian_attachment") as queue_mock, + patch( + "app.routes.obsidian_plugin_routes._queue_obsidian_attachment" + ) as queue_mock, ): sync_resp = await obsidian_sync( SyncBatchRequest( @@ -619,7 +625,5 @@ class TestWireContractSmoke: items_by_path = {it.path: it for it in sync_resp.items} assert items_by_path["ok.md"].status == "ok" assert items_by_path["image.png"].status == "error" - assert "does not match extension" in ( - items_by_path["image.png"].error or "" - ) + assert "does not match extension" in (items_by_path["image.png"].error or "") queue_mock.assert_not_called() diff --git a/surfsense_backend/tests/unit/middleware/test_file_intent_middleware.py b/surfsense_backend/tests/unit/middleware/test_file_intent_middleware.py index 673331b0a..7fd3fe4a7 100644 --- a/surfsense_backend/tests/unit/middleware/test_file_intent_middleware.py +++ b/surfsense_backend/tests/unit/middleware/test_file_intent_middleware.py @@ -45,9 +45,7 @@ async def test_file_write_intent_injects_contract_message(): @pytest.mark.asyncio async def test_non_write_intent_does_not_inject_contract_message(): - llm = _FakeLLM( - '{"intent":"file_read","confidence":0.88,"suggested_filename":null}' - ) + llm = _FakeLLM('{"intent":"file_read","confidence":0.88,"suggested_filename":null}') middleware = FileIntentMiddleware(llm=llm) original_messages = [HumanMessage(content="Read /notes.md")] state = {"messages": original_messages, "turn_id": "abc:def"} @@ -55,7 +53,10 @@ async def test_non_write_intent_does_not_inject_contract_message(): result = await middleware.abefore_agent(state, runtime=None) # type: ignore[arg-type] assert result is not None - assert result["file_operation_contract"]["intent"] == FileOperationIntent.FILE_READ.value + assert ( + result["file_operation_contract"]["intent"] + == FileOperationIntent.FILE_READ.value + ) assert "messages" not in result @@ -211,4 +212,3 @@ def test_fallback_path_keeps_posix_style_absolute_path_for_linux_and_macos() -> ) assert resolved == "/var/log/surfsense/notes.md" - diff --git a/surfsense_backend/tests/unit/middleware/test_filesystem_verification.py b/surfsense_backend/tests/unit/middleware/test_filesystem_verification.py index d00365032..cca15e789 100644 --- a/surfsense_backend/tests/unit/middleware/test_filesystem_verification.py +++ b/surfsense_backend/tests/unit/middleware/test_filesystem_verification.py @@ -2,11 +2,11 @@ from pathlib import Path import pytest +from app.agents.new_chat.filesystem_selection import FilesystemMode +from app.agents.new_chat.middleware.filesystem import SurfSenseFilesystemMiddleware from app.agents.new_chat.middleware.multi_root_local_folder_backend import ( MultiRootLocalFolderBackend, ) -from app.agents.new_chat.filesystem_selection import FilesystemMode -from app.agents.new_chat.middleware.filesystem import SurfSenseFilesystemMiddleware pytestmark = pytest.mark.unit diff --git a/surfsense_backend/tests/unit/test_obsidian_plugin_indexer.py b/surfsense_backend/tests/unit/test_obsidian_plugin_indexer.py index 7ab3c52e0..20795c739 100644 --- a/surfsense_backend/tests/unit/test_obsidian_plugin_indexer.py +++ b/surfsense_backend/tests/unit/test_obsidian_plugin_indexer.py @@ -15,7 +15,6 @@ from app.services.obsidian_plugin_indexer import ( _require_extracted_attachment_content, ) - _FAKE_PNG_B64 = base64.b64encode(b"\x89PNG\r\n\x1a\n").decode("ascii") @@ -102,9 +101,7 @@ async def test_extract_binary_attachment_markdown_uses_etl(monkeypatch) -> None: mime_type="application/pdf", ) - async def _fake_run_etl_extract( # noqa: ANN001 - *, file_path, filename, vision_llm - ): + async def _fake_run_etl_extract(*, file_path, filename, vision_llm): assert filename == "spec.pdf" assert file_path assert vision_llm is None @@ -216,7 +213,7 @@ def test_note_payload_rejects_markdown_with_binary_fields() -> None: def test_require_extracted_attachment_content_rejects_empty_content() -> None: with pytest.raises( - RuntimeError, match="Attachment extraction failed for assets/img.png" + RuntimeError, match=r"Attachment extraction failed for assets/img\.png" ): _require_extracted_attachment_content( content=" ", diff --git a/surfsense_backend/tests/unit/test_stream_new_chat_contract.py b/surfsense_backend/tests/unit/test_stream_new_chat_contract.py index f4adc3d73..034aa484c 100644 --- a/surfsense_backend/tests/unit/test_stream_new_chat_contract.py +++ b/surfsense_backend/tests/unit/test_stream_new_chat_contract.py @@ -45,4 +45,3 @@ def test_contract_enforcement_local_only(): result.filesystem_mode = "cloud" assert not _contract_enforcement_active(result) - diff --git a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx index 06f3bf79f..9f569398e 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/new-chat/[[...chat_id]]/page.tsx @@ -45,8 +45,8 @@ import { } from "@/components/assistant-ui/token-usage-context"; import { useChatSessionStateSync } from "@/hooks/use-chat-session-state"; import { useMessagesSync } from "@/hooks/use-messages-sync"; -import { documentsApiService } from "@/lib/apis/documents-api.service"; import { getAgentFilesystemSelection } from "@/lib/agent-filesystem"; +import { documentsApiService } from "@/lib/apis/documents-api.service"; import { getBearerToken } from "@/lib/auth-utils"; import { convertToThreadMessage } from "@/lib/chat/message-utils"; import { @@ -661,8 +661,7 @@ export default function NewChatPage() { const selection = await getAgentFilesystemSelection(searchSpaceId); if ( selection.filesystem_mode === "desktop_local_folder" && - (!selection.local_filesystem_mounts || - selection.local_filesystem_mounts.length === 0) + (!selection.local_filesystem_mounts || selection.local_filesystem_mounts.length === 0) ) { toast.error("Select a local folder before using Local Folder mode."); return; @@ -842,14 +841,7 @@ export default function NewChatPage() { }); } else { const tcId = `interrupt-${action.name}`; - addToolCall( - contentPartsState, - toolsWithUI, - tcId, - action.name, - action.args, - true - ); + addToolCall(contentPartsState, toolsWithUI, tcId, action.name, action.args, true); updateToolCall(contentPartsState, tcId, { result: { __interrupt__: true, ...interruptData }, }); @@ -1189,14 +1181,7 @@ export default function NewChatPage() { }); } else { const tcId = `interrupt-${action.name}`; - addToolCall( - contentPartsState, - toolsWithUI, - tcId, - action.name, - action.args, - true - ); + addToolCall(contentPartsState, toolsWithUI, tcId, action.name, action.args, true); updateToolCall(contentPartsState, tcId, { result: { __interrupt__: true, diff --git a/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/DesktopShortcutsContent.tsx b/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/DesktopShortcutsContent.tsx index 6207457c4..12a7d00f0 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/DesktopShortcutsContent.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/DesktopShortcutsContent.tsx @@ -111,9 +111,7 @@ function HotkeyRow({ } > {recording ? ( - - Press hotkeys... - + Press hotkeys... ) : ( )} @@ -155,7 +153,9 @@ export function DesktopShortcutsContent() { if (!api) { return (
-

Hotkeys are only available in the SurfSense desktop app.

+

+ Hotkeys are only available in the SurfSense desktop app. +

); } @@ -178,28 +178,26 @@ export function DesktopShortcutsContent() { updateShortcut(key, DEFAULT_SHORTCUTS[key]); }; - return ( - shortcutsLoaded ? ( -
-
- {HOTKEY_ROWS.map((row) => ( - updateShortcut(row.key, accel)} - onReset={() => resetShortcut(row.key)} - /> - ))} -
+ return shortcutsLoaded ? ( +
+
+ {HOTKEY_ROWS.map((row) => ( + updateShortcut(row.key, accel)} + onReset={() => resetShortcut(row.key)} + /> + ))}
- ) : ( -
- -
- ) +
+ ) : ( +
+ +
); } diff --git a/surfsense_web/app/desktop/login/page.tsx b/surfsense_web/app/desktop/login/page.tsx index 451143949..c64eb65f8 100644 --- a/surfsense_web/app/desktop/login/page.tsx +++ b/surfsense_web/app/desktop/login/page.tsx @@ -24,7 +24,12 @@ const isGoogleAuth = AUTH_TYPE === "GOOGLE"; type ShortcutKey = "generalAssist" | "quickAsk" | "autocomplete"; type ShortcutMap = typeof DEFAULT_SHORTCUTS; -const HOTKEY_ROWS: Array<{ key: ShortcutKey; label: string; description: string; icon: React.ElementType }> = [ +const HOTKEY_ROWS: Array<{ + key: ShortcutKey; + label: string; + description: string; + icon: React.ElementType; +}> = [ { key: "generalAssist", label: "General Assist", @@ -369,7 +374,9 @@ export default function DesktopLoginPage() { )} diff --git a/surfsense_web/components/assistant-ui/connector-popup.tsx b/surfsense_web/components/assistant-ui/connector-popup.tsx index 66333a9ef..32943142a 100644 --- a/surfsense_web/components/assistant-ui/connector-popup.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup.tsx @@ -123,9 +123,9 @@ export const ConnectorIndicator = forwardRef ) : viewingMCPList ? ( - handleDisconnectFromList(connector, () => refreshConnectors())} - onAddAccount={handleAddNewMCPFromList} - addButtonText="Add New MCP Server" - /> + + handleDisconnectFromList(connector, () => refreshConnectors()) + } + onAddAccount={handleAddNewMCPFromList} + addButtonText="Add New MCP Server" + /> ) : viewingAccountsType ? ( - handleDisconnectFromList(connector, () => refreshConnectors())} - onAddAccount={() => { + + handleDisconnectFromList(connector, () => refreshConnectors()) + } + onAddAccount={() => { // Check both OAUTH_CONNECTORS and COMPOSIO_CONNECTORS const oauthConnector = OAUTH_CONNECTORS.find( diff --git a/surfsense_web/components/assistant-ui/connector-popup/connect-forms/components/mcp-connect-form.tsx b/surfsense_web/components/assistant-ui/connector-popup/connect-forms/components/mcp-connect-form.tsx index fc9812240..d9a740af2 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connect-forms/components/mcp-connect-form.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connect-forms/components/mcp-connect-form.tsx @@ -213,13 +213,13 @@ export const MCPConnectForm: FC = ({ onSubmit, isSubmitting }) className="w-full h-8 text-[13px] px-3 rounded-lg font-medium bg-white text-slate-700 hover:bg-slate-50 border-0 shadow-xs dark:bg-secondary dark:text-secondary-foreground dark:hover:bg-secondary/80" > {isTesting ? ( - <> - - Testing Connection... - - ) : ( - "Test Connection" - )} + <> + + Testing Connection... + + ) : ( + "Test Connection" + )}
diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/mcp-config.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/mcp-config.tsx index d6f60e824..97b5de675 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/mcp-config.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/mcp-config.tsx @@ -218,13 +218,13 @@ export const MCPConfig: FC = ({ connector, onConfigChange, onNam className="w-full h-8 text-[13px] px-3 rounded-lg font-medium bg-white text-slate-700 hover:bg-slate-50 border-0 shadow-xs dark:bg-secondary dark:text-secondary-foreground dark:hover:bg-secondary/80" > {isTesting ? ( - <> - - Testing Connection... - - ) : ( - "Test Connection" - )} + <> + + Testing Connection... + + ) : ( + "Test Connection" + )} diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/teams-config.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/teams-config.tsx index e96ddfd29..06ce21dae 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/teams-config.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/components/teams-config.tsx @@ -18,9 +18,9 @@ export const TeamsConfig: FC = () => {

Microsoft Teams Access

- Your agent can search and read messages from Teams channels you have access to, - and send messages on your behalf. Make sure you're a member of the teams - you want to interact with. + Your agent can search and read messages from Teams channels you have access to, and send + messages on your behalf. Make sure you're a member of the teams you want to interact + with.

diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-edit-view.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-edit-view.tsx index b2b40dfd6..c104f140a 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-edit-view.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/connector-edit-view.tsx @@ -16,7 +16,7 @@ import { DateRangeSelector } from "../../components/date-range-selector"; import { PeriodicSyncConfig } from "../../components/periodic-sync-config"; import { SummaryConfig } from "../../components/summary-config"; import { VisionLLMConfig } from "../../components/vision-llm-config"; -import { LIVE_CONNECTOR_TYPES, getReauthEndpoint } from "../../constants/connector-constants"; +import { getReauthEndpoint, LIVE_CONNECTOR_TYPES } from "../../constants/connector-constants"; import { getConnectorDisplayName } from "../../tabs/all-connectors-tab"; import { MCPServiceConfig } from "../components/mcp-service-config"; import { getConnectorConfigComponent } from "../index"; @@ -314,8 +314,7 @@ export const ConnectorEditView: FC = ({ {connector.is_indexable && (() => { - const isGoogleDrive = - connector.connector_type === "GOOGLE_DRIVE_CONNECTOR"; + const isGoogleDrive = connector.connector_type === "GOOGLE_DRIVE_CONNECTOR"; const isComposioGoogleDrive = connector.connector_type === "COMPOSIO_GOOGLE_DRIVE_CONNECTOR"; const requiresFolderSelection = isGoogleDrive || isComposioGoogleDrive; @@ -327,8 +326,7 @@ export const ConnectorEditView: FC = ({ (connector.config?.selected_files as | Array<{ id: string; name: string }> | undefined) || []; - const hasItemsSelected = - selectedFolders.length > 0 || selectedFiles.length > 0; + const hasItemsSelected = selectedFolders.length > 0 || selectedFiles.length > 0; const isDisabled = requiresFolderSelection && !hasItemsSelected; return ( @@ -380,8 +378,8 @@ export const ConnectorEditView: FC = ({ {/* Fixed Footer - Action buttons */}
- {showDisconnectConfirm ? ( -
+ {showDisconnectConfirm ? ( +
{isLive ? "Your agent will lose access to this service." diff --git a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/indexing-configuration-view.tsx b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/indexing-configuration-view.tsx index 690333523..982b0be11 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/indexing-configuration-view.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connector-configs/views/indexing-configuration-view.tsx @@ -12,7 +12,10 @@ import { DateRangeSelector } from "../../components/date-range-selector"; import { PeriodicSyncConfig } from "../../components/periodic-sync-config"; import { SummaryConfig } from "../../components/summary-config"; import { VisionLLMConfig } from "../../components/vision-llm-config"; -import { LIVE_CONNECTOR_TYPES, type IndexingConfigState } from "../../constants/connector-constants"; +import { + type IndexingConfigState, + LIVE_CONNECTOR_TYPES, +} from "../../constants/connector-constants"; import { getConnectorDisplayName } from "../../tabs/all-connectors-tab"; import { getConnectorConfigComponent } from "../index"; diff --git a/surfsense_web/components/assistant-ui/connector-popup/tabs/active-connectors-tab.tsx b/surfsense_web/components/assistant-ui/connector-popup/tabs/active-connectors-tab.tsx index fe9aab14f..755086ba5 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/tabs/active-connectors-tab.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/tabs/active-connectors-tab.tsx @@ -9,7 +9,11 @@ import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; import type { SearchSourceConnector } from "@/contracts/types/connector.types"; import { getDocumentTypeLabel } from "@/lib/documents/document-type-labels"; import { cn } from "@/lib/utils"; -import { COMPOSIO_CONNECTORS, LIVE_CONNECTOR_TYPES, OAUTH_CONNECTORS } from "../constants/connector-constants"; +import { + COMPOSIO_CONNECTORS, + LIVE_CONNECTOR_TYPES, + OAUTH_CONNECTORS, +} from "../constants/connector-constants"; import { getDocumentCountForConnector } from "../utils/connector-document-mapping"; import { getConnectorDisplayName } from "./all-connectors-tab"; diff --git a/surfsense_web/components/assistant-ui/connector-popup/views/connector-accounts-list-view.tsx b/surfsense_web/components/assistant-ui/connector-popup/views/connector-accounts-list-view.tsx index b3c087599..8aee7e005 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/views/connector-accounts-list-view.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/views/connector-accounts-list-view.tsx @@ -13,7 +13,7 @@ import type { SearchSourceConnector } from "@/contracts/types/connector.types"; import { authenticatedFetch } from "@/lib/auth-utils"; import { formatRelativeDate } from "@/lib/format-date"; import { cn } from "@/lib/utils"; -import { LIVE_CONNECTOR_TYPES, getReauthEndpoint } from "../constants/connector-constants"; +import { getReauthEndpoint, LIVE_CONNECTOR_TYPES } from "../constants/connector-constants"; import { useConnectorStatus } from "../hooks/use-connector-status"; import { getConnectorDisplayName } from "../tabs/all-connectors-tab"; @@ -182,11 +182,14 @@ export const ConnectorAccountsListView: FC = ({
) : (
- {typeConnectors.map((connector) => { - const isIndexing = indexingConnectorIds.has(connector.id); - const connectorReauthEndpoint = getReauthEndpoint(connector); - const isAuthExpired = !!connectorReauthEndpoint && connector.config?.auth_expired === true; - const isLive = LIVE_CONNECTOR_TYPES.has(connector.connector_type) || Boolean(connector.config?.server_config); + {typeConnectors.map((connector) => { + const isIndexing = indexingConnectorIds.has(connector.id); + const connectorReauthEndpoint = getReauthEndpoint(connector); + const isAuthExpired = + !!connectorReauthEndpoint && connector.config?.auth_expired === true; + const isLive = + LIVE_CONNECTOR_TYPES.has(connector.connector_type) || + Boolean(connector.config?.server_config); return (
= ({

) : null}
- {isAuthExpired ? ( - - ) : isLive && onDisconnect ? ( - confirmDisconnectId === connector.id ? ( -
+ {isAuthExpired ? ( + + ) : isLive && onDisconnect ? ( + confirmDisconnectId === connector.id ? ( +
+ + +
+ ) : ( - -
+ ) ) : ( - ) - ) : ( - - )} + )}
); })} diff --git a/surfsense_web/components/assistant-ui/markdown-text.tsx b/surfsense_web/components/assistant-ui/markdown-text.tsx index 2707e8956..8bb228580 100644 --- a/surfsense_web/components/assistant-ui/markdown-text.tsx +++ b/surfsense_web/components/assistant-ui/markdown-text.tsx @@ -20,7 +20,6 @@ import { openEditorPanelAtom } from "@/atoms/editor/editor-panel.atom"; import { ImagePreview, ImageRoot, ImageZoom } from "@/components/assistant-ui/image"; import "katex/dist/katex.min.css"; import { InlineCitation, UrlCitation } from "@/components/assistant-ui/inline-citation"; -import { useElectronAPI } from "@/hooks/use-platform"; import { Skeleton } from "@/components/ui/skeleton"; import { Table, @@ -30,6 +29,7 @@ import { TableHeader, TableRow, } from "@/components/ui/table"; +import { useElectronAPI } from "@/hooks/use-platform"; import { cn } from "@/lib/utils"; function MarkdownCodeBlockSkeleton() { @@ -493,10 +493,7 @@ const defaultComponents = memoizeMarkdownComponents({ const mounts = (await electronAPI.getAgentFilesystemMounts( resolvedSearchSpaceId )) as AgentFilesystemMount[]; - resolvedLocalPath = normalizeLocalVirtualPathForEditor( - inlineValue, - mounts - ); + resolvedLocalPath = normalizeLocalVirtualPathForEditor(inlineValue, mounts); } catch { // Fall back to the raw inline path if mount lookup fails. } diff --git a/surfsense_web/components/editor-panel/editor-panel.tsx b/surfsense_web/components/editor-panel/editor-panel.tsx index 2fa980d27..3b69ae6e0 100644 --- a/surfsense_web/components/editor-panel/editor-panel.tsx +++ b/surfsense_web/components/editor-panel/editor-panel.tsx @@ -248,7 +248,15 @@ export function EditorPanelContent({ doFetch().catch(() => {}); return () => controller.abort(); - }, [documentId, electronAPI, isLocalFileMode, localFilePath, resolveLocalVirtualPath, searchSpaceId, title]); + }, [ + documentId, + electronAPI, + isLocalFileMode, + localFilePath, + resolveLocalVirtualPath, + searchSpaceId, + title, + ]); useEffect(() => { return () => { @@ -282,69 +290,77 @@ export function EditorPanelContent({ } }, [editorDoc?.source_markdown]); - const handleSave = useCallback(async (_options?: { silent?: boolean }) => { - setSaving(true); - try { - if (isLocalFileMode) { - if (!localFilePath) { - throw new Error("Missing local file path"); + const handleSave = useCallback( + async (_options?: { silent?: boolean }) => { + setSaving(true); + try { + if (isLocalFileMode) { + if (!localFilePath) { + throw new Error("Missing local file path"); + } + if (!electronAPI?.writeAgentLocalFileText) { + throw new Error("Local file editor is available only in desktop mode."); + } + const resolvedLocalPath = await resolveLocalVirtualPath(localFilePath); + const contentToSave = markdownRef.current; + const writeResult = await electronAPI.writeAgentLocalFileText( + resolvedLocalPath, + contentToSave, + searchSpaceId + ); + if (!writeResult.ok) { + throw new Error(writeResult.error || "Failed to save local file"); + } + setEditorDoc((prev) => (prev ? { ...prev, source_markdown: contentToSave } : prev)); + setEditedMarkdown(markdownRef.current === contentToSave ? null : markdownRef.current); + return true; } - if (!electronAPI?.writeAgentLocalFileText) { - throw new Error("Local file editor is available only in desktop mode."); + if (!searchSpaceId || !documentId) { + throw new Error("Missing document context"); } - const resolvedLocalPath = await resolveLocalVirtualPath(localFilePath); - const contentToSave = markdownRef.current; - const writeResult = await electronAPI.writeAgentLocalFileText( - resolvedLocalPath, - contentToSave, - searchSpaceId + const token = getBearerToken(); + if (!token) { + toast.error("Please login to save"); + redirectToLogin(); + return; + } + const response = await authenticatedFetch( + `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-spaces/${searchSpaceId}/documents/${documentId}/save`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ source_markdown: markdownRef.current }), + } ); - if (!writeResult.ok) { - throw new Error(writeResult.error || "Failed to save local file"); + + if (!response.ok) { + const errorData = await response + .json() + .catch(() => ({ detail: "Failed to save document" })); + throw new Error(errorData.detail || "Failed to save document"); } - setEditorDoc((prev) => - prev ? { ...prev, source_markdown: contentToSave } : prev - ); - setEditedMarkdown(markdownRef.current === contentToSave ? null : markdownRef.current); + + setEditorDoc((prev) => (prev ? { ...prev, source_markdown: markdownRef.current } : prev)); + setEditedMarkdown(null); + toast.success("Document saved! Reindexing in background..."); return true; + } catch (err) { + console.error("Error saving document:", err); + toast.error(err instanceof Error ? err.message : "Failed to save document"); + return false; + } finally { + setSaving(false); } - if (!searchSpaceId || !documentId) { - throw new Error("Missing document context"); - } - const token = getBearerToken(); - if (!token) { - toast.error("Please login to save"); - redirectToLogin(); - return; - } - const response = await authenticatedFetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-spaces/${searchSpaceId}/documents/${documentId}/save`, - { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ source_markdown: markdownRef.current }), - } - ); - - if (!response.ok) { - const errorData = await response - .json() - .catch(() => ({ detail: "Failed to save document" })); - throw new Error(errorData.detail || "Failed to save document"); - } - - setEditorDoc((prev) => (prev ? { ...prev, source_markdown: markdownRef.current } : prev)); - setEditedMarkdown(null); - toast.success("Document saved! Reindexing in background..."); - return true; - } catch (err) { - console.error("Error saving document:", err); - toast.error(err instanceof Error ? err.message : "Failed to save document"); - return false; - } finally { - setSaving(false); - } - }, [documentId, electronAPI, isLocalFileMode, localFilePath, resolveLocalVirtualPath, searchSpaceId]); + }, + [ + documentId, + electronAPI, + isLocalFileMode, + localFilePath, + resolveLocalVirtualPath, + searchSpaceId, + ] + ); const isEditableType = editorDoc ? (editorRenderMode === "source_code" || @@ -594,9 +610,7 @@ export function EditorPanelContent({ } }} > - + Download .md @@ -626,7 +640,7 @@ export function EditorPanelContent({
) : isEditableType ? ( ; } diff --git a/surfsense_web/components/editor/plugins/fixed-toolbar-kit.tsx b/surfsense_web/components/editor/plugins/fixed-toolbar-kit.tsx index bdda0263d..346fe0378 100644 --- a/surfsense_web/components/editor/plugins/fixed-toolbar-kit.tsx +++ b/surfsense_web/components/editor/plugins/fixed-toolbar-kit.tsx @@ -1,7 +1,6 @@ "use client"; -import { createPlatePlugin } from "platejs/react"; -import { useEditorReadOnly } from "platejs/react"; +import { createPlatePlugin, useEditorReadOnly } from "platejs/react"; import { useEditorSave } from "@/components/editor/editor-save-context"; import { FixedToolbar } from "@/components/ui/fixed-toolbar"; diff --git a/surfsense_web/components/editor/source-code-editor.tsx b/surfsense_web/components/editor/source-code-editor.tsx index dd4b3bd8e..9102dffe9 100644 --- a/surfsense_web/components/editor/source-code-editor.tsx +++ b/surfsense_web/components/editor/source-code-editor.tsx @@ -1,8 +1,8 @@ "use client"; import dynamic from "next/dynamic"; -import { useEffect, useRef } from "react"; import { useTheme } from "next-themes"; +import { useEffect, useRef } from "react"; import { Spinner } from "@/components/ui/spinner"; const MonacoEditor = dynamic(() => import("@monaco-editor/react"), { diff --git a/surfsense_web/components/layout/ui/right-panel/RightPanel.tsx b/surfsense_web/components/layout/ui/right-panel/RightPanel.tsx index c26cc9b23..04bae010c 100644 --- a/surfsense_web/components/layout/ui/right-panel/RightPanel.tsx +++ b/surfsense_web/components/layout/ui/right-panel/RightPanel.tsx @@ -72,9 +72,7 @@ export function RightPanelExpandButton() { const reportOpen = reportState.isOpen && !!reportState.reportId; const editorOpen = editorState.isOpen && - (editorState.kind === "document" - ? !!editorState.documentId - : !!editorState.localFilePath); + (editorState.kind === "document" ? !!editorState.documentId : !!editorState.localFilePath); const hitlEditOpen = hitlEditState.isOpen && !!hitlEditState.onSave; const hasContent = documentsOpen || reportOpen || editorOpen || hitlEditOpen; @@ -116,9 +114,7 @@ export function RightPanel({ documentsPanel }: RightPanelProps) { const reportOpen = reportState.isOpen && !!reportState.reportId; const editorOpen = editorState.isOpen && - (editorState.kind === "document" - ? !!editorState.documentId - : !!editorState.localFilePath); + (editorState.kind === "document" ? !!editorState.documentId : !!editorState.localFilePath); const hitlEditOpen = hitlEditState.isOpen && !!hitlEditState.onSave; useEffect(() => { diff --git a/surfsense_web/components/layout/ui/sidebar/DesktopLocalTabContent.tsx b/surfsense_web/components/layout/ui/sidebar/DesktopLocalTabContent.tsx index dd7520d24..cd8fca331 100644 --- a/surfsense_web/components/layout/ui/sidebar/DesktopLocalTabContent.tsx +++ b/surfsense_web/components/layout/ui/sidebar/DesktopLocalTabContent.tsx @@ -1,11 +1,9 @@ "use client"; -import { Folder, FolderPlus, Search, X } from "lucide-react"; import { useAtom } from "jotai"; +import { Folder, FolderPlus, Search, X } from "lucide-react"; import { useCallback, useMemo, useRef, useState } from "react"; import { localExpandedFolderKeysAtom } from "@/atoms/documents/folder.atoms"; -import { Input } from "@/components/ui/input"; -import { Separator } from "@/components/ui/separator"; import { DropdownMenu, DropdownMenuContent, @@ -14,6 +12,8 @@ import { DropdownMenuSeparator, DropdownMenuTrigger, } from "@/components/ui/dropdown-menu"; +import { Input } from "@/components/ui/input"; +import { Separator } from "@/components/ui/separator"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; import { useDebouncedValue } from "@/hooks/use-debounced-value"; import { LocalFilesystemBrowser } from "./LocalFilesystemBrowser"; diff --git a/surfsense_web/components/layout/ui/sidebar/DocumentsSidebar.tsx b/surfsense_web/components/layout/ui/sidebar/DocumentsSidebar.tsx index b9c174d71..0a147f7b7 100644 --- a/surfsense_web/components/layout/ui/sidebar/DocumentsSidebar.tsx +++ b/surfsense_web/components/layout/ui/sidebar/DocumentsSidebar.tsx @@ -71,7 +71,7 @@ import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; import type { DocumentTypeEnum } from "@/contracts/types/document.types"; import { useDebouncedValue } from "@/hooks/use-debounced-value"; import { useMediaQuery } from "@/hooks/use-media-query"; -import { usePlatform, useElectronAPI } from "@/hooks/use-platform"; +import { useElectronAPI, usePlatform } from "@/hooks/use-platform"; import { anonymousChatApiService } from "@/lib/apis/anonymous-chat-api.service"; import { documentsApiService } from "@/lib/apis/documents-api.service"; import { foldersApiService } from "@/lib/apis/folders-api.service"; @@ -208,7 +208,8 @@ function AuthenticatedDocumentsSidebarBase({ const [watchedFolderIds, setWatchedFolderIds] = useState>(new Set()); const [folderWatchOpen, setFolderWatchOpen] = useAtom(folderWatchDialogOpenAtom); const [watchInitialFolder, setWatchInitialFolder] = useAtom(folderWatchInitialFolderAtom); - const isElectron = desktopFeaturesEnabled && typeof window !== "undefined" && !!window.electronAPI; + const isElectron = + desktopFeaturesEnabled && typeof window !== "undefined" && !!window.electronAPI; useEffect(() => { if (!electronAPI?.getAgentFilesystemSettings) return; @@ -250,10 +251,13 @@ function AuthenticatedDocumentsSidebarBase({ .filter((rootPath, index, allPaths) => allPaths.indexOf(rootPath) === index) .slice(0, MAX_LOCAL_FILESYSTEM_ROOTS); if (nextLocalRootPaths.length === localRootPaths.length) return; - const updated = await electronAPI.setAgentFilesystemSettings({ - mode: "desktop_local_folder", - localRootPaths: nextLocalRootPaths, - }, searchSpaceId); + const updated = await electronAPI.setAgentFilesystemSettings( + { + mode: "desktop_local_folder", + localRootPaths: nextLocalRootPaths, + }, + searchSpaceId + ); setFilesystemSettings(updated); }, [electronAPI, localRootPaths, searchSpaceId] @@ -282,10 +286,13 @@ function AuthenticatedDocumentsSidebarBase({ const handleRemoveFilesystemRoot = useCallback( async (rootPathToRemove: string) => { if (!electronAPI?.setAgentFilesystemSettings) return; - const updated = await electronAPI.setAgentFilesystemSettings({ - mode: "desktop_local_folder", - localRootPaths: localRootPaths.filter((rootPath) => rootPath !== rootPathToRemove), - }, searchSpaceId); + const updated = await electronAPI.setAgentFilesystemSettings( + { + mode: "desktop_local_folder", + localRootPaths: localRootPaths.filter((rootPath) => rootPath !== rootPathToRemove), + }, + searchSpaceId + ); setFilesystemSettings(updated); }, [electronAPI, localRootPaths, searchSpaceId] @@ -293,19 +300,25 @@ function AuthenticatedDocumentsSidebarBase({ const handleClearFilesystemRoots = useCallback(async () => { if (!electronAPI?.setAgentFilesystemSettings) return; - const updated = await electronAPI.setAgentFilesystemSettings({ - mode: "desktop_local_folder", - localRootPaths: [], - }, searchSpaceId); + const updated = await electronAPI.setAgentFilesystemSettings( + { + mode: "desktop_local_folder", + localRootPaths: [], + }, + searchSpaceId + ); setFilesystemSettings(updated); }, [electronAPI, searchSpaceId]); const handleFilesystemTabChange = useCallback( async (tab: "cloud" | "local") => { if (!electronAPI?.setAgentFilesystemSettings) return; - const updated = await electronAPI.setAgentFilesystemSettings({ - mode: tab === "cloud" ? "cloud" : "desktop_local_folder", - }, searchSpaceId); + const updated = await electronAPI.setAgentFilesystemSettings( + { + mode: tab === "cloud" ? "cloud" : "desktop_local_folder", + }, + searchSpaceId + ); setFilesystemSettings(updated); }, [electronAPI, searchSpaceId] @@ -552,7 +565,9 @@ function AuthenticatedDocumentsSidebarBase({ if (!electronAPI) return; const watchedFolders = (await electronAPI.getWatchedFolders()) as WatchedFolderEntry[]; - const matched = watchedFolders.find((wf: WatchedFolderEntry) => wf.rootFolderId === folder.id); + const matched = watchedFolders.find( + (wf: WatchedFolderEntry) => wf.rootFolderId === folder.id + ); if (!matched) { toast.error("This folder is not being watched"); return; @@ -582,7 +597,9 @@ function AuthenticatedDocumentsSidebarBase({ if (!electronAPI) return; const watchedFolders = (await electronAPI.getWatchedFolders()) as WatchedFolderEntry[]; - const matched = watchedFolders.find((wf: WatchedFolderEntry) => wf.rootFolderId === folder.id); + const matched = watchedFolders.find( + (wf: WatchedFolderEntry) => wf.rootFolderId === folder.id + ); if (!matched) { toast.error("This folder is not being watched"); return; @@ -1015,7 +1032,8 @@ function AuthenticatedDocumentsSidebarBase({ }, [open, onOpenChange, isMobile, setRightPanelCollapsed]); const showFilesystemTabs = !isMobile && !!electronAPI && !!filesystemSettings; - const currentFilesystemTab = filesystemSettings?.mode === "desktop_local_folder" ? "local" : "cloud"; + const currentFilesystemTab = + filesystemSettings?.mode === "desktop_local_folder" ? "local" : "cloud"; const showCloudSkeleton = currentFilesystemTab === "cloud" && (zeroFoldersResult.type !== "complete" || zeroAllDocsResult.type !== "complete"); @@ -1331,8 +1349,8 @@ function AuthenticatedDocumentsSidebarBase({ Trust this workspace? - Local mode can read and edit files inside the folders you select. Continue only if - you trust this workspace and its contents. + Local mode can read and edit files inside the folders you select. Continue only if you + trust this workspace and its contents. {pendingLocalPath && ( diff --git a/surfsense_web/components/layout/ui/sidebar/LocalFilesystemBrowser.tsx b/surfsense_web/components/layout/ui/sidebar/LocalFilesystemBrowser.tsx index 6bfb1d3f1..19c47d605 100644 --- a/surfsense_web/components/layout/ui/sidebar/LocalFilesystemBrowser.tsx +++ b/surfsense_web/components/layout/ui/sidebar/LocalFilesystemBrowser.tsx @@ -141,7 +141,9 @@ export function LocalFilesystemBrowser({ }: LocalFilesystemBrowserProps) { const electronAPI = useElectronAPI(); const [rootStateMap, setRootStateMap] = useState>({}); - const [internalExpandedFolderKeys, setInternalExpandedFolderKeys] = useState>(new Set()); + const [internalExpandedFolderKeys, setInternalExpandedFolderKeys] = useState>( + new Set() + ); const [mountByRootKey, setMountByRootKey] = useState>(new Map()); const [mountStatus, setMountStatus] = useState("idle"); const [mountRefreshInFlight, setMountRefreshInFlight] = useState(false); @@ -188,10 +190,7 @@ export function LocalFilesystemBrowser({ } for (const { rootKey } of rootsToReload) { const nonce = reloadNonceByRoot[rootKey] ?? 0; - lastLoadedSignatureByRootRef.current.set( - rootKey, - `${searchSpaceId}:${rootKey}:${nonce}` - ); + lastLoadedSignatureByRootRef.current.set(rootKey, `${searchSpaceId}:${rootKey}:${nonce}`); } let cancelled = false; @@ -257,35 +256,37 @@ export function LocalFilesystemBrowser({ return; } - const unsubscribe = electronAPI.onAgentFilesystemTreeDirty((event: { - searchSpaceId: number | null; - reason: "watcher_event" | "safety_poll"; - rootPath: string; - changedPath: string | null; - timestamp: number; - }) => { - if ((event.searchSpaceId ?? null) !== (searchSpaceId ?? null)) { - return; + const unsubscribe = electronAPI.onAgentFilesystemTreeDirty( + (event: { + searchSpaceId: number | null; + reason: "watcher_event" | "safety_poll"; + rootPath: string; + changedPath: string | null; + timestamp: number; + }) => { + if ((event.searchSpaceId ?? null) !== (searchSpaceId ?? null)) { + return; + } + const eventRootKey = normalizeRootPathForLookup(event.rootPath, isWindowsPlatform); + const knownRootKeys = new Set( + rootPaths.map((rootPath) => normalizeRootPathForLookup(rootPath, isWindowsPlatform)) + ); + if (!knownRootKeys.has(eventRootKey)) { + setReloadNonceByRoot((prev) => { + const next = { ...prev }; + for (const rootKey of knownRootKeys) { + next[rootKey] = (prev[rootKey] ?? 0) + 1; + } + return next; + }); + return; + } + setReloadNonceByRoot((prev) => ({ + ...prev, + [eventRootKey]: (prev[eventRootKey] ?? 0) + 1, + })); } - const eventRootKey = normalizeRootPathForLookup(event.rootPath, isWindowsPlatform); - const knownRootKeys = new Set( - rootPaths.map((rootPath) => normalizeRootPathForLookup(rootPath, isWindowsPlatform)) - ); - if (!knownRootKeys.has(eventRootKey)) { - setReloadNonceByRoot((prev) => { - const next = { ...prev }; - for (const rootKey of knownRootKeys) { - next[rootKey] = (prev[rootKey] ?? 0) + 1; - } - return next; - }); - return; - } - setReloadNonceByRoot((prev) => ({ - ...prev, - [eventRootKey]: (prev[eventRootKey] ?? 0) + 1, - })); - }); + ); void electronAPI.startAgentFilesystemTreeWatch({ searchSpaceId, rootPaths, @@ -378,22 +379,25 @@ export function LocalFilesystemBrowser({ }); }, [rootPaths, rootStateMap, searchQuery]); - const toggleFolder = useCallback((folderKey: string) => { - const update = (prev: Set) => { - const next = new Set(prev); - if (next.has(folderKey)) { - next.delete(folderKey); - } else { - next.add(folderKey); + const toggleFolder = useCallback( + (folderKey: string) => { + const update = (prev: Set) => { + const next = new Set(prev); + if (next.has(folderKey)) { + next.delete(folderKey); + } else { + next.add(folderKey); + } + return next; + }; + if (onExpandedFolderKeysChange) { + onExpandedFolderKeysChange(update(effectiveExpandedFolderKeys)); + return; } - return next; - }; - if (onExpandedFolderKeysChange) { - onExpandedFolderKeysChange(update(effectiveExpandedFolderKeys)); - return; - } - setInternalExpandedFolderKeys(update); - }, [effectiveExpandedFolderKeys, onExpandedFolderKeysChange]); + setInternalExpandedFolderKeys(update); + }, + [effectiveExpandedFolderKeys, onExpandedFolderKeysChange] + ); const renderFolder = useCallback( (folder: LocalFolderNode, depth: number, mount: string) => { @@ -436,9 +440,7 @@ export function LocalFilesystemBrowser({ : undefined } className={`flex h-8 w-full items-center gap-1.5 rounded-md px-2 text-left text-sm transition-colors ${ - isOpenable - ? "hover:bg-muted/60" - : "cursor-not-allowed opacity-60" + isOpenable ? "hover:bg-muted/60" : "cursor-not-allowed opacity-60" }`} style={{ paddingInlineStart: `${(depth + 1) * 12 + 22}px` }} title={ @@ -528,7 +530,10 @@ export function LocalFilesystemBrowser({ } if (state.error) { return ( -
+

Failed to load local folder

{state.error}

diff --git a/surfsense_web/components/layout/ui/tabs/DocumentTabContent.tsx b/surfsense_web/components/layout/ui/tabs/DocumentTabContent.tsx index 77668a93d..ac5463873 100644 --- a/surfsense_web/components/layout/ui/tabs/DocumentTabContent.tsx +++ b/surfsense_web/components/layout/ui/tabs/DocumentTabContent.tsx @@ -308,9 +308,7 @@ export function DocumentTabContent({ documentId, searchSpaceId, title }: Documen } }} > - + Download .md diff --git a/surfsense_web/components/new-chat/model-selector.tsx b/surfsense_web/components/new-chat/model-selector.tsx index 3f5a5fa8c..9fe9dd8da 100644 --- a/surfsense_web/components/new-chat/model-selector.tsx +++ b/surfsense_web/components/new-chat/model-selector.tsx @@ -8,9 +8,9 @@ import { ChevronLeft, ChevronRight, ChevronUp, - Pencil, ImageIcon, Layers, + Pencil, Plus, ScanEye, Search, @@ -741,9 +741,7 @@ export function ModelSelector({
{!isMobile && ( @@ -769,9 +767,7 @@ export function ModelSelector({
diff --git a/surfsense_web/components/report-panel/report-panel.tsx b/surfsense_web/components/report-panel/report-panel.tsx index ede63d902..621cf13ce 100644 --- a/surfsense_web/components/report-panel/report-panel.tsx +++ b/surfsense_web/components/report-panel/report-panel.tsx @@ -398,7 +398,8 @@ export function ReportPanelContent({ ); - const editingActions = showReportEditingTier && + const editingActions = + showReportEditingTier && !isReadOnly && (isEditing ? ( <> diff --git a/surfsense_web/components/settings/agent-model-manager.tsx b/surfsense_web/components/settings/agent-model-manager.tsx index 988befdd0..a0b700c2d 100644 --- a/surfsense_web/components/settings/agent-model-manager.tsx +++ b/surfsense_web/components/settings/agent-model-manager.tsx @@ -1,15 +1,7 @@ "use client"; import { useAtomValue } from "jotai"; -import { - AlertCircle, - Dot, - FileText, - Info, - Pencil, - RefreshCw, - Trash2, -} from "lucide-react"; +import { AlertCircle, Dot, FileText, Info, Pencil, RefreshCw, Trash2 } from "lucide-react"; import { useMemo, useState } from "react"; import { membersAtom, myAccessAtom } from "@/atoms/members/members-query.atoms"; import { deleteNewLLMConfigMutationAtom } from "@/atoms/new-llm-config/new-llm-config-mutation.atoms"; diff --git a/surfsense_web/components/settings/roles-manager.tsx b/surfsense_web/components/settings/roles-manager.tsx index e7dadc20f..335cfc8a9 100644 --- a/surfsense_web/components/settings/roles-manager.tsx +++ b/surfsense_web/components/settings/roles-manager.tsx @@ -5,10 +5,8 @@ import { useAtomValue } from "jotai"; import { Bot, ChevronRight, - ScanEye, - Pencil, - FileText, Earth, + FileText, Image, Logs, type LucideIcon, @@ -16,11 +14,13 @@ import { MessageSquare, Mic, MoreHorizontal, - Unplug, + Pencil, + ScanEye, Settings, Shield, SlidersHorizontal, Trash2, + Unplug, Users, Video, } from "lucide-react"; @@ -462,9 +462,19 @@ function RolesContent({ return (
+ {/* biome-ignore lint/a11y/useSemanticElements: row contains nested interactive elements (DropdownMenu); using a + )} +
+ + {showForm && ( +
+
+

New permission rule

+ +
+
+ + setFormData((p) => ({ ...p, permission: e.target.value }))} + /> +

+ Match a tool capability. Use * for wildcards. +

+
+ +
+ + setFormData((p) => ({ ...p, pattern: e.target.value }))} + /> +

+ Wildcard against the canonical argument (e.g. prod-*). +

+
+
+ +
+ + +

+ {ACTION_DESCRIPTIONS[formData.action]} +

+
+ +
+ + +
+
+
+ )} + + {sortedRules.length === 0 && !showForm && ( +
+ +

No rules yet

+

+ Without rules the agent uses the deployment default for every tool. +

+
+ )} + + {sortedRules.length > 0 && ( +
+ {sortedRules.map((rule) => { + const badge = ACTION_BADGE[rule.action]; + const isUpdating = + updateMutation.isPending && updateMutation.variables?.ruleId === rule.id; + const isDeleting = deleteMutation.isPending && deleteMutation.variables === rule.id; + + return ( +
+
+
+
+ + {rule.permission} + + {rule.pattern !== "*" && ( + + → {rule.pattern} + + )} + +
+

+ Created {formatRelativeDate(rule.created_at)} +

+
+ +
+ + + +
+
+
+ ); + })} +
+ )} + + !open && setDeleteTarget(null)} + > + + + Delete this rule? + + The agent will fall back to deployment defaults for matching tool calls. + + + + Cancel + { + e.preventDefault(); + handleConfirmDelete(); + }} + disabled={deleteMutation.isPending} + > + {deleteMutation.isPending ? "Deleting…" : "Delete"} + + + + +
+ ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/AgentStatusContent.tsx b/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/AgentStatusContent.tsx new file mode 100644 index 000000000..bd8f03a70 --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/user-settings/components/AgentStatusContent.tsx @@ -0,0 +1,309 @@ +"use client"; + +import { useAtomValue } from "jotai"; +import { CircleCheck, CircleSlash, Cog, RotateCcw } from "lucide-react"; +import { useMemo } from "react"; +import { agentFlagsAtom } from "@/atoms/agent/agent-flags-query.atom"; +import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; +import { Badge } from "@/components/ui/badge"; +import { Separator } from "@/components/ui/separator"; +import { Skeleton } from "@/components/ui/skeleton"; +import type { AgentFeatureFlags } from "@/lib/apis/agent-flags-api.service"; +import { cn } from "@/lib/utils"; + +type FlagKey = keyof AgentFeatureFlags; + +interface FlagDef { + key: FlagKey; + label: string; + description: string; + envVar: string; +} + +interface FlagGroup { + id: string; + title: string; + subtitle: string; + flags: FlagDef[]; +} + +const FLAG_GROUPS: FlagGroup[] = [ + { + id: "tier1", + title: "Tier 1 — Agent quality", + subtitle: "Context editing, retries, fallbacks, doom-loop, tool-call repair.", + flags: [ + { + key: "enable_context_editing", + label: "Context editing", + description: "Trim tool outputs and spill old text into backend storage.", + envVar: "SURFSENSE_ENABLE_CONTEXT_EDITING", + }, + { + key: "enable_compaction_v2", + label: "Compaction v2", + description: "SurfSense-aware compaction replacing safe summarization.", + envVar: "SURFSENSE_ENABLE_COMPACTION_V2", + }, + { + key: "enable_retry_after", + label: "Retry-After", + description: "Honour rate-limit retry-after headers automatically.", + envVar: "SURFSENSE_ENABLE_RETRY_AFTER", + }, + { + key: "enable_model_fallback", + label: "Model fallback", + description: "Fail over to a backup model on persistent errors.", + envVar: "SURFSENSE_ENABLE_MODEL_FALLBACK", + }, + { + key: "enable_model_call_limit", + label: "Model call limit", + description: "Cap total model calls per turn to prevent budget run-aways.", + envVar: "SURFSENSE_ENABLE_MODEL_CALL_LIMIT", + }, + { + key: "enable_tool_call_limit", + label: "Tool call limit", + description: "Cap total tool calls per turn.", + envVar: "SURFSENSE_ENABLE_TOOL_CALL_LIMIT", + }, + { + key: "enable_tool_call_repair", + label: "Tool-call name repair", + description: "Recover from lower-cased / fuzzy tool names emitted by smaller models.", + envVar: "SURFSENSE_ENABLE_TOOL_CALL_REPAIR", + }, + { + key: "enable_doom_loop", + label: "Doom-loop detection", + description: "Detect repeated identical tool calls and ask the user to confirm.", + envVar: "SURFSENSE_ENABLE_DOOM_LOOP", + }, + ], + }, + { + id: "tier2", + title: "Tier 2 — Safety", + subtitle: "Permission rules, busy-mutex, smarter tool selection.", + flags: [ + { + key: "enable_permission", + label: "Permission middleware", + description: "Apply allow/deny/ask rules from the Agent Permissions tab.", + envVar: "SURFSENSE_ENABLE_PERMISSION", + }, + { + key: "enable_busy_mutex", + label: "Busy mutex", + description: "Prevent two concurrent runs from corrupting the same thread.", + envVar: "SURFSENSE_ENABLE_BUSY_MUTEX", + }, + { + key: "enable_llm_tool_selector", + label: "LLM tool selector", + description: "Use a smaller model to pre-filter the tool list per turn.", + envVar: "SURFSENSE_ENABLE_LLM_TOOL_SELECTOR", + }, + ], + }, + { + id: "tier4", + title: "Tier 4 — Skills + subagents", + subtitle: "Built-in skills, specialized subagents, KB planner runnable.", + flags: [ + { + key: "enable_skills", + label: "Skills", + description: "Load on-demand skill packs (kb-research, report-writing, …).", + envVar: "SURFSENSE_ENABLE_SKILLS", + }, + { + key: "enable_specialized_subagents", + label: "Specialized subagents", + description: "Spin up explore / report_writer / connector_negotiator subagents.", + envVar: "SURFSENSE_ENABLE_SPECIALIZED_SUBAGENTS", + }, + { + key: "enable_kb_planner_runnable", + label: "KB planner runnable", + description: "Compile a private planner sub-agent for KB search.", + envVar: "SURFSENSE_ENABLE_KB_PLANNER_RUNNABLE", + }, + ], + }, + { + id: "tier5", + title: "Tier 5 — Audit + revert", + subtitle: "Action log + revert route used by the Agent Actions sheet.", + flags: [ + { + key: "enable_action_log", + label: "Action log", + description: "Persist every tool call to agent_action_log.", + envVar: "SURFSENSE_ENABLE_ACTION_LOG", + }, + { + key: "enable_revert_route", + label: "Revert route", + description: "Allow reverting reversible actions from the action log.", + envVar: "SURFSENSE_ENABLE_REVERT_ROUTE", + }, + ], + }, + { + id: "tier6", + title: "Tier 6 — Plugins", + subtitle: "Optional middleware loaded from entry points.", + flags: [ + { + key: "enable_plugin_loader", + label: "Plugin loader", + description: "Load surfsense.plugins entry-point middleware.", + envVar: "SURFSENSE_ENABLE_PLUGIN_LOADER", + }, + ], + }, + { + id: "obs", + title: "Observability", + subtitle: "Telemetry pipelines (orthogonal to feature gating).", + flags: [ + { + key: "enable_otel", + label: "OpenTelemetry", + description: "Emit OTel spans (also requires OTEL_EXPORTER_OTLP_ENDPOINT).", + envVar: "SURFSENSE_ENABLE_OTEL", + }, + ], + }, +]; + +function FlagRow({ def, value }: { def: FlagDef; value: boolean }) { + return ( +
+
+
+ {def.label} + + {def.envVar} + +
+

{def.description}

+
+ + {value ? : } + {value ? "On" : "Off"} + +
+ ); +} + +export function AgentStatusContent() { + const { data: flags, isLoading, isError, error, refetch } = useAtomValue(agentFlagsAtom); + + const enabledCount = useMemo(() => { + if (!flags) return 0; + return Object.entries(flags).filter(([k, v]) => k !== "disable_new_agent_stack" && v === true) + .length; + }, [flags]); + + if (isLoading) { + return ( +
+ + + +
+ ); + } + + if (isError || !flags) { + return ( + + Failed to load agent status + + {error instanceof Error ? error.message : "Unknown error."} + + + + ); + } + + const masterOff = flags.disable_new_agent_stack; + + return ( +
+ {masterOff ? ( + + + Master kill-switch is on + + + SURFSENSE_DISABLE_NEW_AGENT_STACK=true + + forces every new middleware off, regardless of the individual flags below. Restart the + backend after changing it. + + + ) : ( + + + + Agent stack + + {enabledCount} on + + + + Read-only mirror of the backend's AgentFeatureFlags. Flip an env var and + restart the backend to change a value. + + + )} + + {FLAG_GROUPS.map((group, groupIdx) => { + const allOff = group.flags.every((f) => !flags[f.key]); + return ( +
+ {groupIdx > 0 && } +
+
+
+

{group.title}

+

{group.subtitle}

+
+ {allOff && ( + + all off + + )} +
+
+ {group.flags.map((def) => ( + + ))} +
+
+
+ ); + })} +
+ ); +} diff --git a/surfsense_web/atoms/agent/action-log-sheet.atom.ts b/surfsense_web/atoms/agent/action-log-sheet.atom.ts new file mode 100644 index 000000000..f88d3ed1e --- /dev/null +++ b/surfsense_web/atoms/agent/action-log-sheet.atom.ts @@ -0,0 +1,19 @@ +import { atom } from "jotai"; + +interface ActionLogSheetState { + open: boolean; + threadId: number | null; +} + +export const actionLogSheetAtom = atom({ + open: false, + threadId: null, +}); + +export const openActionLogSheetAtom = atom(null, (_get, set, threadId: number) => { + set(actionLogSheetAtom, { open: true, threadId }); +}); + +export const closeActionLogSheetAtom = atom(null, (_get, set) => { + set(actionLogSheetAtom, { open: false, threadId: null }); +}); diff --git a/surfsense_web/atoms/agent/agent-flags-query.atom.ts b/surfsense_web/atoms/agent/agent-flags-query.atom.ts new file mode 100644 index 000000000..30158deaa --- /dev/null +++ b/surfsense_web/atoms/agent/agent-flags-query.atom.ts @@ -0,0 +1,17 @@ +import { atomWithQuery } from "jotai-tanstack-query"; +import { agentFlagsApiService } from "@/lib/apis/agent-flags-api.service"; +import { getBearerToken } from "@/lib/auth-utils"; + +export const AGENT_FLAGS_QUERY_KEY = ["agent", "flags"] as const; + +/** + * Reads the backend agent feature flags. Cached for the lifetime of the + * page (flags only change on backend restart) so we can drive UI gating + * without re-hitting the API. + */ +export const agentFlagsAtom = atomWithQuery(() => ({ + queryKey: AGENT_FLAGS_QUERY_KEY, + staleTime: 10 * 60 * 1000, + enabled: !!getBearerToken(), + queryFn: () => agentFlagsApiService.get(), +})); diff --git a/surfsense_web/components/agent-action-log/action-log-button.tsx b/surfsense_web/components/agent-action-log/action-log-button.tsx new file mode 100644 index 000000000..1c0383136 --- /dev/null +++ b/surfsense_web/components/agent-action-log/action-log-button.tsx @@ -0,0 +1,50 @@ +"use client"; + +import { useAtomValue, useSetAtom } from "jotai"; +import { Activity } from "lucide-react"; +import { useCallback } from "react"; +import { openActionLogSheetAtom } from "@/atoms/agent/action-log-sheet.atom"; +import { agentFlagsAtom } from "@/atoms/agent/agent-flags-query.atom"; +import { Button } from "@/components/ui/button"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; + +interface ActionLogButtonProps { + threadId: number | null; +} + +/** + * Header button that opens the agent action log sheet for the current + * thread. Renders nothing when: + * - the action log feature flag is off (graceful no-op for older + * deployments), OR + * - there is no active thread (lazy-created chats haven't started). + */ +export function ActionLogButton({ threadId }: ActionLogButtonProps) { + const { data: flags } = useAtomValue(agentFlagsAtom); + const open = useSetAtom(openActionLogSheetAtom); + + const enabled = !!flags?.enable_action_log && !flags?.disable_new_agent_stack; + + const handleClick = useCallback(() => { + if (threadId !== null) open(threadId); + }, [open, threadId]); + + if (!enabled || threadId === null) return null; + + return ( + + + + + Agent actions + + ); +} diff --git a/surfsense_web/components/agent-action-log/action-log-item.tsx b/surfsense_web/components/agent-action-log/action-log-item.tsx new file mode 100644 index 000000000..425714c1f --- /dev/null +++ b/surfsense_web/components/agent-action-log/action-log-item.tsx @@ -0,0 +1,215 @@ +"use client"; + +import { ChevronRight, RotateCcw, ShieldOff, Undo2 } from "lucide-react"; +import { useState } from "react"; +import { toast } from "sonner"; +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, + AlertDialogTrigger, +} from "@/components/ui/alert-dialog"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Separator } from "@/components/ui/separator"; +import { getToolIcon } from "@/contracts/enums/toolIcons"; +import { type AgentAction, agentActionsApiService } from "@/lib/apis/agent-actions-api.service"; +import { AppError } from "@/lib/error"; +import { formatRelativeDate } from "@/lib/format-date"; +import { cn } from "@/lib/utils"; + +function formatToolName(name: string): string { + return name.replace(/_/g, " ").replace(/\b\w/g, (c) => c.toUpperCase()); +} + +interface ActionLogItemProps { + action: AgentAction; + threadId: number; + onRevertSuccess: () => void; +} + +export function ActionLogItem({ action, threadId, onRevertSuccess }: ActionLogItemProps) { + const [isExpanded, setIsExpanded] = useState(false); + const [isReverting, setIsReverting] = useState(false); + const [confirmOpen, setConfirmOpen] = useState(false); + + const isAlreadyReverted = action.reverted_by_action_id !== null; + const isRevertAction = action.is_revert_action; + const hasError = action.error !== null && action.error !== undefined; + + const Icon = getToolIcon(action.tool_name); + const displayName = formatToolName(action.tool_name); + + const argsPreview = action.args ? JSON.stringify(action.args, null, 2) : null; + const truncatedArgs = + argsPreview && argsPreview.length > 600 ? `${argsPreview.slice(0, 600)}…` : argsPreview; + + const canRevert = action.reversible && !isAlreadyReverted && !isRevertAction && !hasError; + + const handleRevert = async () => { + setIsReverting(true); + try { + const response = await agentActionsApiService.revert(threadId, action.id); + toast.success(response.message || "Action reverted successfully."); + onRevertSuccess(); + } catch (err) { + const message = + err instanceof AppError + ? err.message + : err instanceof Error + ? err.message + : "Failed to revert action."; + toast.error(message); + } finally { + setIsReverting(false); + setConfirmOpen(false); + } + }; + + return ( +
+ + + {isExpanded && ( +
+ {truncatedArgs && ( +
+

+ Arguments +

+ 
+								{truncatedArgs}
+
+
+ )} + {action.error && ( +
+

+ Error +

+ 
+								{JSON.stringify(action.error, null, 2)}
+
+
+ )} + {action.reverse_descriptor && ( +
+

+ Reverse plan +

+ 
+								{JSON.stringify(action.reverse_descriptor, null, 2)}
+
+
+ )} + + + +
+

+ Action ID: {action.id} +

+ {canRevert ? ( + + + + + + + Revert this action? + + This will undo {displayName} and append a + new audit entry. The agent's chat history is preserved — only the tool's + effects on your knowledge base or connectors will be reversed where possible. + + + + Cancel + { + e.preventDefault(); + handleRevert(); + }} + disabled={isReverting} + > + {isReverting ? "Reverting…" : "Revert"} + + + + + ) : ( +
+ + {isAlreadyReverted + ? "Already reverted" + : isRevertAction + ? "Revert entry" + : hasError + ? "Cannot revert errored action" + : "Not reversible"} +
+ )} +
+
+ )} +
+ ); +} diff --git a/surfsense_web/components/agent-action-log/action-log-sheet.tsx b/surfsense_web/components/agent-action-log/action-log-sheet.tsx new file mode 100644 index 000000000..68d2ffef3 --- /dev/null +++ b/surfsense_web/components/agent-action-log/action-log-sheet.tsx @@ -0,0 +1,185 @@ +"use client"; + +import { useQuery, useQueryClient } from "@tanstack/react-query"; +import { useAtom, useAtomValue } from "jotai"; +import { Activity, RefreshCcw } from "lucide-react"; +import { useCallback, useMemo } from "react"; +import { actionLogSheetAtom } from "@/atoms/agent/action-log-sheet.atom"; +import { agentFlagsAtom } from "@/atoms/agent/agent-flags-query.atom"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Separator } from "@/components/ui/separator"; +import { + Sheet, + SheetContent, + SheetDescription, + SheetHeader, + SheetTitle, +} from "@/components/ui/sheet"; +import { Skeleton } from "@/components/ui/skeleton"; +import { agentActionsApiService } from "@/lib/apis/agent-actions-api.service"; +import { ActionLogItem } from "./action-log-item"; + +const ACTION_LOG_PAGE_SIZE = 50; + +function actionLogQueryKey(threadId: number) { + return ["agent-actions", threadId] as const; +} + +function EmptyState() { + return ( +
+
+ +
+
+

No actions logged yet

+

+ Once the agent calls a tool in this thread, it will show up here. From the log you can + inspect arguments and revert reversible actions. +

+
+
+ ); +} + +function DisabledState() { + return ( +
+
+ +
+
+

Action log is disabled

+

+ This deployment hasn't enabled the agent action log. An admin can flip + + SURFSENSE_ENABLE_ACTION_LOG + + . +

+
+
+ ); +} + +const SKELETON_KEYS = ["s1", "s2", "s3", "s4"] as const; + +function LoadingState() { + return ( +
+ {SKELETON_KEYS.map((key) => ( + + ))} +
+ ); +} + +export function ActionLogSheet() { + const [state, setState] = useAtom(actionLogSheetAtom); + const queryClient = useQueryClient(); + + const { data: flags } = useAtomValue(agentFlagsAtom); + const actionLogEnabled = !!flags?.enable_action_log && !flags?.disable_new_agent_stack; + const revertEnabled = !!flags?.enable_revert_route && !flags?.disable_new_agent_stack; + + const threadId = state.threadId; + + const { data, isLoading, isFetching, isError, error, refetch } = useQuery({ + queryKey: threadId !== null ? actionLogQueryKey(threadId) : ["agent-actions", "none"], + queryFn: () => + agentActionsApiService.listForThread(threadId as number, { + page: 0, + pageSize: ACTION_LOG_PAGE_SIZE, + }), + enabled: state.open && threadId !== null && actionLogEnabled, + staleTime: 15 * 1000, + }); + + const handleRevertSuccess = useCallback(() => { + if (threadId !== null) { + queryClient.invalidateQueries({ queryKey: actionLogQueryKey(threadId) }); + } + }, [queryClient, threadId]); + + const items = useMemo(() => data?.items ?? [], [data]); + + return ( + setState((s) => ({ ...s, open }))}> + + +
+
+ + Agent actions + {data?.total !== undefined && data.total > 0 && ( + + {data.total} + + )} +
+ +
+ + Audit trail of every tool call the agent made in this thread. + {revertEnabled + ? " Reversible actions can be undone in place." + : " Reverts are read-only on this deployment."} + + + + + +
+ {!actionLogEnabled ? ( + + ) : threadId === null ? ( + + ) : isLoading ? ( + + ) : isError ? ( +
+

Failed to load actions

+

+ {error instanceof Error ? error.message : "Unknown error"} +

+ +
+ ) : items.length === 0 ? ( + + ) : ( +
+ {items.map((action) => ( + + ))} + {data?.has_more && ( +

+ Showing {items.length} of {data.total}. Older actions are paginated. +

+ )} +
+ )} +
+ + + ); +} diff --git a/surfsense_web/components/assistant-ui/markdown-text.tsx b/surfsense_web/components/assistant-ui/markdown-text.tsx index 8bb228580..7655e10cc 100644 --- a/surfsense_web/components/assistant-ui/markdown-text.tsx +++ b/surfsense_web/components/assistant-ui/markdown-text.tsx @@ -85,10 +85,13 @@ function preprocessMarkdown(content: string): string { } ); + // All math forms are normalised to $$...$$ so we can disable single-dollar + // inline math in remark-math (otherwise currency like "$3,120.00 and $0.00" + // gets parsed as a LaTeX expression). // 1. Block math: \[...\] → $$...$$ content = content.replace(/\\\[([\s\S]*?)\\\]/g, (_, inner) => `$$${inner}$$`); - // 2. Inline math: \(...\) → $...$ - content = content.replace(/\\\(([\s\S]*?)\\\)/g, (_, inner) => `$${inner}$`); + // 2. Inline math: \(...\) → $$...$$ + content = content.replace(/\\\(([\s\S]*?)\\\)/g, (_, inner) => `$$${inner}$$`); // 3. Block: \begin{equation}...\end{equation} → $$...$$ content = content.replace( /\\begin\{equation\}([\s\S]*?)\\end\{equation\}/g, @@ -99,8 +102,11 @@ function preprocessMarkdown(content: string): string { /\\begin\{displaymath\}([\s\S]*?)\\end\{displaymath\}/g, (_, inner) => `$$${inner}$$` ); - // 5. Inline: \begin{math}...\end{math} → $...$ - content = content.replace(/\\begin\{math\}([\s\S]*?)\\end\{math\}/g, (_, inner) => `$${inner}$`); + // 5. Inline: \begin{math}...\end{math} → $$...$$ + content = content.replace( + /\\begin\{math\}([\s\S]*?)\\end\{math\}/g, + (_, inner) => `$$${inner}$$` + ); // 6. Strip backtick wrapping around math: `$$...$$` → $$...$$ and `$...$` → $...$ content = content.replace(/`(\${1,2})((?:(?!\1).)+)\1`/g, "$1$2$1"); @@ -180,7 +186,7 @@ const MarkdownTextImpl = () => { return ( { if (isInterruptResult(props.result)) { + if (isDoomLoopInterrupt(props.result)) { + return ; + } return ; } return ; diff --git a/surfsense_web/components/layout/providers/LayoutDataProvider.tsx b/surfsense_web/components/layout/providers/LayoutDataProvider.tsx index aecf55a27..3efdab03b 100644 --- a/surfsense_web/components/layout/providers/LayoutDataProvider.tsx +++ b/surfsense_web/components/layout/providers/LayoutDataProvider.tsx @@ -28,6 +28,7 @@ import { import { currentUserAtom } from "@/atoms/user/user-query.atoms"; import { SearchSpaceSettingsDialog } from "@/components/settings/search-space-settings-dialog"; import { TeamDialog } from "@/components/settings/team-dialog"; +import { ActionLogSheet } from "@/components/agent-action-log/action-log-sheet"; import { UserSettingsDialog } from "@/components/settings/user-settings-dialog"; import { AlertDialog, @@ -909,6 +910,9 @@ export function LayoutDataProvider({ searchSpaceId, children }: LayoutDataProvid + + {/* Agent action log + revert sheet */} + ); } diff --git a/surfsense_web/components/layout/ui/header/Header.tsx b/surfsense_web/components/layout/ui/header/Header.tsx index ec54cb901..f49d7fb88 100644 --- a/surfsense_web/components/layout/ui/header/Header.tsx +++ b/surfsense_web/components/layout/ui/header/Header.tsx @@ -5,6 +5,7 @@ import { usePathname } from "next/navigation"; import { currentThreadAtom } from "@/atoms/chat/current-thread.atom"; import { activeSearchSpaceIdAtom } from "@/atoms/search-spaces/search-space-query.atoms"; import { activeTabAtom, tabsAtom } from "@/atoms/tabs/tabs.atom"; +import { ActionLogButton } from "@/components/agent-action-log/action-log-button"; import { ChatHeader } from "@/components/new-chat/chat-header"; import { ChatShareButton } from "@/components/new-chat/chat-share-button"; import { useIsMobile } from "@/hooks/use-mobile"; @@ -69,6 +70,7 @@ export function Header({ mobileMenuTrigger }: HeaderProps) { {/* Right side - Actions */}
+ {hasThread && } {hasThread && ( )} diff --git a/surfsense_web/components/markdown-viewer.tsx b/surfsense_web/components/markdown-viewer.tsx index 5775fe083..c4d73e30b 100644 --- a/surfsense_web/components/markdown-viewer.tsx +++ b/surfsense_web/components/markdown-viewer.tsx @@ -10,7 +10,11 @@ const code = createCodePlugin({ }); const math = createMathPlugin({ - singleDollarTextMath: true, + // Disabled so currency like "$3,120.00 and ... $0.00" isn't parsed as + // inline LaTeX. convertLatexDelimiters() below normalises any genuine + // inline math (\(...\), $...$ starting with a LaTeX command, etc.) to + // $$...$$, so this flip doesn't lose any math rendering. + singleDollarTextMath: false, }); interface MarkdownViewerProps { diff --git a/surfsense_web/components/settings/user-settings-dialog.tsx b/surfsense_web/components/settings/user-settings-dialog.tsx index 6740aad92..a04ce16dd 100644 --- a/surfsense_web/components/settings/user-settings-dialog.tsx +++ b/surfsense_web/components/settings/user-settings-dialog.tsx @@ -2,6 +2,7 @@ import { useAtom } from "jotai"; import { + Activity, Brain, CircleUser, Globe, @@ -9,6 +10,7 @@ import { KeyRound, Monitor, ReceiptText, + ShieldCheck, Sparkles, } from "lucide-react"; import dynamic from "next/dynamic"; @@ -74,6 +76,20 @@ const MemoryContent = dynamic( ), { ssr: false } ); +const AgentPermissionsContent = dynamic( + () => + import( + "@/app/dashboard/[search_space_id]/user-settings/components/AgentPermissionsContent" + ).then((m) => ({ default: m.AgentPermissionsContent })), + { ssr: false } +); +const AgentStatusContent = dynamic( + () => + import("@/app/dashboard/[search_space_id]/user-settings/components/AgentStatusContent").then( + (m) => ({ default: m.AgentStatusContent }) + ), + { ssr: false } +); export function UserSettingsDialog() { const t = useTranslations("userSettings"); @@ -103,6 +119,16 @@ export function UserSettingsDialog() { label: "Memory", icon: , }, + { + value: "agent-permissions", + label: "Agent Permissions", + icon: , + }, + { + value: "agent-status", + label: "Agent Status", + icon: , + }, { value: "purchases", label: "Purchase History", @@ -141,6 +167,8 @@ export function UserSettingsDialog() { {state.initialTab === "prompts" && } {state.initialTab === "community-prompts" && } {state.initialTab === "memory" && } + {state.initialTab === "agent-permissions" && } + {state.initialTab === "agent-status" && } {state.initialTab === "purchases" && } {state.initialTab === "desktop" && } {state.initialTab === "desktop-shortcuts" && } diff --git a/surfsense_web/components/tool-ui/doom-loop-approval.tsx b/surfsense_web/components/tool-ui/doom-loop-approval.tsx new file mode 100644 index 000000000..6132a71ed --- /dev/null +++ b/surfsense_web/components/tool-ui/doom-loop-approval.tsx @@ -0,0 +1,187 @@ +"use client"; + +import type { ToolCallMessagePartComponent } from "@assistant-ui/react"; +import { CornerDownLeftIcon, OctagonAlert } from "lucide-react"; +import { useCallback, useEffect, useMemo } from "react"; +import { TextShimmerLoader } from "@/components/prompt-kit/loader"; +import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Separator } from "@/components/ui/separator"; +import { useHitlPhase } from "@/hooks/use-hitl-phase"; +import type { HitlDecision, InterruptResult } from "@/lib/hitl"; +import { isInterruptResult, useHitlDecision } from "@/lib/hitl"; + +/** + * Specialized HITL card for ``DoomLoopMiddleware`` interrupts. The + * backend signals these by setting ``context.permission === "doom_loop"`` + * on the ``permission_ask`` interrupt. + * + * The card replaces the generic "approve/reject" framing with a + * "continue/stop" affordance that better matches the user's mental + * model: the agent is stuck repeating itself, not asking permission + * for a destructive action. + */ +function DoomLoopCard({ + toolName, + args, + interruptData, + onDecision, +}: { + toolName: string; + args: Record; + interruptData: InterruptResult; + onDecision: (decision: HitlDecision) => void; +}) { + const { phase, setProcessing, setRejected } = useHitlPhase(interruptData); + + const context = (interruptData.context ?? {}) as Record; + const threshold = typeof context.threshold === "number" ? context.threshold : 3; + const stuckTool = (typeof context.tool === "string" && context.tool) || toolName; + const recentSignatures = Array.isArray(context.recent_signatures) + ? (context.recent_signatures as string[]) + : []; + const displayName = stuckTool.replace(/_/g, " ").replace(/\b\w/g, (c) => c.toUpperCase()); + + const argPreview = useMemo(() => { + if (!args || Object.keys(args).length === 0) return null; + try { + const json = JSON.stringify(args, null, 2); + return json.length > 600 ? `${json.slice(0, 600)}…` : json; + } catch { + return null; + } + }, [args]); + + const handleContinue = useCallback(() => { + if (phase !== "pending") return; + setProcessing(); + onDecision({ type: "approve" }); + }, [phase, setProcessing, onDecision]); + + const handleStop = useCallback(() => { + if (phase !== "pending") return; + setRejected(); + onDecision({ type: "reject", message: "Doom loop: user requested stop." }); + }, [phase, setRejected, onDecision]); + + useEffect(() => { + const handler = (e: KeyboardEvent) => { + if (phase !== "pending") return; + if (e.key === "Enter" && !e.shiftKey && !e.ctrlKey && !e.metaKey) { + e.preventDefault(); + handleStop(); + } + }; + window.addEventListener("keydown", handler); + return () => window.removeEventListener("keydown", handler); + }, [phase, handleStop]); + + const isResolved = phase === "complete" || phase === "rejected"; + + return ( + + + + + {phase === "rejected" + ? "Stopped" + : phase === "processing" + ? "Continuing…" + : phase === "complete" + ? "Continued" + : "I might be stuck"} + + {!isResolved && ( + + doom-loop + + )} + + + {phase === "processing" ? ( + + ) : phase === "rejected" ? ( +

+ I stopped retrying {displayName} as you asked. +

+ ) : phase === "complete" ? ( +

+ Continuing to call {displayName} as you asked. +

+ ) : ( +

+ I called {displayName} {threshold} times in a row + with similar arguments. Should I keep going or stop and rethink? +

+ )} + + {argPreview && phase === "pending" && ( + <> + +
+

+ Last arguments +

+ 
+								{argPreview}
+
+
+ + )} + + {recentSignatures.length > 0 && phase === "pending" && ( +
+ + Show repeated signatures ({recentSignatures.length}) + +
    + {recentSignatures.map((sig) => ( +
  • + {sig} +
    • + ))} +
+
+ )} + + {phase === "pending" && ( +
+ + +
+ )} + + + ); +} + +export const DoomLoopApprovalToolUI: ToolCallMessagePartComponent = ({ + toolName, + args, + result, +}) => { + const { dispatch } = useHitlDecision(); + + if (!result || !isInterruptResult(result)) return null; + + return ( + } + interruptData={result} + onDecision={(decision) => dispatch([decision])} + /> + ); +}; + +export function isDoomLoopInterrupt(result: unknown): boolean { + if (!isInterruptResult(result)) return false; + const ctx = (result.context ?? {}) as Record; + return ctx.permission === "doom_loop"; +} diff --git a/surfsense_web/lib/apis/agent-actions-api.service.ts b/surfsense_web/lib/apis/agent-actions-api.service.ts new file mode 100644 index 000000000..007bb131e --- /dev/null +++ b/surfsense_web/lib/apis/agent-actions-api.service.ts @@ -0,0 +1,64 @@ +import { z } from "zod"; +import { baseApiService } from "./base-api.service"; + +const AgentActionReadSchema = z.object({ + id: z.number(), + thread_id: z.number(), + user_id: z.string().nullable(), + search_space_id: z.number(), + tool_name: z.string(), + args: z.record(z.string(), z.unknown()).nullable(), + result_id: z.string().nullable(), + reversible: z.boolean(), + reverse_descriptor: z.record(z.string(), z.unknown()).nullable(), + error: z.record(z.string(), z.unknown()).nullable(), + reverse_of: z.number().nullable(), + reverted_by_action_id: z.number().nullable(), + is_revert_action: z.boolean(), + created_at: z.string(), +}); + +export type AgentAction = z.infer; + +const AgentActionListResponseSchema = z.object({ + items: z.array(AgentActionReadSchema), + total: z.number(), + page: z.number(), + page_size: z.number(), + has_more: z.boolean(), +}); + +export type AgentActionListResponse = z.infer; + +const RevertResponseSchema = z.object({ + status: z.literal("ok"), + message: z.string(), + new_action_id: z.number().nullable().optional(), +}); + +export type RevertResponse = z.infer; + +class AgentActionsApiService { + listForThread = async ( + threadId: number, + opts: { page?: number; pageSize?: number } = {} + ): Promise => { + const params = new URLSearchParams(); + params.set("page", String(opts.page ?? 0)); + params.set("page_size", String(opts.pageSize ?? 50)); + return baseApiService.get( + `/api/v1/threads/${threadId}/actions?${params.toString()}`, + AgentActionListResponseSchema + ); + }; + + revert = async (threadId: number, actionId: number): Promise => { + return baseApiService.post( + `/api/v1/threads/${threadId}/revert/${actionId}`, + RevertResponseSchema, + { body: {} } + ); + }; +} + +export const agentActionsApiService = new AgentActionsApiService(); diff --git a/surfsense_web/lib/apis/agent-flags-api.service.ts b/surfsense_web/lib/apis/agent-flags-api.service.ts new file mode 100644 index 000000000..87332ca9f --- /dev/null +++ b/surfsense_web/lib/apis/agent-flags-api.service.ts @@ -0,0 +1,40 @@ +import { z } from "zod"; +import { baseApiService } from "./base-api.service"; + +const AgentFeatureFlagsSchema = z.object({ + disable_new_agent_stack: z.boolean(), + + enable_context_editing: z.boolean(), + enable_compaction_v2: z.boolean(), + enable_retry_after: z.boolean(), + enable_model_fallback: z.boolean(), + enable_model_call_limit: z.boolean(), + enable_tool_call_limit: z.boolean(), + enable_tool_call_repair: z.boolean(), + enable_doom_loop: z.boolean(), + + enable_permission: z.boolean(), + enable_busy_mutex: z.boolean(), + enable_llm_tool_selector: z.boolean(), + + enable_skills: z.boolean(), + enable_specialized_subagents: z.boolean(), + enable_kb_planner_runnable: z.boolean(), + + enable_action_log: z.boolean(), + enable_revert_route: z.boolean(), + + enable_plugin_loader: z.boolean(), + + enable_otel: z.boolean(), +}); + +export type AgentFeatureFlags = z.infer; + +class AgentFlagsApiService { + get = async (): Promise => { + return baseApiService.get(`/api/v1/agent/flags`, AgentFeatureFlagsSchema); + }; +} + +export const agentFlagsApiService = new AgentFlagsApiService(); diff --git a/surfsense_web/lib/apis/agent-permissions-api.service.ts b/surfsense_web/lib/apis/agent-permissions-api.service.ts new file mode 100644 index 000000000..6927c55d0 --- /dev/null +++ b/surfsense_web/lib/apis/agent-permissions-api.service.ts @@ -0,0 +1,90 @@ +import { z } from "zod"; +import { ValidationError } from "@/lib/error"; +import { baseApiService } from "./base-api.service"; + +const ActionEnum = z.enum(["allow", "deny", "ask"]); +export type AgentPermissionAction = z.infer; + +const AgentPermissionRuleSchema = z.object({ + id: z.number(), + search_space_id: z.number(), + user_id: z.string().nullable(), + thread_id: z.number().nullable(), + permission: z.string(), + pattern: z.string(), + action: ActionEnum, + created_at: z.string(), +}); + +export type AgentPermissionRule = z.infer; + +const AgentPermissionRuleListSchema = z.array(AgentPermissionRuleSchema); + +const AgentPermissionRuleCreateSchema = z.object({ + permission: z + .string() + .min(1, "Permission is required") + .max(255) + .regex(/^[a-zA-Z0-9_:.\-*]+$/, "Use letters, digits, '.', '_', ':', '-', or '*' wildcards."), + pattern: z.string().min(1).max(255).default("*"), + action: ActionEnum, + user_id: z.string().nullable().optional(), + thread_id: z.number().nullable().optional(), +}); + +export type AgentPermissionRuleCreate = z.infer; + +const AgentPermissionRuleUpdateSchema = z.object({ + pattern: z.string().min(1).max(255).optional(), + action: ActionEnum.optional(), +}); + +export type AgentPermissionRuleUpdate = z.infer; + +class AgentPermissionsApiService { + list = async (searchSpaceId: number): Promise => { + return baseApiService.get( + `/api/v1/searchspaces/${searchSpaceId}/agent/permissions/rules`, + AgentPermissionRuleListSchema + ); + }; + + create = async ( + searchSpaceId: number, + payload: AgentPermissionRuleCreate + ): Promise => { + const parsed = AgentPermissionRuleCreateSchema.safeParse(payload); + if (!parsed.success) { + throw new ValidationError(parsed.error.issues.map((i) => i.message).join(", ")); + } + return baseApiService.post( + `/api/v1/searchspaces/${searchSpaceId}/agent/permissions/rules`, + AgentPermissionRuleSchema, + { body: parsed.data } + ); + }; + + update = async ( + searchSpaceId: number, + ruleId: number, + payload: AgentPermissionRuleUpdate + ): Promise => { + const parsed = AgentPermissionRuleUpdateSchema.safeParse(payload); + if (!parsed.success) { + throw new ValidationError(parsed.error.issues.map((i) => i.message).join(", ")); + } + return baseApiService.patch( + `/api/v1/searchspaces/${searchSpaceId}/agent/permissions/rules/${ruleId}`, + AgentPermissionRuleSchema, + { body: parsed.data } + ); + }; + + remove = async (searchSpaceId: number, ruleId: number): Promise => { + await baseApiService.delete( + `/api/v1/searchspaces/${searchSpaceId}/agent/permissions/rules/${ruleId}` + ); + }; +} + +export const agentPermissionsApiService = new AgentPermissionsApiService(); From b9a66cb417d04bd445b6be1a7838a2278ae3cefe Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Tue, 28 Apr 2026 21:30:53 -0700 Subject: [PATCH 5/8] feat: various UI fixes, prompt optimizations, and allowing duplicate docs - Updated `content_hash` in the `Document` model to remove global uniqueness, allowing identical content across different paths. - Enhanced `_create_document` function to handle path uniqueness and prevent session-poisoning from `IntegrityError`. - Added detailed comments for clarity on the changes and their implications. - Introduced new citation handling in the editor for improved user experience with citation jumps. - Updated package dependencies in the frontend for better functionality. --- .../133_drop_documents_content_hash_unique.py | 107 +++ .../new_chat/middleware/kb_persistence.py | 65 +- .../app/agents/new_chat/prompts/composer.py | 41 +- .../new_chat/prompts/providers/anthropic.md | 21 +- .../new_chat/prompts/providers/deepseek.md | 18 + .../new_chat/prompts/providers/google.md | 20 +- .../agents/new_chat/prompts/providers/grok.md | 17 + .../agents/new_chat/prompts/providers/kimi.md | 21 + .../prompts/providers/openai_classic.md | 22 +- .../prompts/providers/openai_codex.md | 19 + .../prompts/providers/openai_reasoning.md | 22 +- surfsense_backend/app/db.py | 10 +- .../agents/new_chat/prompts/test_composer.py | 74 +- .../test_kb_persistence_filesystem_parity.py | 168 ++++ surfsense_web/app/globals.css | 21 + .../pending-chunk-highlight.atom.ts | 19 + .../assistant-ui/inline-citation.tsx | 228 +++++- .../components/editor-panel/editor-panel.tsx | 530 +++++++++++-- .../components/editor/plate-editor.tsx | 31 + surfsense_web/components/editor/presets.ts | 28 + .../new-chat/source-detail-panel.tsx | 719 ------------------ .../settings/user-settings-dialog.tsx | 3 - .../components/ui/search-highlight-node.tsx | 45 ++ surfsense_web/lib/citation-search.ts | 125 +++ surfsense_web/package.json | 1 + surfsense_web/pnpm-lock.yaml | 17 + 26 files changed, 1540 insertions(+), 852 deletions(-) create mode 100644 surfsense_backend/alembic/versions/133_drop_documents_content_hash_unique.py create mode 100644 surfsense_backend/app/agents/new_chat/prompts/providers/deepseek.md create mode 100644 surfsense_backend/app/agents/new_chat/prompts/providers/grok.md create mode 100644 surfsense_backend/app/agents/new_chat/prompts/providers/kimi.md create mode 100644 surfsense_backend/app/agents/new_chat/prompts/providers/openai_codex.md create mode 100644 surfsense_backend/tests/unit/middleware/test_kb_persistence_filesystem_parity.py create mode 100644 surfsense_web/atoms/document-viewer/pending-chunk-highlight.atom.ts delete mode 100644 surfsense_web/components/new-chat/source-detail-panel.tsx create mode 100644 surfsense_web/components/ui/search-highlight-node.tsx create mode 100644 surfsense_web/lib/citation-search.ts diff --git a/surfsense_backend/alembic/versions/133_drop_documents_content_hash_unique.py b/surfsense_backend/alembic/versions/133_drop_documents_content_hash_unique.py new file mode 100644 index 000000000..88c3e203f --- /dev/null +++ b/surfsense_backend/alembic/versions/133_drop_documents_content_hash_unique.py @@ -0,0 +1,107 @@ +"""133_drop_documents_content_hash_unique + +Revision ID: 133 +Revises: 132 +Create Date: 2026-04-29 + +Drop the global UNIQUE constraint on ``documents.content_hash`` so the +new-chat agent's ``write_file`` flow can persist legitimate file copies +(two paths, identical content) without hitting a constraint that mirrors +no real filesystem semantic. + +Path uniqueness still lives on ``documents.unique_identifier_hash`` (per +search space), which is the right invariant — exactly like an inode at a +given path on a POSIX filesystem. + +The non-unique INDEX on ``content_hash`` is preserved so connector +indexers' "have we seen this content before?" lookup +(:func:`app.tasks.document_processors.base.check_duplicate_document`, +which already uses ``.scalars().first()`` and is therefore tolerant of +duplicates) stays cheap. +""" + +from __future__ import annotations + +from collections.abc import Sequence + +from sqlalchemy import inspect + +from alembic import op + +revision: str = "133" +down_revision: str | None = "132" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def _existing_constraint_names(bind, table: str) -> set[str]: + inspector = inspect(bind) + return {c["name"] for c in inspector.get_unique_constraints(table)} + + +def _existing_index_names(bind, table: str) -> set[str]: + inspector = inspect(bind) + return {i["name"] for i in inspector.get_indexes(table)} + + +def upgrade() -> None: + bind = op.get_bind() + + # Both the named UniqueConstraint (added in revision 8) and the + # implicit-unique-index variant SQLAlchemy may emit need draining. + constraints = _existing_constraint_names(bind, "documents") + if "uq_documents_content_hash" in constraints: + op.drop_constraint( + "uq_documents_content_hash", "documents", type_="unique" + ) + + indexes = _existing_index_names(bind, "documents") + # Some Postgres versions surface the unique constraint via a unique + # index of the same name; check for that too. + for idx_name in ("uq_documents_content_hash",): + if idx_name in indexes: + op.drop_index(idx_name, table_name="documents") + + # Ensure the non-unique index is present for fast lookups. + if "ix_documents_content_hash" not in indexes: + op.create_index( + "ix_documents_content_hash", + "documents", + ["content_hash"], + unique=False, + ) + + +def downgrade() -> None: + bind = op.get_bind() + + # Re-applying UNIQUE is destructive: there may now be legitimate + # duplicates (e.g. two NOTE documents that share content because the + # user explicitly copied one to a new path). To avoid the migration + # silently deleting user data, we keep only the lowest-id row per + # content_hash — same strategy revision 8 used when first introducing + # the constraint. + op.execute( + """ + DELETE FROM documents + WHERE id NOT IN ( + SELECT MIN(id) + FROM documents + GROUP BY content_hash + ) + """ + ) + + indexes = _existing_index_names(bind, "documents") + if "ix_documents_content_hash" in indexes: + op.drop_index("ix_documents_content_hash", table_name="documents") + + op.create_index( + "ix_documents_content_hash", + "documents", + ["content_hash"], + unique=False, + ) + op.create_unique_constraint( + "uq_documents_content_hash", "documents", ["content_hash"] + ) diff --git a/surfsense_backend/app/agents/new_chat/middleware/kb_persistence.py b/surfsense_backend/app/agents/new_chat/middleware/kb_persistence.py index 5682977d9..378b83950 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/kb_persistence.py +++ b/surfsense_backend/app/agents/new_chat/middleware/kb_persistence.py @@ -28,6 +28,7 @@ from langchain.agents.middleware import AgentMiddleware, AgentState from langchain_core.callbacks import dispatch_custom_event from langgraph.runtime import Runtime from sqlalchemy import delete, select +from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession from app.agents.new_chat.filesystem_selection import FilesystemMode @@ -150,10 +151,11 @@ async def _create_document( virtual_path, search_space_id, ) - # Guard against the unique_identifier_hash constraint: another row at the - # same virtual_path (this search space) already owns the hash. Callers are - # expected to upsert via the wrapper, but this defends against bypasses - # and gives a clean ValueError instead of a session-poisoning IntegrityError. + # Filesystem-parity invariant: the only thing that *must* be unique is + # the path. Two notes can legitimately share content (e.g. ``cp a b``). + # Guard against the path-derived ``unique_identifier_hash`` constraint + # so we surface a clean ValueError instead of letting the INSERT poison + # the session with an IntegrityError. path_collision = await session.execute( select(Document.id).where( Document.search_space_id == search_space_id, @@ -165,17 +167,14 @@ async def _create_document( f"a document already exists at path '{virtual_path}' " "(unique_identifier_hash collision)" ) + # ``content_hash`` is intentionally NOT checked for uniqueness here. + # In a real filesystem two files at different paths can hold identical + # bytes, and the agent's ``write_file`` path needs that semantic to + # support copy/duplicate operations. The hash remains useful as a + # change-detection hint for connector indexers, which still consult it + # via :func:`check_duplicate_document` but do so with a non-unique + # lookup (``.first()``). content_hash = generate_content_hash(content, search_space_id) - content_collision = await session.execute( - select(Document.id).where( - Document.search_space_id == search_space_id, - Document.content_hash == content_hash, - ) - ) - if content_collision.scalar_one_or_none() is not None: - raise ValueError( - f"a document with identical content already exists for path '{virtual_path}'" - ) doc = Document( title=title, document_type=DocumentType.NOTE, @@ -493,19 +492,43 @@ async def commit_staged_filesystem_state( } ) else: + # Wrap each create in a SAVEPOINT so a residual + # ``IntegrityError`` (e.g. a deployment that hasn't run + # migration 133 yet, where ``documents.content_hash`` + # still carries its legacy global UNIQUE constraint) + # rolls back only this one create instead of poisoning + # the whole turn's transaction. try: - new_doc = await _create_document( - session, - virtual_path=path, - content=content, - search_space_id=search_space_id, - created_by_id=created_by_id, - ) + async with session.begin_nested(): + new_doc = await _create_document( + session, + virtual_path=path, + content=content, + search_space_id=search_space_id, + created_by_id=created_by_id, + ) except ValueError as exc: logger.warning( "kb_persistence: skipping %s create: %s", path, exc ) continue + except IntegrityError as exc: + # The path-uniqueness check above already protected + # against ``unique_identifier_hash`` collisions, so + # the most likely culprit is the legacy + # ``ix_documents_content_hash`` UNIQUE constraint + # that migration 133 drops. Log loudly so operators + # know to run the migration; do NOT silently swallow. + msg = str(exc.orig) if exc.orig is not None else str(exc) + logger.error( + "kb_persistence: IntegrityError creating %s: %s. " + "If this mentions content_hash, run alembic " + "upgrade to apply migration 133 which drops the " + "global UNIQUE constraint on documents.content_hash.", + path, + msg, + ) + continue doc_id_by_path[path] = new_doc.id committed_creates.append( { diff --git a/surfsense_backend/app/agents/new_chat/prompts/composer.py b/surfsense_backend/app/agents/new_chat/prompts/composer.py index 44060f75f..bad033490 100644 --- a/surfsense_backend/app/agents/new_chat/prompts/composer.py +++ b/surfsense_backend/app/agents/new_chat/prompts/composer.py @@ -38,12 +38,38 @@ from app.db import ChatVisibility # Provider variant detection # ----------------------------------------------------------------------------- -ProviderVariant = str # "anthropic" | "openai_reasoning" | "openai_classic" | "google" | "default" +# String literal alias for the supported provider-specific prompt variants. +# When adding a new variant, also drop a matching ``providers/.md`` +# file in this package and (if appropriate) extend the regex matchers below. +# +# Stylistic clusters mirror OpenCode's prompt-per-family layout but adapted +# to SurfSense's "supplemental hints" architecture (each fragment is a +# focused style nudge, NOT a full system prompt — the main prompt is +# already assembled from base/ + tools/ + routing/). +ProviderVariant = str +# Known values: +# "anthropic" — Claude family (XML-friendly, narrative todos) +# "openai_reasoning" — GPT-5 / o-series (channel-aware pragmatic) +# "openai_classic" — GPT-4 family (autonomous persistence) +# "openai_codex" — gpt-*-codex (code-purist, terse, file:line refs) +# "google" — Gemini (formal, <3-line, numbered workflow) +# "kimi" — Moonshot Kimi-K* (action-bias, parallel tools) +# "grok" — xAI Grok (extreme-terse, one-word ok) +# "deepseek" — DeepSeek V3 / R1 (terse, R1-aware reasoning) +# "default" — fallback, no provider-specific block emitted +# IMPORTANT: order of evaluation matters in :func:`detect_provider_variant`. +# More specific patterns must come first (e.g. ``codex`` before +# ``openai_reasoning`` because codex model ids contain ``gpt``). + +_OPENAI_CODEX_RE = re.compile(r"\b(gpt-codex|codex-mini|gpt-[\d.]+-codex)\b", re.IGNORECASE) _OPENAI_REASONING_RE = re.compile(r"\b(gpt-5|o\d|o-)", re.IGNORECASE) _OPENAI_CLASSIC_RE = re.compile(r"\bgpt-4", re.IGNORECASE) _ANTHROPIC_RE = re.compile(r"\bclaude\b", re.IGNORECASE) _GOOGLE_RE = re.compile(r"\bgemini\b", re.IGNORECASE) +_KIMI_RE = re.compile(r"\b(kimi[-\d.]*|moonshot)\b", re.IGNORECASE) +_GROK_RE = re.compile(r"\bgrok\b", re.IGNORECASE) +_DEEPSEEK_RE = re.compile(r"\bdeepseek\b", re.IGNORECASE) def detect_provider_variant(model_name: str | None) -> ProviderVariant: @@ -51,10 +77,17 @@ def detect_provider_variant(model_name: str | None) -> ProviderVariant: Heuristic match on the model id; returns ``"default"`` when nothing matches so the composer can fall back to the empty placeholder file. + + Order is significant: more-specific patterns are tried first so + ``gpt-5-codex`` routes to ``"openai_codex"`` rather than + ``"openai_reasoning"`` (mirrors OpenCode's + ``packages/opencode/src/session/system.ts`` dispatch). """ if not model_name: return "default" name = model_name.strip() + if _OPENAI_CODEX_RE.search(name): + return "openai_codex" if _OPENAI_REASONING_RE.search(name): return "openai_reasoning" if _OPENAI_CLASSIC_RE.search(name): @@ -63,6 +96,12 @@ def detect_provider_variant(model_name: str | None) -> ProviderVariant: return "anthropic" if _GOOGLE_RE.search(name): return "google" + if _KIMI_RE.search(name): + return "kimi" + if _GROK_RE.search(name): + return "grok" + if _DEEPSEEK_RE.search(name): + return "deepseek" return "default" diff --git a/surfsense_backend/app/agents/new_chat/prompts/providers/anthropic.md b/surfsense_backend/app/agents/new_chat/prompts/providers/anthropic.md index 6e22ef265..f574da541 100644 --- a/surfsense_backend/app/agents/new_chat/prompts/providers/anthropic.md +++ b/surfsense_backend/app/agents/new_chat/prompts/providers/anthropic.md @@ -1,5 +1,20 @@ -You are running on an Anthropic Claude model. Use XML tags liberally to structure -intermediate reasoning when the task is complex. Prefer step-by-step plans inside -`` blocks before producing the final answer. +You are running on an Anthropic Claude model. + +Structured reasoning: +- Use XML tags liberally to organise intermediate reasoning when a task is non-trivial. `...` blocks are encouraged before tool calls or before producing a complex final answer. +- For multi-step requests, briefly outline a plan inside a `` block before issuing the first tool call. + +Professional objectivity: +- Prioritise technical accuracy over validating the user's beliefs. Provide direct, factual guidance without unnecessary superlatives, praise, or emotional validation. +- When uncertain, investigate (search the KB, fetch the page) rather than confirming the user's assumption. +- Disagree with the user when the evidence warrants it; respectful correction beats false agreement. + +Task management: +- For tasks with 3+ distinct steps use the todo / planning tool aggressively. Mark items in_progress before starting, completed immediately when finished — do not batch completions. +- Narrate progress through the todo list itself, not through chatty status lines. + +Tool calls: +- Run independent tool calls in parallel within one response. Sequence them only when a later call genuinely needs an earlier one's output. +- Never chain bash-like commands with `;` or `&&` to "narrate" — use prose between tool calls instead. diff --git a/surfsense_backend/app/agents/new_chat/prompts/providers/deepseek.md b/surfsense_backend/app/agents/new_chat/prompts/providers/deepseek.md new file mode 100644 index 000000000..8acf008ca --- /dev/null +++ b/surfsense_backend/app/agents/new_chat/prompts/providers/deepseek.md @@ -0,0 +1,18 @@ + +You are running on a DeepSeek model (DeepSeek-V3 chat / DeepSeek-R1 reasoning). + +Reasoning hygiene (R1-aware): +- If the model surfaces explicit `` blocks, keep that internal scratch focused — do NOT restate the user's question inside it; jump straight to the analysis. +- Never paste the contents of `` into your final answer. Final answer should reflect only the conclusion, citations, and any user-facing rationale. +- Do not let chain-of-thought leak into tool-call arguments — keep tool inputs minimal and structural. + +Output style: +- Be concise. Default to a one-paragraph answer; expand only when the user asks for detail. +- Don't open with sycophantic phrasing ("Great question", "Sure, here you go"). Lead with the answer or the next action. +- For factual answers, cite once with `[citation:chunk_id]` and stop. + +Tool calls: +- Issue independent tool calls in parallel within a single turn. +- Prefer the knowledge-base search tools before any web-search; this model has strong recall but stale training data. +- Don't fabricate file paths, chunk ids, or URLs — only use values returned by tools or provided by the user. + diff --git a/surfsense_backend/app/agents/new_chat/prompts/providers/google.md b/surfsense_backend/app/agents/new_chat/prompts/providers/google.md index 4b31a8388..cac3b328b 100644 --- a/surfsense_backend/app/agents/new_chat/prompts/providers/google.md +++ b/surfsense_backend/app/agents/new_chat/prompts/providers/google.md @@ -1,4 +1,20 @@ -You are running on a Google Gemini model. Prefer concise, structured responses. -When using tools, follow the function-calling protocol and avoid verbose preludes. +You are running on a Google Gemini model. + +Output style: +- Concise & direct. Aim for fewer than 3 lines of prose (excluding tool output, citations, and code/snippets) when the task allows. +- No conversational filler — skip openers like "Okay, I will now…" and closers like "I have finished the changes…". Get straight to the action or answer. +- Format with GitHub-flavoured Markdown; assume monospace rendering. +- For one-line factual answers, just answer. No headers, no bullets. + +Workflow for non-trivial tasks (Understand → Plan → Act → Verify): +1. **Understand:** read the user's request and the relevant KB / connector context. Use search and read tools (in parallel when independent) before assuming anything. +2. **Plan:** when the task touches multiple steps, share an extremely concise plan first. +3. **Act:** call the appropriate tools, strictly adhering to the prompts/routing already established for this agent. +4. **Verify:** confirm with a follow-up read or search where it materially de-risks the answer. + +Discipline: +- Do not take significant actions beyond the clear scope of the user's request without confirming first. +- Do not assume a connector / tool / file exists — check (e.g. via `get_connected_accounts`) before referencing it. +- Path arguments must be the exact strings returned by tools; do not synthesise file paths. diff --git a/surfsense_backend/app/agents/new_chat/prompts/providers/grok.md b/surfsense_backend/app/agents/new_chat/prompts/providers/grok.md new file mode 100644 index 000000000..95b8fcc14 --- /dev/null +++ b/surfsense_backend/app/agents/new_chat/prompts/providers/grok.md @@ -0,0 +1,17 @@ + +You are running on an xAI Grok model. + +Maximum terseness: +- Answer in fewer than 4 lines unless the user asks for detail. One-word answers are best when they suffice. +- No preamble ("The answer is", "Here's what I'll do"), no postamble ("Hope that helps", "Let me know"). Get straight to the answer. +- Avoid restating the user's question. +- For factual lookups inside the knowledge base, give the answer with a single `[citation:chunk_id]` and stop. + +Tool discipline: +- Use exactly ONE tool per assistant turn when investigating; wait for the result before deciding the next call. Do not loop on the same tool with the same arguments — pick a result and act. +- For obviously parallelizable read-only batches (multiple independent searches), one turn with several tool calls is fine — but never chain into a fishing expedition. + +Style: +- No emojis unless the user asked. No nested bullets, no headers for short answers. +- If you can't help, say so in 1-2 sentences without explaining "why this could lead to…". + diff --git a/surfsense_backend/app/agents/new_chat/prompts/providers/kimi.md b/surfsense_backend/app/agents/new_chat/prompts/providers/kimi.md new file mode 100644 index 000000000..c3c11ad5e --- /dev/null +++ b/surfsense_backend/app/agents/new_chat/prompts/providers/kimi.md @@ -0,0 +1,21 @@ + +You are running on a Moonshot Kimi model (Kimi-K1.5 / Kimi-K2 / Kimi-K2.5+). + +Action bias: +- Default to taking action with tools rather than describing solutions in prose. If a tool can answer the question, call the tool. +- Don't narrate routine reads, searches, or obvious next steps. Combine related progress into one short status line. +- Be thorough in actions (test what you build, verify what you change). Be brief in explanations. + +Tool calls: +- Output multiple non-interfering tool calls in a SINGLE response — parallelism is a major efficiency win on this model. +- When the `task` tool is available, delegate focused subtasks to a subagent with full context (subagents don't inherit yours). +- Don't apologise or pre-announce tool calls. The tool call itself is self-explanatory. + +Language: +- Respond in the SAME language as the user's most recent turn unless explicitly instructed otherwise. + +Discipline: +- Stay on track. Never give the user more than what they asked for. +- Fact-check before stating anything as factual; don't fabricate citations. +- Keep it stupidly simple. Don't overcomplicate. + diff --git a/surfsense_backend/app/agents/new_chat/prompts/providers/openai_classic.md b/surfsense_backend/app/agents/new_chat/prompts/providers/openai_classic.md index 7ea4366c4..9128609e0 100644 --- a/surfsense_backend/app/agents/new_chat/prompts/providers/openai_classic.md +++ b/surfsense_backend/app/agents/new_chat/prompts/providers/openai_classic.md @@ -1,5 +1,21 @@ -You are running on a classic OpenAI chat model (GPT-4 family). Use direct -function-calling for tools. When editing files, use the standard `edit_file` -or `write_file` tools rather than diff-based patches. +You are running on a classic OpenAI chat model (GPT-4 family). + +Persistence: +- Keep going until the user's query is completely resolved before yielding back. Don't end the turn at "I would do X" — actually do X. +- When you say "Next I will…" or "Now I will…", you MUST actually take that action in the same turn. +- If a tool call fails, diagnose and try again with corrected arguments; do not surface the raw error and stop. + +Planning: +- Plan extensively before each tool call and reflect briefly on the result of the previous call. For tasks with 3+ steps, use the todo / planning tool and mark items as `in_progress` / `completed` as you go. +- Always announce the next action in ONE concise sentence before making a non-trivial tool call ("I'll search the KB for the migration spec."). + +Output style: +- Conversational but professional. Plain prose for explanations, bullet points for findings, fenced code blocks (with language tags) for code. +- Don't dump tool output verbatim — summarise the relevant lines. +- Don't add a closing recap unless the user asked for one. After completing the work, just stop. + +Tool calls: +- Issue independent tool calls in parallel within one response. +- Use specialised tools over generic ones (e.g. KB search before web search; named connectors over MCP fallback). diff --git a/surfsense_backend/app/agents/new_chat/prompts/providers/openai_codex.md b/surfsense_backend/app/agents/new_chat/prompts/providers/openai_codex.md new file mode 100644 index 000000000..6167d4b06 --- /dev/null +++ b/surfsense_backend/app/agents/new_chat/prompts/providers/openai_codex.md @@ -0,0 +1,19 @@ + +You are running on an OpenAI Codex-class model (gpt-codex / codex-mini / gpt-*-codex). + +Output style: +- Be concise. Don't dump fetched/searched content back at the user — reference paths or chunk ids instead. +- Reference sources as `path:line` (or `chunk:`) so they're clickable. Stand-alone paths per reference, even when repeated. +- Prefer numbered lists (`1.`, `2.`, `3.`) when offering options the user can pick by replying with a single number. +- Skip headers and heavy formatting for simple confirmations. +- No emojis, no em-dashes, no nested bullets. Single-level lists only. + +Code & structured-output tasks: +- Lead with a one-sentence explanation of the change before context. Don't open with "Summary:" — jump in. +- Suggest natural next steps (run tests, diff review, commit) only when they're genuinely the next move. +- For multi-line snippets use fenced code blocks with a language tag. + +Tool calls: +- Run independent tool calls in parallel; chain only when later calls need earlier results. +- Don't ask permission ("Should I proceed?") — proceed with the most reasonable default and state what you did. + diff --git a/surfsense_backend/app/agents/new_chat/prompts/providers/openai_reasoning.md b/surfsense_backend/app/agents/new_chat/prompts/providers/openai_reasoning.md index 935d3f207..dd7a61536 100644 --- a/surfsense_backend/app/agents/new_chat/prompts/providers/openai_reasoning.md +++ b/surfsense_backend/app/agents/new_chat/prompts/providers/openai_reasoning.md @@ -1,5 +1,21 @@ -You are running on an OpenAI reasoning model (o-series / GPT-5+). Be terse and -direct in your responses. When editing files, prefer the `apply_patch` tool format -where available. Avoid restating the user request before answering. +You are running on an OpenAI reasoning model (GPT-5+ / o-series). + +Output style: +- Be terse and direct. Don't restate the user's request before answering. +- Don't begin with conversational openers ("Done!", "Got it", "Great question", "Sure thing"). Get to the answer or the action. +- Match response complexity to the task: simple questions → one-line answer; substantial work → lead with the outcome, then context, then any next steps. +- No nested bullets — keep lists flat (single level). For options the user can pick by replying with a number, use `1.` `2.` `3.`. +- Use inline backticks for paths/commands/identifiers; fenced code blocks (with language tags) for multi-line snippets. + +Channels (for clients that support them): +- `commentary` — short progress updates only when they add genuinely new information (a discovery, a tradeoff, a blocker, the start of a non-trivial step). Don't narrate routine reads or obvious next steps. +- `final` — the completed response. Keep it self-contained; no "see above" / "see below" cross-references. + +Tool calls: +- Parallelise independent tool calls in a single response (`multi_tool_use.parallel` where supported). Only sequence when a later call needs an earlier one's output. +- Don't ask permission ("Should I proceed?", "Do you want me to…?"). Pick the most reasonable default, do it, and state what you did. + +Autonomy: +- Persist until the task is fully resolved within the current turn whenever feasible. Don't stop at analysis when the user clearly wants the change applied. diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py index fcd342d29..75342a8e1 100644 --- a/surfsense_backend/app/db.py +++ b/surfsense_backend/app/db.py @@ -976,7 +976,15 @@ class Document(BaseModel, TimestampMixin): document_metadata = Column(JSON, nullable=True) content = Column(Text, nullable=False) - content_hash = Column(String, nullable=False, index=True, unique=True) + # ``content_hash`` is intentionally NOT globally unique. In a real + # filesystem two files at different paths can hold identical bytes, + # and the agent's ``write_file`` flow needs that semantic to support + # copy / duplicate operations. Path uniqueness lives on + # ``unique_identifier_hash`` (per search space). The hash remains + # indexed because connector indexers consult it as a change-detection + # / cross-source dedup hint via :func:`check_duplicate_document`. + # See migration 133. + content_hash = Column(String, nullable=False, index=True) unique_identifier_hash = Column(String, nullable=True, index=True, unique=True) embedding = Column(Vector(config.embedding_model_instance.dimension)) diff --git a/surfsense_backend/tests/unit/agents/new_chat/prompts/test_composer.py b/surfsense_backend/tests/unit/agents/new_chat/prompts/test_composer.py index d35b7aa8b..d08bbc8cf 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/prompts/test_composer.py +++ b/surfsense_backend/tests/unit/agents/new_chat/prompts/test_composer.py @@ -25,17 +25,33 @@ class TestProviderVariantDetection: @pytest.mark.parametrize( "model_name,expected", [ + # GPT-4 family routes to "classic" (autonomous-persistence style) ("openai:gpt-4o-mini", "openai_classic"), ("openai:gpt-4-turbo", "openai_classic"), + # GPT-5 / o-series route to "reasoning" (channel-aware pragmatic) ("openai:gpt-5", "openai_reasoning"), - ("openai:gpt-5-codex", "openai_reasoning"), ("openai:o1-preview", "openai_reasoning"), ("openai:o3-mini", "openai_reasoning"), + # Codex family beats reasoning (more specific). Mirrors OpenCode + # ``system.ts`` — ``gpt-*-codex`` gets the code-purist prompt. + ("openai:gpt-5-codex", "openai_codex"), + ("openai:gpt-codex", "openai_codex"), + ("openai:codex-mini", "openai_codex"), + # Anthropic + Google ("anthropic:claude-3-5-sonnet", "anthropic"), ("anthropic/claude-opus-4", "anthropic"), ("google:gemini-2.0-flash", "google"), ("vertex:gemini-1.5-pro", "google"), + # Newly-covered families + ("moonshot:kimi-k2", "kimi"), + ("openrouter:moonshot/kimi-k2.5", "kimi"), + ("xai:grok-2", "grok"), + ("openrouter:x-ai/grok-3", "grok"), + ("openai:deepseek-v3", "deepseek"), + ("deepseek:deepseek-r1", "deepseek"), + # Unknown families fall back to default (no provider block emitted) ("groq:mixtral-8x7b", "default"), + ("together:llama-3.1-70b", "default"), (None, "default"), ("", "default"), ], @@ -43,6 +59,16 @@ class TestProviderVariantDetection: def test_detection(self, model_name: str | None, expected: str) -> None: assert detect_provider_variant(model_name) == expected + def test_codex_takes_precedence_over_reasoning(self) -> None: + """Regression guard: ``gpt-5-codex`` must NOT match the generic + ``gpt-5`` reasoning regex first. Codex is the more specialised + prompt and mirrors OpenCode's dispatch order. + """ + from app.agents.new_chat.prompts.composer import detect_provider_variant + + assert detect_provider_variant("openai:gpt-5-codex") == "openai_codex" + assert detect_provider_variant("openai:gpt-5") == "openai_reasoning" + class TestCompose: def test_default_prompt_has_required_blocks(self, fixed_today: datetime) -> None: @@ -149,6 +175,52 @@ class TestCompose: prompt = compose_system_prompt(today=fixed_today, model_name="custom:foo") assert "" not in prompt + @pytest.mark.parametrize( + "model_name,expected_marker", + [ + # Each marker is a unique-ish phrase from the corresponding fragment. + # If a fragment is renamed/rewritten such that the marker is gone, + # update both the fragment and this test deliberately. + ("openai:gpt-5-codex", "Codex-class"), + ("openai:gpt-5", "OpenAI reasoning model"), + ("openai:gpt-4o", "classic OpenAI chat model"), + ("anthropic:claude-3-5-sonnet", "Anthropic Claude"), + ("google:gemini-2.0-flash", "Google Gemini"), + ("moonshot:kimi-k2", "Moonshot Kimi"), + ("xai:grok-2", "xAI Grok"), + ("deepseek:deepseek-r1", "DeepSeek"), + ], + ) + def test_each_known_variant_renders_with_its_marker( + self, + fixed_today: datetime, + model_name: str, + expected_marker: str, + ) -> None: + """Every supported variant must produce a ```` block + containing its identifying marker. This pins the dispatch + the + on-disk fragments together so a missing/renamed file is caught + immediately. + """ + prompt = compose_system_prompt(today=fixed_today, model_name=model_name) + assert "" in prompt, ( + f"variant for {model_name!r} did not emit a provider_hints block; " + "the corresponding providers/.md may be missing" + ) + assert expected_marker in prompt, ( + f"variant for {model_name!r} emitted hints but lacked the " + f"expected marker {expected_marker!r} — the fragment may have " + "drifted from the dispatch table" + ) + + def test_provider_blocks_are_byte_stable_across_calls( + self, fixed_today: datetime + ) -> None: + """Cache-stability guard: same model id → byte-identical prompt.""" + a = compose_system_prompt(today=fixed_today, model_name="moonshot:kimi-k2") + b = compose_system_prompt(today=fixed_today, model_name="moonshot:kimi-k2") + assert a == b + def test_custom_system_instructions_override_default( self, fixed_today: datetime ) -> None: diff --git a/surfsense_backend/tests/unit/middleware/test_kb_persistence_filesystem_parity.py b/surfsense_backend/tests/unit/middleware/test_kb_persistence_filesystem_parity.py new file mode 100644 index 000000000..8b464d48d --- /dev/null +++ b/surfsense_backend/tests/unit/middleware/test_kb_persistence_filesystem_parity.py @@ -0,0 +1,168 @@ +"""Unit tests for kb_persistence filesystem-parity invariants. + +Specifically, these tests pin down that the agent-driven write_file flow +treats path uniqueness — not content uniqueness — as the only hard +invariant. This mirrors a real filesystem: ``cp a b`` produces two files +with identical bytes living at different paths, and that should round-trip +through :class:`KnowledgeBasePersistenceMiddleware` without losing the copy. +""" + +from __future__ import annotations + +from typing import Any +from unittest.mock import AsyncMock, MagicMock + +import numpy as np +import pytest + +from app.agents.new_chat.middleware import kb_persistence +from app.db import Document + + +class _FakeResult: + """Minimal stand-in for ``sqlalchemy.engine.Result``.""" + + def __init__(self, value: Any = None) -> None: + self._value = value + + def scalar_one_or_none(self) -> Any: + return self._value + + def scalar(self) -> Any: + return self._value + + +class _FakeSession: + """Minimal AsyncSession stand-in scoped to ``_create_document`` needs. + + Records every ``add`` so we can assert against the resulting Documents + and Chunks. ``execute`` always returns "no row" by default — i.e. no + folder hierarchy preexists and no path collision exists. Tests that + want a path collision can override that on a per-call basis. + """ + + def __init__(self) -> None: + self.added: list[Any] = [] + self.execute = AsyncMock(return_value=_FakeResult(None)) + self.flush = AsyncMock() + + # Simulate ``await session.flush()`` assigning an id to the doc; + # we increment a counter so each Document gets a unique id. + self._next_id = 1 + + async def _flush_assigning_ids() -> None: + for obj in self.added: + if getattr(obj, "id", None) is None: + obj.id = self._next_id + self._next_id += 1 + + self.flush.side_effect = _flush_assigning_ids + + def add(self, obj: Any) -> None: + self.added.append(obj) + + def add_all(self, objs: list[Any]) -> None: + self.added.extend(objs) + + +@pytest.fixture(autouse=True) +def _stub_embeddings_and_chunks(monkeypatch: pytest.MonkeyPatch) -> None: + """Avoid loading the embedding model in unit tests.""" + monkeypatch.setattr( + kb_persistence, + "embed_texts", + lambda texts: [np.zeros(8, dtype=np.float32) for _ in texts], + ) + monkeypatch.setattr(kb_persistence, "chunk_text", lambda content: [content]) + + +@pytest.mark.asyncio +async def test_create_document_allows_identical_content_at_different_paths() -> None: + """The core regression: ``cp /a/notes.md /b/notes-copy.md``. + + Both create calls must succeed even though the bytes are byte-for-byte + identical, because path is the only filesystem-style unique key. + """ + session = _FakeSession() + content = "# Same body\n\nIdentical content used by two different paths.\n" + + first = await kb_persistence._create_document( + session, # type: ignore[arg-type] + virtual_path="/documents/a/notes.md", + content=content, + search_space_id=42, + created_by_id="user-1", + ) + assert isinstance(first, Document) + assert first.title == "notes.md" + + # Second create with byte-identical content at a different path should + # not raise — that's the whole point of the filesystem-parity fix. + second = await kb_persistence._create_document( + session, # type: ignore[arg-type] + virtual_path="/documents/b/notes-copy.md", + content=content, + search_space_id=42, + created_by_id="user-1", + ) + assert isinstance(second, Document) + assert second.title == "notes-copy.md" + + # Both rows share the same content_hash but live at distinct paths + # (distinct ``unique_identifier_hash``). That's the desired contract. + assert first.content_hash == second.content_hash + assert first.unique_identifier_hash != second.unique_identifier_hash + + +@pytest.mark.asyncio +async def test_create_document_still_rejects_path_collision() -> None: + """Path uniqueness remains the hard invariant. + + If ``unique_identifier_hash`` already points at an existing row in + the same search space, the create call must raise ``ValueError`` + with a clear message — matching the behavior the commit loop relies + on to upsert via the existing-row code path. + """ + session = _FakeSession() + + # Path with no folder parts so ``_ensure_folder_hierarchy`` is a + # no-op and the only SELECT executed is the path-collision check. + # That SELECT returns an existing doc id, triggering the guard. + session.execute = AsyncMock(return_value=_FakeResult(value=99)) + + with pytest.raises(ValueError, match="already exists at path"): + await kb_persistence._create_document( + session, # type: ignore[arg-type] + virtual_path="/documents/notes.md", + content="anything", + search_space_id=42, + created_by_id="user-1", + ) + + +@pytest.mark.asyncio +async def test_create_document_does_not_query_for_content_hash_collision( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Regression guard: the legacy second SELECT (content_hash collision + pre-check) must be gone. Counting ``execute`` calls is a brittle but + effective way to lock that in. + + The current flow runs exactly one ``execute`` for the path-collision + SELECT (no folder parts in this path → ``_ensure_folder_hierarchy`` + short-circuits). If a future refactor reintroduces a content-hash + SELECT, this test will fail loud. + """ + session = _FakeSession() + await kb_persistence._create_document( + session, # type: ignore[arg-type] + virtual_path="/documents/notes.md", + content="hello", + search_space_id=42, + created_by_id="user-1", + ) + # Path-collision SELECT only. No content_hash SELECT. + assert session.execute.await_count == 1, ( + f"Unexpected execute count {session.execute.await_count}; " + "did the legacy content_hash collision pre-check get re-added?" + ) diff --git a/surfsense_web/app/globals.css b/surfsense_web/app/globals.css index a37ddb8f3..f54bc2197 100644 --- a/surfsense_web/app/globals.css +++ b/surfsense_web/app/globals.css @@ -210,6 +210,27 @@ button { } } +/* Citation-jump highlight — entrance pulse only. The `SearchHighlightLeaf` + (see components/ui/search-highlight-node.tsx) is otherwise statically + tinted; this animation runs once on mount to draw the eye to the cited + text after `scrollIntoView` lands. The highlight itself is permanent + until the user clicks inside the editor (or another dismissal trigger + fires in `EditorPanelContent`). */ +@keyframes citation-flash-in { + 0% { + background-color: transparent; + box-shadow: 0 0 0 0 transparent; + } + 40% { + background-color: color-mix(in oklab, var(--primary) 30%, transparent); + box-shadow: 0 0 0 3px color-mix(in oklab, var(--primary) 25%, transparent); + } + 100% { + background-color: color-mix(in oklab, var(--primary) 15%, transparent); + box-shadow: 0 0 0 1px color-mix(in oklab, var(--primary) 40%, transparent); + } +} + /* Human-in-the-loop approval card animations */ @keyframes pulse-subtle { 0%, diff --git a/surfsense_web/atoms/document-viewer/pending-chunk-highlight.atom.ts b/surfsense_web/atoms/document-viewer/pending-chunk-highlight.atom.ts new file mode 100644 index 000000000..a3f8357e8 --- /dev/null +++ b/surfsense_web/atoms/document-viewer/pending-chunk-highlight.atom.ts @@ -0,0 +1,19 @@ +import { atom } from "jotai"; + +/** + * Cross-component handoff for citation jumps. Set by `InlineCitation` when a + * numeric chunk badge is clicked (after the document has been resolved); read + * by `DocumentTabContent` once the matching document tab mounts so it can + * scroll to and softly highlight the cited chunk inside the rendered markdown. + * + * Cleared by `DocumentTabContent` only after a terminal state — exact / + * approximate / miss — has been reached, so that an escalation refetch (2MB + * preview → 16MB) keeps the pending intent alive across the re-render. + */ +export interface PendingChunkHighlight { + documentId: number; + chunkId: number; + chunkText: string; +} + +export const pendingChunkHighlightAtom = atom(null); diff --git a/surfsense_web/components/assistant-ui/inline-citation.tsx b/surfsense_web/components/assistant-ui/inline-citation.tsx index eb4bd9af8..ae8d434a8 100644 --- a/surfsense_web/components/assistant-ui/inline-citation.tsx +++ b/surfsense_web/components/assistant-ui/inline-citation.tsx @@ -1,26 +1,45 @@ "use client"; -import { FileText } from "lucide-react"; +import { useQuery, useQueryClient } from "@tanstack/react-query"; +import { useSetAtom } from "jotai"; +import { ExternalLink, FileText } from "lucide-react"; import type { FC } from "react"; -import { useState } from "react"; +import { useCallback, useEffect, useRef, useState } from "react"; +import { toast } from "sonner"; +import { pendingChunkHighlightAtom } from "@/atoms/document-viewer/pending-chunk-highlight.atom"; +import { openEditorPanelAtom } from "@/atoms/editor/editor-panel.atom"; import { useCitationMetadata } from "@/components/assistant-ui/citation-metadata-context"; -import { SourceDetailPanel } from "@/components/new-chat/source-detail-panel"; +import { MarkdownViewer } from "@/components/markdown-viewer"; import { Citation } from "@/components/tool-ui/citation"; +import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; +import { Spinner } from "@/components/ui/spinner"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; +import { documentsApiService } from "@/lib/apis/documents-api.service"; +import { cacheKeys } from "@/lib/query-client/cache-keys"; interface InlineCitationProps { chunkId: number; isDocsChunk?: boolean; } +const POPOVER_HOVER_CLOSE_DELAY_MS = 150; + /** - * Inline citation for knowledge-base chunks (numeric chunk IDs). - * Renders a clickable badge showing the actual chunk ID that opens the SourceDetailPanel. - * Negative chunk IDs indicate anonymous/synthetic uploads and render as a static badge. + * Inline citation badge for knowledge-base chunks (numeric chunk IDs) and + * Surfsense documentation chunks (`isDocsChunk`). Negative chunk IDs render as + * a static "doc" pill (anonymous/synthetic uploads). + * + * Numeric KB chunks: clicking resolves the parent document via + * `getDocumentByChunk`, opens the document in the right side panel (alongside + * the chat — does not replace it), and stages the cited chunk text in + * `pendingChunkHighlightAtom` so `EditorPanelContent` can scroll to and softly + * highlight it inside the rendered markdown. + * + * Surfsense docs chunks: rendered as a hover-controlled shadcn Popover that + * lazily fetches and previews the cited chunk inline, since those docs aren't + * indexed into the user's search space and have no tab to open. */ export const InlineCitation: FC = ({ chunkId, isDocsChunk = false }) => { - const [isOpen, setIsOpen] = useState(false); - if (chunkId < 0) { return ( @@ -38,26 +57,185 @@ export const InlineCitation: FC = ({ chunkId, isDocsChunk = ); } + if (isDocsChunk) { + return ; + } + + return ; +}; + +const NumericChunkCitation: FC<{ chunkId: number }> = ({ chunkId }) => { + const queryClient = useQueryClient(); + const setPendingHighlight = useSetAtom(pendingChunkHighlightAtom); + const openEditorPanel = useSetAtom(openEditorPanelAtom); + const [resolving, setResolving] = useState(false); + + const handleClick = useCallback(async () => { + if (resolving) return; + setResolving(true); + console.log("[citation:click] start", { chunkId }); + try { + const data = await queryClient.fetchQuery({ + // Local key with explicit window. The shared `cacheKeys.documents.byChunk` + // is window-agnostic (latent footgun); namespace the call to avoid + // reusing a different-window cached result. + queryKey: ["documents", "by-chunk", chunkId, "w0"] as const, + queryFn: () => + documentsApiService.getDocumentByChunk({ chunk_id: chunkId, chunk_window: 0 }), + staleTime: 5 * 60 * 1000, + }); + const cited = data.chunks.find((c) => c.id === chunkId) ?? data.chunks[0]; + console.log("[citation:click] fetched doc-by-chunk", { + docId: data.id, + docTitle: data.title, + chunksReturned: data.chunks.length, + citedChunkId: cited?.id, + citedChunkContentLen: cited?.content?.length ?? 0, + citedChunkPreview: + cited?.content && cited.content.length > 120 + ? `${cited.content.slice(0, 120)}…(+${cited.content.length - 120})` + : (cited?.content ?? ""), + }); + // Stage the highlight BEFORE opening the panel so `EditorPanelContent` + // already sees the pending intent on its very first render — avoids a + // "fetch → render → no-pending → next-tick render with pending" race. + setPendingHighlight({ + documentId: data.id, + chunkId, + chunkText: cited?.content ?? "", + }); + openEditorPanel({ + documentId: data.id, + searchSpaceId: data.search_space_id, + title: data.title, + }); + console.log("[citation:click] staged highlight + opened editor panel", { + documentId: data.id, + }); + } catch (err) { + console.warn("[citation:click] failed", err); + toast.error(err instanceof Error ? err.message : "Couldn't open cited document"); + } finally { + setResolving(false); + } + }, [chunkId, openEditorPanel, queryClient, resolving, setPendingHighlight]); + return ( - - + ); +}; + +const SurfsenseDocCitation: FC<{ chunkId: number }> = ({ chunkId }) => { + const [open, setOpen] = useState(false); + const closeTimerRef = useRef | null>(null); + + const cancelClose = useCallback(() => { + if (closeTimerRef.current) { + clearTimeout(closeTimerRef.current); + closeTimerRef.current = null; + } + }, []); + + const scheduleClose = useCallback(() => { + cancelClose(); + closeTimerRef.current = setTimeout(() => { + setOpen(false); + closeTimerRef.current = null; + }, POPOVER_HOVER_CLOSE_DELAY_MS); + }, [cancelClose]); + + useEffect(() => () => cancelClose(), [cancelClose]); + + const { data, isLoading, error } = useQuery({ + queryKey: cacheKeys.documents.byChunk(`doc-${chunkId}`), + queryFn: () => documentsApiService.getSurfsenseDocByChunk(chunkId), + enabled: open, + staleTime: 5 * 60 * 1000, + }); + + const citedChunk = data?.chunks.find((c) => c.id === chunkId) ?? data?.chunks[0]; + + return ( + + + + + e.preventDefault()} > - {chunkId} - - +
+
+

+ {data?.title ?? "Surfsense documentation"} +

+

Chunk #{chunkId}

+
+ {data?.source && ( + + + Open + + )} +
+
+ {isLoading && ( +
+ + Loading… +
+ )} + {error && ( +

+ {error instanceof Error ? error.message : "Failed to load chunk"} +

+ )} + {!isLoading && !error && citedChunk?.content && ( + + )} + {!isLoading && !error && !citedChunk?.content && ( +

No content available.

+ )} +
+ + ); }; diff --git a/surfsense_web/components/editor-panel/editor-panel.tsx b/surfsense_web/components/editor-panel/editor-panel.tsx index 3b69ae6e0..0c4e9485b 100644 --- a/surfsense_web/components/editor-panel/editor-panel.tsx +++ b/surfsense_web/components/editor-panel/editor-panel.tsx @@ -1,5 +1,6 @@ "use client"; +import { FindReplacePlugin } from "@platejs/find-replace"; import { useAtomValue, useSetAtom } from "jotai"; import { Check, @@ -14,17 +15,21 @@ import { import dynamic from "next/dynamic"; import { useCallback, useEffect, useRef, useState } from "react"; import { toast } from "sonner"; +import { pendingChunkHighlightAtom } from "@/atoms/document-viewer/pending-chunk-highlight.atom"; import { closeEditorPanelAtom, editorPanelAtom } from "@/atoms/editor/editor-panel.atom"; import { VersionHistoryButton } from "@/components/documents/version-history"; +import type { PlateEditorInstance } from "@/components/editor/plate-editor"; import { SourceCodeEditor } from "@/components/editor/source-code-editor"; import { MarkdownViewer } from "@/components/markdown-viewer"; import { Alert, AlertDescription } from "@/components/ui/alert"; import { Button } from "@/components/ui/button"; import { Drawer, DrawerContent, DrawerHandle, DrawerTitle } from "@/components/ui/drawer"; +import { CITATION_HIGHLIGHT_CLASS } from "@/components/ui/search-highlight-node"; import { Spinner } from "@/components/ui/spinner"; import { useMediaQuery } from "@/hooks/use-media-query"; import { useElectronAPI } from "@/hooks/use-platform"; import { authenticatedFetch, getBearerToken, redirectToLogin } from "@/lib/auth-utils"; +import { buildCitationSearchCandidates } from "@/lib/citation-search"; import { inferMonacoLanguageFromPath } from "@/lib/editor-language"; const PlateEditor = dynamic( @@ -32,7 +37,10 @@ const PlateEditor = dynamic( { ssr: false, loading: () => } ); +type CitationHighlightStatus = "exact" | "miss"; + const LARGE_DOCUMENT_THRESHOLD = 2 * 1024 * 1024; // 2MB +const CITATION_MAX_LENGTH = 16 * 1024 * 1024; // 16MB on-demand cap for citation jumps interface EditorContent { document_id: number; @@ -136,6 +144,61 @@ export function EditorPanelContent({ const [displayTitle, setDisplayTitle] = useState(title || "Untitled"); const isLocalFileMode = kind === "local_file"; const editorRenderMode: EditorRenderMode = isLocalFileMode ? "source_code" : "rich_markdown"; + + // --- Citation-jump highlight wiring ---------------------------------- + // `EditorPanelContent` is the consumer of `pendingChunkHighlightAtom`: when + // a citation badge is clicked, the badge stages `{documentId, chunkId, + // chunkText}` and opens this panel. We drive Plate's `FindReplacePlugin` + // (registered in every preset) to highlight the cited text natively via + // Slate decorations — no DOM walking, no Range gymnastics. The state + // machine below escalates the document fetch from 2MB → 16MB once if no + // candidate snippet matched in the preview, and surfaces miss outcomes + // via an inline alert. + const pending = useAtomValue(pendingChunkHighlightAtom); + const setPendingHighlight = useSetAtom(pendingChunkHighlightAtom); + const [fetchKey, setFetchKey] = useState(0); + const [maxLengthOverride, setMaxLengthOverride] = useState(null); + const [highlightResult, setHighlightResult] = useState(null); + const editorRef = useRef(null); + const escalatedForRef = useRef(null); + const lastAppliedChunkIdRef = useRef(null); + // Tracks whether a citation highlight is currently decorated in the + // editor. We use a ref (not state) because the click-to-dismiss handler + // runs in a stable callback that would otherwise close over stale state. + const isHighlightActiveRef = useRef(false); + // Once a citation jump targets this doc we have to keep `PlateEditor` + // mounted for the *rest of the doc session* — even after the highlight + // effect clears `pendingChunkHighlightAtom` (which it does as soon as + // the decoration is applied, so a follow-up citation on the same chunk + // can re-trigger). Without this latch, non-editable docs would re-render + // back into `MarkdownViewer` the instant `pending` is released, tearing + // down the Plate decorations and dropping the highlight after a frame. + const [stickyPlateMode, setStickyPlateMode] = useState(false); + + const clearCitationSearch = useCallback(() => { + isHighlightActiveRef.current = false; + const editor = editorRef.current; + if (!editor) return; + try { + editor.setOption(FindReplacePlugin, "search", ""); + editor.api.redecorate(); + } catch (err) { + console.warn("[EditorPanelContent] clearCitationSearch failed:", err); + } + }, []); + + // Dismiss the highlight when the user interacts with the editor surface. + // `onPointerDown` fires before focus / selection changes so the click + // itself feels responsive — the highlight clears in the same event tick + // that places the cursor. No-op when nothing is highlighted, so we don't + // thrash `redecorate` on every click in normal editing. + const handleEditorPointerDown = useCallback(() => { + if (!isHighlightActiveRef.current) return; + clearCitationSearch(); + setHighlightResult(null); + }, [clearCitationSearch]); + + const isCitationTarget = !!pending && !isLocalFileMode && pending.documentId === documentId; const resolveLocalVirtualPath = useCallback( async (candidatePath: string): Promise => { if (!electronAPI?.getAgentFilesystemMounts) { @@ -155,6 +218,8 @@ export function EditorPanelContent({ const isLargeDocument = (editorDoc?.content_size_bytes ?? 0) > LARGE_DOCUMENT_THRESHOLD; + // `fetchKey` is an explicit re-fetch trigger (escalation bumps it to force + // a new request even when documentId/searchSpaceId haven't changed). useEffect(() => { const controller = new AbortController(); setIsLoading(true); @@ -166,6 +231,12 @@ export function EditorPanelContent({ setIsEditing(false); initialLoadDone.current = false; changeCountRef.current = 0; + // Clear any in-flight FindReplacePlugin search before the editor + // re-mounts on new content (a fresh editor key is generated below + // from documentId + isEditing, so the previous editor + its + // decorations are about to be discarded anyway, but we belt-and- + // brace here for the case where only `fetchKey` changed). + clearCitationSearch(); const doFetch = async () => { try { @@ -210,7 +281,11 @@ export function EditorPanelContent({ const url = new URL( `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-spaces/${searchSpaceId}/documents/${documentId}/editor-content` ); - url.searchParams.set("max_length", String(LARGE_DOCUMENT_THRESHOLD)); + url.searchParams.set("max_length", String(maxLengthOverride ?? LARGE_DOCUMENT_THRESHOLD)); + // `fetchKey` participates here so biome's noUnusedVariables sees it + // as consumed; bumping it forces a fresh request even when the URL + // is otherwise identical. + if (fetchKey > 0) url.searchParams.set("_n", String(fetchKey)); const response = await authenticatedFetch(url.toString(), { method: "GET" }); @@ -256,8 +331,259 @@ export function EditorPanelContent({ resolveLocalVirtualPath, searchSpaceId, title, + fetchKey, + maxLengthOverride, + clearCitationSearch, ]); + // Reset citation-jump bookkeeping whenever the panel switches to a different + // document (or local file). Body only writes setters — the deps are the + // real triggers we want to react to. + // biome-ignore lint/correctness/useExhaustiveDependencies: documentId/localFilePath are intentional triggers. + useEffect(() => { + clearCitationSearch(); + escalatedForRef.current = null; + lastAppliedChunkIdRef.current = null; + setHighlightResult(null); + setMaxLengthOverride(null); + setFetchKey(0); + // Drop sticky Plate mode when the panel moves to a different doc + // — the next doc starts in its preferred render mode (Plate for + // editable, MarkdownViewer for everything else) until/unless a + // citation jump targets it. + setStickyPlateMode(false); + }, [documentId, localFilePath, clearCitationSearch]); + + // Latch sticky Plate mode the first time a citation jump targets this + // doc. We keep it sticky for the remainder of this doc session so the + // highlight effect's `setPendingHighlight(null)` doesn't unmount the + // editor mid-flight (see comment on `stickyPlateMode` declaration). + useEffect(() => { + if (isCitationTarget) setStickyPlateMode(true); + }, [isCitationTarget]); + + // `isEditorReady` is what `useEffect` actually depends on — `editorRef` + // is a ref so changes don't trigger re-runs. We flip this to `true` once + // `PlateEditor` calls back with its live editor instance (its + // `usePlateEditor` value-init runs synchronously, so by the time this + // flips true the markdown is already deserialized into the Slate tree). + const [isEditorReady, setIsEditorReady] = useState(false); + const handleEditorReady = useCallback((editor: PlateEditorInstance | null) => { + console.log("[citation:editor] handleEditorReady", { ready: !!editor }); + editorRef.current = editor; + setIsEditorReady(!!editor); + }, []); + + // --- Citation jump highlight effect ----------------------------------- + // Drives Plate's FindReplacePlugin to highlight the cited chunk: + // 1. Build candidate snippets from the chunk text (first sentence, + // first 8 words, full chunk if short). Plate's decorate runs per- + // block and won't cross block boundaries, so the shorter + // candidates exist to give us something that fits in one + // paragraph / heading. + // 2. For each candidate: setOption('search', ...) → redecorate → + // wait two animation frames for React to flush → query the editor + // DOM for `.${CITATION_HIGHLIGHT_CLASS}`. First hit wins. + // + // Why a className and not a `data-*` attribute? Plate's + // `PlateLeaf` runs its props through `useNodeAttributes`, which + // only forwards `attributes`, `className`, `ref`, and `style` — + // arbitrary `data-*` attributes are silently dropped. `className` + // is the only escape hatch guaranteed to survive into the DOM. + // 3. On hit: smooth-scroll the first match into view, mark the + // highlight active (so a click inside the editor can dismiss it), + // release the pending atom. + // 4. On terminal miss: if the doc was truncated and we haven't + // escalated yet, bump the fetch's `max_length` to the citation + // cap and re-fetch — the post-refetch render will re-run this + // effect against the larger preview. Otherwise, release the + // atom and show the miss alert. + useEffect(() => { + console.log("[citation:effect] fired", { + isCitationTarget, + pendingDocId: pending?.documentId, + pendingChunkId: pending?.chunkId, + pendingChunkTextLen: pending?.chunkText?.length, + documentId, + isLocalFileMode, + isEditing, + hasMarkdown: !!editorDoc?.source_markdown, + markdownLen: editorDoc?.source_markdown?.length, + truncated: editorDoc?.truncated, + isEditorReady, + editorRefSet: !!editorRef.current, + maxLengthOverride, + }); + if (!isCitationTarget || !pending) { + console.log("[citation:effect] guard ✗ no citation target / no pending"); + return; + } + if (isLocalFileMode || isEditing) { + console.log("[citation:effect] guard ✗ localFileMode/editing"); + return; + } + if (!editorDoc?.source_markdown) { + console.log("[citation:effect] guard ✗ source_markdown not ready"); + return; + } + if (!isEditorReady) { + console.log("[citation:effect] guard ✗ editor not ready yet"); + return; + } + const editor = editorRef.current; + if (!editor) { + console.log("[citation:effect] guard ✗ editorRef.current is null"); + return; + } + + if (lastAppliedChunkIdRef.current !== pending.chunkId) { + lastAppliedChunkIdRef.current = pending.chunkId; + } + + let cancelled = false; + + const finishMiss = () => { + console.log("[citation:effect] terminal miss — no candidate matched"); + try { + editor.setOption(FindReplacePlugin, "search", ""); + editor.api.redecorate(); + } catch (err) { + console.warn("[EditorPanelContent] reset search after miss failed:", err); + } + const canEscalate = + editorDoc.truncated === true && + (maxLengthOverride ?? LARGE_DOCUMENT_THRESHOLD) < CITATION_MAX_LENGTH && + escalatedForRef.current !== pending.chunkId; + console.log("[citation:effect] miss decision", { + truncated: editorDoc.truncated, + currentMaxLength: maxLengthOverride ?? LARGE_DOCUMENT_THRESHOLD, + canEscalate, + }); + if (canEscalate) { + escalatedForRef.current = pending.chunkId; + setMaxLengthOverride(CITATION_MAX_LENGTH); + setFetchKey((k) => k + 1); + // Keep the atom set so the post-refetch render re-runs. + return; + } + setHighlightResult("miss"); + setPendingHighlight(null); + }; + + const tryCandidates = async () => { + const candidates = buildCitationSearchCandidates(pending.chunkText); + console.log("[citation:effect] candidates built", { + count: candidates.length, + previews: candidates.map((c) => c.slice(0, 60)), + }); + if (candidates.length === 0) { + if (!cancelled) finishMiss(); + return; + } + // Resolve the editor's rendered DOM root via Slate's stable + // `[data-slate-editor="true"]` attribute (set by slate-react's + // ``). Scoping queries to this root prevents + // `` elements rendered elsewhere on the page (e.g. chat + // search-highlight leaves in another mounted PlateEditor) from + // being mistaken for citation hits. + const editorRoot = document.querySelector('[data-slate-editor="true"]'); + console.log("[citation:effect] editor root", { + hasRoot: !!editorRoot, + }); + const root: ParentNode = editorRoot ?? document; + + for (let i = 0; i < candidates.length; i++) { + const candidate = candidates[i]; + if (cancelled) return; + try { + editor.setOption(FindReplacePlugin, "search", candidate); + editor.api.redecorate(); + console.log(`[citation:effect] try #${i} setOption + redecorate`, { + len: candidate.length, + preview: candidate.slice(0, 80), + }); + } catch (err) { + console.warn("[EditorPanelContent] setOption/redecorate failed:", err); + continue; + } + // Two rAFs: first lets Slate flush its onChange, second lets + // React commit the decoration leaves into the DOM. + await new Promise((resolve) => + requestAnimationFrame(() => requestAnimationFrame(() => resolve())) + ); + if (cancelled) return; + // Primary probe: by our stable class on the rendered . + let el = root.querySelector(`.${CITATION_HIGHLIGHT_CLASS}`); + const classMarkCount = root.querySelectorAll(`.${CITATION_HIGHLIGHT_CLASS}`).length; + // Diagnostic fallback: any inside the editor root. + // If we ever see allMarks > 0 but classMarkCount === 0, + // the className was stripped again and we need to revisit + // `useNodeAttributes` filtering. + const allMarkCount = root.querySelectorAll("mark").length; + if (!el && allMarkCount > 0) { + el = root.querySelector("mark"); + } + console.log(`[citation:effect] try #${i} DOM probe`, { + foundEl: !!el, + classMarkCount, + allMarkCount, + usedFallback: !!el && classMarkCount === 0, + }); + if (el) { + try { + el.scrollIntoView({ block: "center", behavior: "smooth" }); + } catch { + el.scrollIntoView(); + } + isHighlightActiveRef.current = true; + setHighlightResult("exact"); + console.log(`[citation:effect] ✓ exact via candidate #${i} — atom released`); + // No auto-clear timer — the highlight is intentionally + // permanent until the user clicks inside the editor (see + // `handleEditorPointerDown`) or another dismissal trigger + // fires (doc switch, edit-mode toggle, panel unmount, + // next citation jump). Sticky Plate mode keeps the + // editor mounted after the atom clears. + setPendingHighlight(null); + return; + } + } + if (!cancelled) finishMiss(); + }; + + void tryCandidates(); + + return () => { + cancelled = true; + }; + }, [ + isCitationTarget, + pending, + documentId, + editorDoc?.source_markdown, + editorDoc?.truncated, + isLocalFileMode, + isEditing, + isEditorReady, + maxLengthOverride, + clearCitationSearch, + setPendingHighlight, + ]); + + // Cleanup any active highlight on unmount. + useEffect(() => { + return () => clearCitationSearch(); + }, [clearCitationSearch]); + + // Toggling into edit mode swaps Plate out of readOnly. Clear the citation + // search so stale leaves don't linger in the editing surface. + useEffect(() => { + if (isEditing) { + clearCitationSearch(); + setHighlightResult(null); + } + }, [isEditing, clearCitationSearch]); + useEffect(() => { return () => { if (copyResetTimeoutRef.current) { @@ -367,6 +693,15 @@ export function EditorPanelContent({ EDITABLE_DOCUMENT_TYPES.has(editorDoc.document_type ?? "")) && !isLargeDocument : false; + // Use PlateEditor for any of: + // - Editable doc types (FILE/NOTE) — existing editing UX. + // - Active citation jump in flight (`isCitationTarget`) — covers the + // mount in the very first render where the atom is set but the + // sticky effect hasn't fired yet. + // - Sticky Plate mode latched on a previous citation jump — keeps + // the editor mounted (with its decorations) after the highlight + // effect clears the atom. Resets when the doc changes. + const renderInPlateEditor = isEditableType || isCitationTarget || stickyPlateMode; const hasUnsavedChanges = editedMarkdown !== null; const showDesktopHeader = !!onClose; const showEditingActions = isEditableType && isEditing; @@ -381,6 +716,90 @@ export function EditorPanelContent({ setIsEditing(false); }, [editorDoc?.source_markdown]); + const handleDownloadMarkdown = useCallback(async () => { + if (!searchSpaceId || !documentId) return; + setDownloading(true); + try { + const response = await authenticatedFetch( + `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-spaces/${searchSpaceId}/documents/${documentId}/download-markdown`, + { method: "GET" } + ); + if (!response.ok) throw new Error("Download failed"); + const blob = await response.blob(); + const url = URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + const disposition = response.headers.get("content-disposition"); + const match = disposition?.match(/filename="(.+)"/); + a.download = match?.[1] ?? `${editorDoc?.title || "document"}.md`; + document.body.appendChild(a); + a.click(); + a.remove(); + URL.revokeObjectURL(url); + toast.success("Download started"); + } catch { + toast.error("Failed to download document"); + } finally { + setDownloading(false); + } + }, [documentId, editorDoc?.title, searchSpaceId]); + + // We no longer surface an "approximate" status — Plate's FindReplacePlugin + // either decorates an exact match or it doesn't, and the candidate snippet + // strategy (first sentence → first 8 words → full chunk) means we either + // land on the citation start or fall through to the miss alert. + const showMissAlert = isCitationTarget && highlightResult === "miss"; + + const citationAlerts = showMissAlert && ( + + + + Cited section couldn't be located in this view. + {editorDoc?.truncated && ( + + )} + + + ); + + const largeDocAlert = isLargeDocument && !isLocalFileMode && editorDoc && ( + + + + + This document is too large for the editor ( + {Math.round((editorDoc.content_size_bytes ?? 0) / 1024 / 1024)}MB,{" "} + {editorDoc.chunk_count ?? 0} chunks). Showing a preview below. + + + + + ); + return ( <> {showDesktopHeader ? ( @@ -565,61 +984,6 @@ export function EditorPanelContent({

- ) : isLargeDocument && !isLocalFileMode ? ( -
- - - - - This document is too large for the editor ( - {Math.round((editorDoc.content_size_bytes ?? 0) / 1024 / 1024)}MB,{" "} - {editorDoc.chunk_count ?? 0} chunks). Showing a preview below. - - - - - -
) : editorRenderMode === "source_code" ? (
- ) : isEditableType ? ( - + ) : isLargeDocument && !isLocalFileMode && !isCitationTarget ? ( + // Large doc, no active citation — fast Streamdown preview + // + download CTA. We only fall back to MarkdownViewer here + // because Plate is heavy on multi-MB docs and the user + // isn't waiting on a specific citation to render. +
+ {largeDocAlert} + +
+ ) : renderInPlateEditor ? ( + // Editable doc (FILE/NOTE) OR active citation jump (any + // doc type). The citation path uses Plate's + // FindReplacePlugin for native, decoration-based + // highlighting — see the citation-jump highlight effect + // above for how `editorRef` and `handleEditorReady` are + // wired. +
+ {(citationAlerts || (isLargeDocument && isCitationTarget && !isLocalFileMode)) && ( +
+ {isLargeDocument && isCitationTarget && largeDocAlert} + {citationAlerts} +
+ )} +
+ +
+
) : (
diff --git a/surfsense_web/components/editor/plate-editor.tsx b/surfsense_web/components/editor/plate-editor.tsx index 481a420fb..eef18ef6a 100644 --- a/surfsense_web/components/editor/plate-editor.tsx +++ b/surfsense_web/components/editor/plate-editor.tsx @@ -12,6 +12,12 @@ import { type EditorPreset, presetMap } from "@/components/editor/presets"; import { escapeMdxExpressions } from "@/components/editor/utils/escape-mdx"; import { Editor, EditorContainer } from "@/components/ui/editor"; +/** Live editor instance returned by `usePlateEditor`. Exposed via the + * `onEditorReady` prop so callers (e.g. `EditorPanelContent`) can drive + * plugin options imperatively — most notably setting + * `FindReplacePlugin`'s `search` option for citation-jump highlights. */ +export type PlateEditorInstance = ReturnType; + export interface PlateEditorProps { /** Markdown string to load as initial content */ markdown?: string; @@ -62,6 +68,15 @@ export interface PlateEditorProps { * without modifying the core editor component. */ extraPlugins?: AnyPluginConfig[]; + /** + * Called whenever the live editor instance (re)mounts, with `null` on + * unmount. Used by callers that need to drive plugin options imperatively + * — e.g. `EditorPanelContent` setting `FindReplacePlugin`'s `search` + * option for citation-jump highlights. The callback is invoked exactly + * once per editor lifetime (the parent's `key` prop forces a fresh + * editor when needed, e.g. on edit-mode toggle). + */ + onEditorReady?: (editor: PlateEditorInstance | null) => void; } function PlateEditorContent({ @@ -100,6 +115,7 @@ export function PlateEditor({ defaultEditing = false, preset = "full", extraPlugins = [], + onEditorReady, }: PlateEditorProps) { const lastMarkdownRef = useRef(markdown); const lastHtmlRef = useRef(html); @@ -156,6 +172,21 @@ export function PlateEditor({ : undefined, }); + // Expose the live editor instance to imperative callers (e.g. citation + // jump highlights). We deliberately don't depend on `onEditorReady` + // itself in the cleanup closure — callers commonly pass an arrow that + // closes over a stable ref setter, but if they pass a freshly-bound + // callback per render, the `onEditorReady?.(editor)` re-fires which is + // idempotent for ref-style setters. + const onEditorReadyRef = useRef(onEditorReady); + useEffect(() => { + onEditorReadyRef.current = onEditorReady; + }, [onEditorReady]); + useEffect(() => { + onEditorReadyRef.current?.(editor); + return () => onEditorReadyRef.current?.(null); + }, [editor]); + // Update editor content when html prop changes externally useEffect(() => { if (html !== undefined && html !== lastHtmlRef.current) { diff --git a/surfsense_web/components/editor/presets.ts b/surfsense_web/components/editor/presets.ts index c207b5e56..49f53ecf1 100644 --- a/surfsense_web/components/editor/presets.ts +++ b/surfsense_web/components/editor/presets.ts @@ -1,5 +1,6 @@ "use client"; +import { FindReplacePlugin } from "@platejs/find-replace"; import type { AnyPluginConfig } from "platejs"; import { TrailingBlockPlugin } from "platejs"; @@ -17,6 +18,30 @@ import { SelectionKit } from "@/components/editor/plugins/selection-kit"; import { SlashCommandKit } from "@/components/editor/plugins/slash-command-kit"; import { TableKit } from "@/components/editor/plugins/table-kit"; import { ToggleKit } from "@/components/editor/plugins/toggle-kit"; +import { SearchHighlightLeaf } from "@/components/ui/search-highlight-node"; + +/** + * Citation-jump highlighter. Re-uses Plate's built-in `FindReplacePlugin` + * (decorate-only, no editing surface) to drive the "scroll-to-cited-text" + * UX in `EditorPanelContent`. We register it in every preset because: + * - Decorate is a no-op when `search` is empty (single getOptions() check + * per block), so cost is effectively zero for non-citation viewers. + * - Keeping it preset-agnostic means citations work whether the doc is + * opened in editable (`full`) or pure-viewer (`readonly`) modes. + * + * The parent component drives `setOption(FindReplacePlugin, 'search', ...)` + * + `editor.api.redecorate()` to trigger highlights, then queries the + * editor DOM for `.citation-highlight-leaf` to scroll the first match + * into view. (We can't use a `data-*` attribute here — Plate's + * `PlateLeaf` runs props through `useNodeAttributes`, which only forwards + * `attributes`, `className`, `ref`, `style`; arbitrary `data-*` props are + * silently dropped.) See `components/ui/search-highlight-node.tsx` for + * the leaf component and `CITATION_HIGHLIGHT_CLASS` constant. + */ +const CitationFindReplacePlugin = FindReplacePlugin.configure({ + options: { search: "" }, + render: { node: SearchHighlightLeaf }, +}); /** * Full preset – every plugin kit enabled. @@ -38,6 +63,7 @@ export const fullPreset: AnyPluginConfig[] = [ ...AutoformatKit, ...DndKit, TrailingBlockPlugin, + CitationFindReplacePlugin, ]; /** @@ -52,6 +78,7 @@ export const minimalPreset: AnyPluginConfig[] = [ ...LinkKit, ...AutoformatKit, TrailingBlockPlugin, + CitationFindReplacePlugin, ]; /** @@ -68,6 +95,7 @@ export const readonlyPreset: AnyPluginConfig[] = [ ...CalloutKit, ...ToggleKit, ...MathKit, + CitationFindReplacePlugin, ]; /** All available preset names */ diff --git a/surfsense_web/components/new-chat/source-detail-panel.tsx b/surfsense_web/components/new-chat/source-detail-panel.tsx deleted file mode 100644 index aded206c7..000000000 --- a/surfsense_web/components/new-chat/source-detail-panel.tsx +++ /dev/null @@ -1,719 +0,0 @@ -"use client"; - -import { useQuery } from "@tanstack/react-query"; -import { - BookOpen, - ChevronDown, - ChevronUp, - ExternalLink, - FileQuestionMark, - FileText, - Hash, - Loader2, - Sparkles, - X, -} from "lucide-react"; -import { AnimatePresence, motion, useReducedMotion } from "motion/react"; -import { useTranslations } from "next-intl"; -import type React from "react"; -import { forwardRef, memo, type ReactNode, useCallback, useEffect, useRef, useState } from "react"; -import { createPortal } from "react-dom"; -import { MarkdownViewer } from "@/components/markdown-viewer"; -import { Badge } from "@/components/ui/badge"; -import { Button } from "@/components/ui/button"; -import { ScrollArea } from "@/components/ui/scroll-area"; -import { Spinner } from "@/components/ui/spinner"; -import type { - GetDocumentByChunkResponse, - GetSurfsenseDocsByChunkResponse, -} from "@/contracts/types/document.types"; -import { documentsApiService } from "@/lib/apis/documents-api.service"; -import { cacheKeys } from "@/lib/query-client/cache-keys"; -import { cn } from "@/lib/utils"; - -type DocumentData = GetDocumentByChunkResponse | GetSurfsenseDocsByChunkResponse; - -interface SourceDetailPanelProps { - open: boolean; - onOpenChange: (open: boolean) => void; - chunkId: number; - sourceType: string; - title: string; - description?: string; - url?: string; - children?: ReactNode; - isDocsChunk?: boolean; -} - -const formatDocumentType = (type: string) => { - if (!type) return ""; - return type - .split("_") - .map((word) => word.charAt(0) + word.slice(1).toLowerCase()) - .join(" "); -}; - -// Chunk card component -// For large documents (>30 chunks), we disable animation to prevent layout shifts -// which break auto-scroll functionality -interface ChunkCardProps { - chunk: { id: number; content: string }; - localIndex: number; - chunkNumber: number; - totalChunks: number; - isCited: boolean; - isActive: boolean; - disableLayoutAnimation?: boolean; -} - -const ChunkCard = memo( - forwardRef( - ({ chunk, localIndex, chunkNumber, totalChunks, isCited }, ref) => { - return ( -
- {isCited &&
} - -
-
-
- {chunkNumber} -
- - Chunk {chunkNumber} of {totalChunks} - -
- {isCited && ( - - - Cited Source - - )} -
- -
- -
-
- ); - } - ) -); -ChunkCard.displayName = "ChunkCard"; - -export function SourceDetailPanel({ - open, - onOpenChange, - chunkId, - sourceType, - title, - description, - url, - children, - isDocsChunk = false, -}: SourceDetailPanelProps) { - const t = useTranslations("dashboard"); - const scrollAreaRef = useRef(null); - const hasScrolledRef = useRef(false); // Use ref to avoid stale closures - const scrollTimersRef = useRef[]>([]); - const [activeChunkIndex, setActiveChunkIndex] = useState(null); - const [mounted, setMounted] = useState(false); - const shouldReduceMotion = useReducedMotion(); - - useEffect(() => { - setMounted(true); - }, []); - - const { - data: documentData, - isLoading: isDocumentByChunkFetching, - error: documentByChunkFetchingError, - } = useQuery({ - queryKey: isDocsChunk - ? cacheKeys.documents.byChunk(`doc-${chunkId}`) - : cacheKeys.documents.byChunk(chunkId.toString()), - queryFn: async () => { - if (isDocsChunk) { - return documentsApiService.getSurfsenseDocByChunk(chunkId); - } - return documentsApiService.getDocumentByChunk({ chunk_id: chunkId, chunk_window: 5 }); - }, - enabled: !!chunkId && open, - staleTime: 5 * 60 * 1000, - }); - - const totalChunks = - documentData && "total_chunks" in documentData - ? (documentData.total_chunks ?? documentData.chunks.length) - : (documentData?.chunks?.length ?? 0); - const [beforeChunks, setBeforeChunks] = useState< - Array<{ id: number; content: string; created_at: string }> - >([]); - const [afterChunks, setAfterChunks] = useState< - Array<{ id: number; content: string; created_at: string }> - >([]); - const [loadingBefore, setLoadingBefore] = useState(false); - const [loadingAfter, setLoadingAfter] = useState(false); - - useEffect(() => { - setBeforeChunks([]); - setAfterChunks([]); - }, [chunkId, open]); - - const chunkStartIndex = - documentData && "chunk_start_index" in documentData ? (documentData.chunk_start_index ?? 0) : 0; - const initialChunks = documentData?.chunks ?? []; - const allChunks = [...beforeChunks, ...initialChunks, ...afterChunks]; - const absoluteStart = chunkStartIndex - beforeChunks.length; - const absoluteEnd = chunkStartIndex + initialChunks.length + afterChunks.length; - const canLoadBefore = absoluteStart > 0; - const canLoadAfter = absoluteEnd < totalChunks; - - const EXPAND_SIZE = 10; - - const loadBefore = useCallback(async () => { - if (!documentData || !("search_space_id" in documentData) || !canLoadBefore) return; - setLoadingBefore(true); - try { - const count = Math.min(EXPAND_SIZE, absoluteStart); - const result = await documentsApiService.getDocumentChunks({ - document_id: documentData.id, - page: 0, - page_size: count, - start_offset: absoluteStart - count, - }); - const existingIds = new Set(allChunks.map((c) => c.id)); - const newChunks = result.items - .filter((c) => !existingIds.has(c.id)) - .map((c) => ({ id: c.id, content: c.content, created_at: c.created_at })); - setBeforeChunks((prev) => [...newChunks, ...prev]); - } catch (err) { - console.error("Failed to load earlier chunks:", err); - } finally { - setLoadingBefore(false); - } - }, [documentData, absoluteStart, canLoadBefore, allChunks]); - - const loadAfter = useCallback(async () => { - if (!documentData || !("search_space_id" in documentData) || !canLoadAfter) return; - setLoadingAfter(true); - try { - const result = await documentsApiService.getDocumentChunks({ - document_id: documentData.id, - page: 0, - page_size: EXPAND_SIZE, - start_offset: absoluteEnd, - }); - const existingIds = new Set(allChunks.map((c) => c.id)); - const newChunks = result.items - .filter((c) => !existingIds.has(c.id)) - .map((c) => ({ id: c.id, content: c.content, created_at: c.created_at })); - setAfterChunks((prev) => [...prev, ...newChunks]); - } catch (err) { - console.error("Failed to load later chunks:", err); - } finally { - setLoadingAfter(false); - } - }, [documentData, absoluteEnd, canLoadAfter, allChunks]); - - const isDirectRenderSource = - sourceType === "TAVILY_API" || - sourceType === "LINKUP_API" || - sourceType === "SEARXNG_API" || - sourceType === "BAIDU_SEARCH_API"; - - const citedChunkIndex = allChunks.findIndex((chunk) => chunk.id === chunkId); - - // Simple scroll function that scrolls to a chunk by index - const scrollToChunkByIndex = useCallback( - (chunkIndex: number, smooth = true) => { - const scrollContainer = scrollAreaRef.current; - if (!scrollContainer) return; - - const viewport = scrollContainer.querySelector( - "[data-radix-scroll-area-viewport]" - ) as HTMLElement | null; - if (!viewport) return; - - const chunkElement = scrollContainer.querySelector( - `[data-chunk-index="${chunkIndex}"]` - ) as HTMLElement | null; - if (!chunkElement) return; - - // Get positions using getBoundingClientRect for accuracy - const viewportRect = viewport.getBoundingClientRect(); - const chunkRect = chunkElement.getBoundingClientRect(); - - // Calculate where to scroll to center the chunk - const currentScrollTop = viewport.scrollTop; - const chunkTopRelativeToViewport = chunkRect.top - viewportRect.top + currentScrollTop; - const scrollTarget = - chunkTopRelativeToViewport - viewportRect.height / 2 + chunkRect.height / 2; - - viewport.scrollTo({ - top: Math.max(0, scrollTarget), - behavior: smooth && !shouldReduceMotion ? "smooth" : "auto", - }); - - setActiveChunkIndex(chunkIndex); - }, - [shouldReduceMotion] - ); - - // Callback ref for the cited chunk - scrolls when the element mounts - const citedChunkRefCallback = useCallback( - (node: HTMLDivElement | null) => { - if (node && !hasScrolledRef.current && open) { - hasScrolledRef.current = true; // Mark immediately to prevent duplicate scrolls - - // Store the node reference for the delayed scroll - const scrollToCitedChunk = () => { - const scrollContainer = scrollAreaRef.current; - if (!scrollContainer || !node.isConnected) return false; - - const viewport = scrollContainer.querySelector( - "[data-radix-scroll-area-viewport]" - ) as HTMLElement | null; - if (!viewport) return false; - - // Get positions - const viewportRect = viewport.getBoundingClientRect(); - const chunkRect = node.getBoundingClientRect(); - - // Calculate scroll position to center the chunk - const currentScrollTop = viewport.scrollTop; - const chunkTopRelativeToViewport = chunkRect.top - viewportRect.top + currentScrollTop; - const scrollTarget = - chunkTopRelativeToViewport - viewportRect.height / 2 + chunkRect.height / 2; - - viewport.scrollTo({ - top: Math.max(0, scrollTarget), - behavior: "auto", // Instant scroll for initial positioning - }); - - return true; - }; - - // Scroll multiple times with delays to handle progressive content rendering - // Each subsequent scroll will correct for any layout shifts - const scrollAttempts = [50, 150, 300, 600, 1000]; - - scrollAttempts.forEach((delay) => { - scrollTimersRef.current.push( - setTimeout(() => { - scrollToCitedChunk(); - }, delay) - ); - }); - - // After final attempt, mark the cited chunk as active - scrollTimersRef.current.push( - setTimeout( - () => { - setActiveChunkIndex(citedChunkIndex); - }, - scrollAttempts[scrollAttempts.length - 1] + 50 - ) - ); - } - }, - [open, citedChunkIndex] - ); - - // Reset scroll state when panel closes - useEffect(() => { - if (!open) { - scrollTimersRef.current.forEach(clearTimeout); - scrollTimersRef.current = []; - hasScrolledRef.current = false; - setActiveChunkIndex(null); - } - return () => { - scrollTimersRef.current.forEach(clearTimeout); - scrollTimersRef.current = []; - }; - }, [open]); - - // Handle escape key - useEffect(() => { - const handleEscape = (e: KeyboardEvent) => { - if (e.key === "Escape" && open) { - onOpenChange(false); - } - }; - window.addEventListener("keydown", handleEscape); - return () => window.removeEventListener("keydown", handleEscape); - }, [open, onOpenChange]); - - // Prevent body scroll when open - useEffect(() => { - if (open) { - document.body.style.overflow = "hidden"; - } else { - document.body.style.overflow = ""; - } - return () => { - document.body.style.overflow = ""; - }; - }, [open]); - - const handleUrlClick = (e: React.MouseEvent, clickUrl: string) => { - e.preventDefault(); - e.stopPropagation(); - window.open(clickUrl, "_blank", "noopener,noreferrer"); - }; - - const scrollToChunk = useCallback( - (index: number) => { - scrollToChunkByIndex(index, true); - }, - [scrollToChunkByIndex] - ); - - const panelContent = ( - - {open && ( - <> - {/* Backdrop */} - onOpenChange(false)} - /> - - {/* Panel */} - - {/* Header */} - -
-

- {documentData?.title || title || "Source Document"} -

-

- {documentData && "document_type" in documentData - ? formatDocumentType(documentData.document_type) - : sourceType && formatDocumentType(sourceType)} - {totalChunks > 0 && ( - - • {totalChunks} chunk{totalChunks !== 1 ? "s" : ""} - {allChunks.length < totalChunks && ` (showing ${allChunks.length})`} - - )} -

-
-
- {url && ( - - )} - -
- - - {/* Loading State */} - {!isDirectRenderSource && isDocumentByChunkFetching && ( -
- - -

- {t("loading_document")} -

- -
- )} - - {/* Error State */} - {!isDirectRenderSource && documentByChunkFetchingError && ( -
- -
- -
-
-

Document unavailable

-

- {documentByChunkFetchingError.message || - "An unexpected error occurred. Please try again."} -

-
- - -
- )} - - {/* Direct render for web search providers */} - {isDirectRenderSource && ( - -
- {url && ( - - )} - -

- - Source Information -

-
- {title || "Untitled"} -
-
- {description || "No content available"} -
- -
- - )} - - {/* API-fetched document content */} - {!isDirectRenderSource && documentData && ( -
- {/* Chunk Navigation Sidebar */} - {allChunks.length > 1 && ( - - -
- {allChunks.map((chunk, idx) => { - const absNum = absoluteStart + idx + 1; - const isCited = chunk.id === chunkId; - const isActive = activeChunkIndex === idx; - return ( - scrollToChunk(idx)} - initial={{ opacity: 0, scale: 0.8 }} - animate={{ opacity: 1, scale: 1 }} - transition={{ delay: Math.min(idx * 0.02, 0.2) }} - className={cn( - "relative w-11 h-9 mx-auto rounded-lg text-xs font-semibold transition-all duration-200 flex items-center justify-center", - isCited - ? "bg-primary text-primary-foreground shadow-md" - : isActive - ? "bg-muted text-foreground" - : "bg-muted/50 text-muted-foreground hover:bg-muted hover:text-foreground" - )} - title={isCited ? `Chunk ${absNum} (Cited)` : `Chunk ${absNum}`} - > - {absNum} - {isCited && ( - - - - )} - - ); - })} -
- - - )} - - {/* Main Content */} - -
- {/* Document Metadata */} - {"document_metadata" in documentData && - documentData.document_metadata && - Object.keys(documentData.document_metadata).length > 0 && ( - -

- - Document Information -

-
- {Object.entries(documentData.document_metadata).map(([key, value]) => ( -
-
- {key.replace(/_/g, " ")} -
-
{String(value)}
-
- ))} -
- - )} - - {/* Chunks Header */} -
-

- - Chunks {absoluteStart + 1}–{absoluteEnd} of {totalChunks} -

- {citedChunkIndex !== -1 && ( - - )} -
- - {/* Load Earlier */} - {canLoadBefore && ( -
- -
- )} - - {/* Chunks */} -
- {allChunks.map((chunk, idx) => { - const isCited = chunk.id === chunkId; - const chunkNumber = absoluteStart + idx + 1; - return ( - 30} - /> - ); - })} -
- - {/* Load Later */} - {canLoadAfter && ( -
- -
- )} -
- -
- )} - - - )} - - ); - - if (!mounted) return <>{children}; - - return ( - <> - {children} - {createPortal(panelContent, globalThis.document.body)} - - ); -} diff --git a/surfsense_web/components/settings/user-settings-dialog.tsx b/surfsense_web/components/settings/user-settings-dialog.tsx index 7352a82ee..a04ce16dd 100644 --- a/surfsense_web/components/settings/user-settings-dialog.tsx +++ b/surfsense_web/components/settings/user-settings-dialog.tsx @@ -67,9 +67,6 @@ const DesktopShortcutsContent = dynamic( import( "@/app/dashboard/[search_space_id]/user-settings/components/DesktopShortcutsContent" ).then((m) => ({ default: m.DesktopShortcutsContent })), - import( - "@/app/dashboard/[search_space_id]/user-settings/components/DesktopShortcutsContent" - ).then((m) => ({ default: m.DesktopShortcutsContent })), { ssr: false } ); const MemoryContent = dynamic( diff --git a/surfsense_web/components/ui/search-highlight-node.tsx b/surfsense_web/components/ui/search-highlight-node.tsx new file mode 100644 index 000000000..e3f316cce --- /dev/null +++ b/surfsense_web/components/ui/search-highlight-node.tsx @@ -0,0 +1,45 @@ +"use client"; + +import type { PlateLeafProps } from "platejs/react"; +import { PlateLeaf } from "platejs/react"; + +/** + * Stable class name used to identify Plate-rendered citation highlight + * leaves in the DOM. We can't use a `data-*` attribute here — Plate's + * `PlateLeaf` runs its props through `useNodeAttributes`, which only + * forwards `attributes`, `className`, `ref`, and `style` to the rendered + * element; arbitrary `data-*` props are silently dropped (verified + * against `@platejs/core/dist/react/index.js` v52). So `className` is + * the only escape hatch that's guaranteed to survive into the DOM. + */ +export const CITATION_HIGHLIGHT_CLASS = "citation-highlight-leaf"; + +/** + * Leaf rendered for ranges decorated by `@platejs/find-replace`'s + * `FindReplacePlugin`. We re-purpose that plugin to drive the citation-jump + * highlight: when a citation is staged, the parent sets the plugin's `search` + * option to a snippet of the chunk text and Plate decorates every match with + * `searchHighlight: true`. This component renders those decorations as a + * `` tagged with `CITATION_HIGHLIGHT_CLASS` so the parent can: + * 1. Query the first match in DOM order to scroll it into view. + * 2. Detect the active-highlight state without a separate React ref. + * + * The highlight is **persistent** — it does not auto-fade. The parent in + * `EditorPanelContent` clears it by setting the plugin's `search` option + * back to "" when one of: (a) the user clicks anywhere inside the editor, + * (b) the panel switches to a different document, (c) the user toggles + * into edit mode, (d) another citation jump is staged, (e) the panel + * unmounts. We use a brief entrance pulse (`citation-flash-in`, see + * `globals.css`) purely to draw the eye after `scrollIntoView` lands. + */ +export function SearchHighlightLeaf(props: PlateLeafProps) { + return ( + + {props.children} + + ); +} diff --git a/surfsense_web/lib/citation-search.ts b/surfsense_web/lib/citation-search.ts new file mode 100644 index 000000000..f80f13076 --- /dev/null +++ b/surfsense_web/lib/citation-search.ts @@ -0,0 +1,125 @@ +/** + * Snippet generation for the citation-jump highlight, driven by Plate's + * `FindReplacePlugin`. The plugin runs `decorate` per-block and only matches + * within blocks whose children are all `Text` nodes (so it crosses inline + * marks like bold/italic but **not** block boundaries, and a block that + * contains even one inline element such as a link is silently skipped). + * That means a full chunk that spans heading + paragraph won't match as a + * single string — we have to pick a shorter snippet that fits inside one + * rendered block. + * + * `buildCitationSearchCandidates` returns search strings ordered from + * "most-specific anchor" to "broadest fallback": + * 1. First sentence of the chunk (capped at `FIRST_SENTENCE_MAX`). + * 2. First `FIRST_PHRASE_WORDS` words. + * 3. Each non-trivial line of the chunk, in source order — gives us a + * separate attempt for each rendered block, so a heading line with + * an inline link doesn't doom the whole jump. + * 4. Full chunk (only if it's already short enough to plausibly fit + * inside one block). + * + * The caller tries each candidate in turn — set the plugin's `search` + * option, `editor.api.redecorate()`, then check the editor DOM for a + * `.citation-highlight-leaf` element. First candidate that produces one + * wins; subsequent candidates are skipped. + */ + +const FIRST_SENTENCE_MAX = 120; +const FIRST_PHRASE_WORDS = 8; +const MIN_SNIPPET_LENGTH = 6; +const FULL_CHUNK_MAX = FIRST_SENTENCE_MAX * 2; +const MAX_LINE_CANDIDATES = 6; +const LINE_CANDIDATE_MAX = FIRST_SENTENCE_MAX; + +function normalizeWhitespace(input: string): string { + return input.replace(/\s+/g, " ").trim(); +} + +/** + * Strip the markdown syntax that won't survive into the rendered editor's + * plain text, so the chunk text (which comes back from the indexer as raw + * source markdown) can be matched against the literal text values stored + * in Plate's Slate tree. + * + * Order matters: handle multi-char and "container" syntax before single- + * char emphasis, otherwise `**text**` collapses to `*text*` first. + * + * Heuristic only — we don't aim to be a full markdown parser, just to + * remove the common markers (`**bold**`, `[text](url)`, `# headings`, + * `- list`, etc.) that show up in connector-doc chunks and would break + * literal substring search. + */ +export function stripMarkdownForMatch(input: string): string { + let s = input; + s = s.replace(/```[a-z0-9_+-]*\n?([\s\S]*?)```/gi, (_, body: string) => body); + s = s.replace(//g, " "); + s = s.replace(/!\[([^\]]*)\]\([^)]*\)/g, "$1"); + s = s.replace(/!\[([^\]]*)\]\[[^\]]*\]/g, "$1"); + s = s.replace(/\[([^\]]+)\]\([^)]*\)/g, "$1"); + s = s.replace(/\[([^\]]+)\]\[[^\]]*\]/g, "$1"); + s = s.replace(/<((?:https?|mailto):[^>\s]+)>/g, "$1"); + s = s.replace(/`+([^`\n]+?)`+/g, "$1"); + s = s.replace(/(\*\*|__)([\s\S]+?)\1/g, "$2"); + s = s.replace(/(?+[ \t]?/gm, ""); + s = s.replace(/^[ \t]*[-*+][ \t]+/gm, ""); + s = s.replace(/^[ \t]*\d+\.[ \t]+/gm, ""); + s = s.replace(/^[ \t]{0,3}(?:[-*_])(?:[ \t]*[-*_]){2,}[ \t]*$/gm, ""); + s = s.replace(/^[ \t]*\|?(?:[ \t]*:?-+:?[ \t]*\|)+[ \t]*:?-+:?[ \t]*\|?[ \t]*$/gm, ""); + s = s.replace(/\\([\\`*_{}[\]()#+\-.!~>])/g, "$1"); + return s; +} + +export function buildCitationSearchCandidates(rawText: string): string[] { + if (!rawText) return []; + const stripped = stripMarkdownForMatch(rawText); + const normalized = normalizeWhitespace(stripped); + if (normalized.length < MIN_SNIPPET_LENGTH) return []; + + const out: string[] = []; + const seen = new Set(); + const push = (s: string) => { + const t = normalizeWhitespace(s); + if (t.length >= MIN_SNIPPET_LENGTH && !seen.has(t)) { + out.push(t); + seen.add(t); + } + }; + + const sentenceMatch = normalized.match(/^[^.!?]+[.!?]/); + if (sentenceMatch) { + const sentence = sentenceMatch[0]; + push(sentence.length > FIRST_SENTENCE_MAX ? sentence.slice(0, FIRST_SENTENCE_MAX) : sentence); + } else if (normalized.length > FIRST_SENTENCE_MAX) { + push(normalized.slice(0, FIRST_SENTENCE_MAX)); + } + + const words = normalized.split(" ").filter(Boolean); + if (words.length > FIRST_PHRASE_WORDS) { + push(words.slice(0, FIRST_PHRASE_WORDS).join(" ")); + } + + // Per-line candidates: each chunk line is roughly one block in the + // rendered editor. Trying them in order gives us a separate decorate + // attempt for each block, which matters when the first line is a + // heading containing a link (Plate's `FindReplacePlugin` will skip + // any block whose children aren't all text nodes). + const rawLines = stripped.split(/\r?\n/); + let lineCount = 0; + for (const line of rawLines) { + if (lineCount >= MAX_LINE_CANDIDATES) break; + const trimmed = normalizeWhitespace(line); + if (trimmed.length < MIN_SNIPPET_LENGTH) continue; + push(trimmed.length > LINE_CANDIDATE_MAX ? trimmed.slice(0, LINE_CANDIDATE_MAX) : trimmed); + lineCount++; + } + + if (normalized.length <= FULL_CHUNK_MAX) { + push(normalized); + } + + return out; +} diff --git a/surfsense_web/package.json b/surfsense_web/package.json index 41175daeb..665490e4f 100644 --- a/surfsense_web/package.json +++ b/surfsense_web/package.json @@ -36,6 +36,7 @@ "@platejs/code-block": "^52.0.11", "@platejs/combobox": "^52.0.15", "@platejs/dnd": "^52.0.11", + "@platejs/find-replace": "^52.3.10", "@platejs/floating": "^52.0.11", "@platejs/indent": "^52.0.11", "@platejs/link": "^52.0.11", diff --git a/surfsense_web/pnpm-lock.yaml b/surfsense_web/pnpm-lock.yaml index b1730e842..a1a7bea12 100644 --- a/surfsense_web/pnpm-lock.yaml +++ b/surfsense_web/pnpm-lock.yaml @@ -53,6 +53,9 @@ importers: '@platejs/dnd': specifier: ^52.0.11 version: 52.0.11(platejs@52.0.17(@types/react@19.2.14)(immer@10.2.0)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(scheduler@0.27.0)(use-sync-external-store@1.6.0(react@19.2.4)))(react-dnd-html5-backend@16.0.1)(react-dnd@16.0.1(@types/node@20.19.33)(@types/react@19.2.14)(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@platejs/find-replace': + specifier: ^52.3.10 + version: 52.3.10(platejs@52.0.17(@types/react@19.2.14)(immer@10.2.0)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(scheduler@0.27.0)(use-sync-external-store@1.6.0(react@19.2.4)))(react-dom@19.2.4(react@19.2.4))(react@19.2.4) '@platejs/floating': specifier: ^52.0.11 version: 52.0.11(platejs@52.0.17(@types/react@19.2.14)(immer@10.2.0)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(scheduler@0.27.0)(use-sync-external-store@1.6.0(react@19.2.4)))(react-dom@19.2.4(react@19.2.4))(react@19.2.4) @@ -2827,6 +2830,13 @@ packages: react-dnd-html5-backend: '>=14.0.0' react-dom: '>=18.0.0' + '@platejs/find-replace@52.3.10': + resolution: {integrity: sha512-V/MOMMUYxHfEn/skd2+YO213xSATFDVsl8FzVzVRV/XaxwwVefH2EPD1lAVIvmYjennTVTTsHHtEI9K9iOsEaA==} + peerDependencies: + platejs: '>=52.0.11' + react: '>=18.0.0' + react-dom: '>=18.0.0' + '@platejs/floating@52.0.11': resolution: {integrity: sha512-ApNpw4KWml+kuK+XTTpji+f/7GxTR4nRzlnfJMvGBrJpLPQ4elS5MABm3oUi81DZn+aub5HvsyH7UqCw7F76IA==} peerDependencies: @@ -11105,6 +11115,13 @@ snapshots: react-dnd-html5-backend: 16.0.1 react-dom: 19.2.4(react@19.2.4) + '@platejs/find-replace@52.3.10(platejs@52.0.17(@types/react@19.2.14)(immer@10.2.0)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(scheduler@0.27.0)(use-sync-external-store@1.6.0(react@19.2.4)))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + platejs: 52.0.17(@types/react@19.2.14)(immer@10.2.0)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(scheduler@0.27.0)(use-sync-external-store@1.6.0(react@19.2.4)) + react: 19.2.4 + react-compiler-runtime: 1.0.0(react@19.2.4) + react-dom: 19.2.4(react@19.2.4) + '@platejs/floating@52.0.11(platejs@52.0.17(@types/react@19.2.14)(immer@10.2.0)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(scheduler@0.27.0)(use-sync-external-store@1.6.0(react@19.2.4)))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': dependencies: '@floating-ui/core': 1.7.4 From ca9bbee06dbd2e9e50be27f54a3967e20dfc0e7d Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Tue, 28 Apr 2026 21:37:51 -0700 Subject: [PATCH 6/8] chore: linting --- .../versions/130_add_agent_action_log.py | 4 +- .../133_drop_documents_content_hash_unique.py | 4 +- .../app/agents/new_chat/chat_deepagent.py | 19 ++++--- .../app/agents/new_chat/feature_flags.py | 20 +++++-- .../agents/new_chat/middleware/action_log.py | 4 +- .../agents/new_chat/middleware/compaction.py | 4 +- .../new_chat/middleware/context_editing.py | 7 ++- .../agents/new_chat/middleware/doom_loop.py | 21 ++++--- .../new_chat/middleware/knowledge_search.py | 11 ++-- .../new_chat/middleware/noop_injection.py | 12 ++-- .../agents/new_chat/middleware/otel_span.py | 14 ++--- .../agents/new_chat/middleware/permission.py | 41 ++++++++------ .../agents/new_chat/middleware/retry_after.py | 12 +++- .../new_chat/middleware/skills_backends.py | 17 ++++-- .../new_chat/middleware/tool_call_repair.py | 10 ++-- .../new_chat/plugins/year_substituter.py | 29 +++++----- .../app/agents/new_chat/prompts/composer.py | 8 +-- .../app/agents/new_chat/subagents/config.py | 4 +- .../app/agents/new_chat/tools/registry.py | 2 + surfsense_backend/app/observability/otel.py | 9 +-- .../app/routes/agent_flags_route.py | 2 +- .../app/routes/agent_permissions_route.py | 8 +-- .../app/routes/agent_revert_route.py | 6 +- .../app/routes/new_chat_routes.py | 4 +- .../app/services/revert_service.py | 4 +- .../app/utils/user_message_multimodal.py | 4 +- .../agents/new_chat/prompts/test_composer.py | 17 +++--- .../unit/agents/new_chat/test_action_log.py | 56 ++++++++++--------- .../unit/agents/new_chat/test_compaction.py | 20 +++++-- .../agents/new_chat/test_context_editing.py | 3 +- .../agents/new_chat/test_dedup_tool_calls.py | 18 +++++- .../test_default_permissions_layering.py | 8 +-- .../unit/agents/new_chat/test_doom_loop.py | 15 ++--- .../agents/new_chat/test_noop_injection.py | 8 ++- .../new_chat/test_permission_middleware.py | 4 +- .../agents/new_chat/test_plugin_loader.py | 12 ++-- .../unit/agents/new_chat/test_retry_after.py | 10 ++-- .../new_chat/test_specialized_subagents.py | 31 +++++----- .../agents/new_chat/test_tool_call_repair.py | 54 ++++++++++++------ .../test_kb_persistence_filesystem_parity.py | 2 +- .../unit/services/test_revert_service.py | 20 ++----- 41 files changed, 314 insertions(+), 244 deletions(-) diff --git a/surfsense_backend/alembic/versions/130_add_agent_action_log.py b/surfsense_backend/alembic/versions/130_add_agent_action_log.py index 5793988cb..2f06b8ddd 100644 --- a/surfsense_backend/alembic/versions/130_add_agent_action_log.py +++ b/surfsense_backend/alembic/versions/130_add_agent_action_log.py @@ -88,7 +88,5 @@ def upgrade() -> None: def downgrade() -> None: - op.drop_index( - "ix_agent_action_log_thread_created", table_name="agent_action_log" - ) + op.drop_index("ix_agent_action_log_thread_created", table_name="agent_action_log") op.drop_table("agent_action_log") diff --git a/surfsense_backend/alembic/versions/133_drop_documents_content_hash_unique.py b/surfsense_backend/alembic/versions/133_drop_documents_content_hash_unique.py index 88c3e203f..eec53ecb6 100644 --- a/surfsense_backend/alembic/versions/133_drop_documents_content_hash_unique.py +++ b/surfsense_backend/alembic/versions/133_drop_documents_content_hash_unique.py @@ -51,9 +51,7 @@ def upgrade() -> None: # implicit-unique-index variant SQLAlchemy may emit need draining. constraints = _existing_constraint_names(bind, "documents") if "uq_documents_content_hash" in constraints: - op.drop_constraint( - "uq_documents_content_hash", "documents", type_="unique" - ) + op.drop_constraint("uq_documents_content_hash", "documents", type_="unique") indexes = _existing_index_names(bind, "documents") # Some Postgres versions surface the unique constraint via a unique diff --git a/surfsense_backend/app/agents/new_chat/chat_deepagent.py b/surfsense_backend/app/agents/new_chat/chat_deepagent.py index 672570696..3ca44dd4f 100644 --- a/surfsense_backend/app/agents/new_chat/chat_deepagent.py +++ b/surfsense_backend/app/agents/new_chat/chat_deepagent.py @@ -416,10 +416,10 @@ async def create_surfsense_deep_agent( # cheap to build. ``SubAgentMiddleware.__init__`` calls ``create_agent`` # synchronously to compile the general-purpose subagent's full state graph # (every tool + every middleware → pydantic schemas + langgraph compile). - # On gpt-5.x agents that's roughly 1.5–2s of pure CPU work. If we run it + # On gpt-5.x agents that's roughly 1.5-2s of pure CPU work. If we run it # directly here it blocks the asyncio event loop for the whole streaming # task (and any other coroutine sharing this loop), which is why - # "agent creation" wall-clock time used to stretch to ~3–4s. Move the + # "agent creation" wall-clock time used to stretch to ~3-4s. Move the # entire middleware build + main-graph compile into a single # ``asyncio.to_thread`` so the heavy CPU work runs off-loop and the # event loop stays responsive. @@ -587,10 +587,7 @@ def _build_compiled_agent_blocking( # by name. Off by default until the flag flips so existing deployments # don't see new agent types in the task tool description. specialized_subagents: list[SubAgent] = [] - if ( - flags.enable_specialized_subagents - and not flags.disable_new_agent_stack - ): + if flags.enable_specialized_subagents and not flags.disable_new_agent_stack: try: # Specialized subagents share the parent's filesystem + # todo view so their system prompts (which promise @@ -696,7 +693,9 @@ def _build_compiled_agent_blocking( else None ) tool_call_limit_mw = ( - ToolCallLimitMiddleware(thread_limit=300, run_limit=80, exit_behavior="continue") + ToolCallLimitMiddleware( + thread_limit=300, run_limit=80, exit_behavior="continue" + ) if flags.enable_tool_call_limit and not flags.disable_new_agent_stack else None ) @@ -879,7 +878,11 @@ def _build_compiled_agent_blocking( max_tools=12, always_include=[ name - for name in ("update_memory", "get_connected_accounts", "scrape_webpage") + for name in ( + "update_memory", + "get_connected_accounts", + "scrape_webpage", + ) if name in {t.name for t in tools} ], ) diff --git a/surfsense_backend/app/agents/new_chat/feature_flags.py b/surfsense_backend/app/agents/new_chat/feature_flags.py index ce0a3b3fa..89c4fb14f 100644 --- a/surfsense_backend/app/agents/new_chat/feature_flags.py +++ b/surfsense_backend/app/agents/new_chat/feature_flags.py @@ -65,7 +65,9 @@ class AgentFeatureFlags: enable_model_call_limit: bool = False enable_tool_call_limit: bool = False enable_tool_call_repair: bool = False - enable_doom_loop: bool = False # Default OFF until UI handles permission='doom_loop' + enable_doom_loop: bool = ( + False # Default OFF until UI handles permission='doom_loop' + ) # Tier 2 — Safety enable_permission: bool = False # Default OFF for first deploy @@ -79,7 +81,9 @@ class AgentFeatureFlags: # Tier 5 — Snapshot / revert enable_action_log: bool = False - enable_revert_route: bool = False # Backend ships before UI; route returns 503 until this flips + enable_revert_route: bool = ( + False # Backend ships before UI; route returns 503 until this flips + ) # Tier 6 — Plugins enable_plugin_loader: bool = False @@ -109,14 +113,20 @@ class AgentFeatureFlags: enable_compaction_v2=_env_bool("SURFSENSE_ENABLE_COMPACTION_V2", False), enable_retry_after=_env_bool("SURFSENSE_ENABLE_RETRY_AFTER", False), enable_model_fallback=_env_bool("SURFSENSE_ENABLE_MODEL_FALLBACK", False), - enable_model_call_limit=_env_bool("SURFSENSE_ENABLE_MODEL_CALL_LIMIT", False), + enable_model_call_limit=_env_bool( + "SURFSENSE_ENABLE_MODEL_CALL_LIMIT", False + ), enable_tool_call_limit=_env_bool("SURFSENSE_ENABLE_TOOL_CALL_LIMIT", False), - enable_tool_call_repair=_env_bool("SURFSENSE_ENABLE_TOOL_CALL_REPAIR", False), + enable_tool_call_repair=_env_bool( + "SURFSENSE_ENABLE_TOOL_CALL_REPAIR", False + ), enable_doom_loop=_env_bool("SURFSENSE_ENABLE_DOOM_LOOP", False), # Tier 2 enable_permission=_env_bool("SURFSENSE_ENABLE_PERMISSION", False), enable_busy_mutex=_env_bool("SURFSENSE_ENABLE_BUSY_MUTEX", False), - enable_llm_tool_selector=_env_bool("SURFSENSE_ENABLE_LLM_TOOL_SELECTOR", False), + enable_llm_tool_selector=_env_bool( + "SURFSENSE_ENABLE_LLM_TOOL_SELECTOR", False + ), # Tier 4 enable_skills=_env_bool("SURFSENSE_ENABLE_SKILLS", False), enable_specialized_subagents=_env_bool( diff --git a/surfsense_backend/app/agents/new_chat/middleware/action_log.py b/surfsense_backend/app/agents/new_chat/middleware/action_log.py index cf0b57fd4..3675064e8 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/action_log.py +++ b/surfsense_backend/app/agents/new_chat/middleware/action_log.py @@ -101,9 +101,7 @@ class ActionLogMiddleware(AgentMiddleware): async def awrap_tool_call( self, request: ToolCallRequest, - handler: Callable[ - [ToolCallRequest], Awaitable[ToolMessage | Command[Any]] - ], + handler: Callable[[ToolCallRequest], Awaitable[ToolMessage | Command[Any]]], ) -> ToolMessage | Command[Any]: if not self._enabled(): return await handler(request) diff --git a/surfsense_backend/app/agents/new_chat/middleware/compaction.py b/surfsense_backend/app/agents/new_chat/middleware/compaction.py index 8b02089c9..b0a1a7ec5 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/compaction.py +++ b/surfsense_backend/app/agents/new_chat/middleware/compaction.py @@ -177,8 +177,8 @@ class SurfSenseCompactionMiddleware(SummarizationMiddleware): messages_in=len(conversation_messages), extra={"compaction.cutoff_index": int(cutoff_index)}, ): - messages_to_summarize, preserved_messages = ( - super()._partition_messages(conversation_messages, cutoff_index) + messages_to_summarize, preserved_messages = super()._partition_messages( + conversation_messages, cutoff_index ) protected: list[AnyMessage] = [] diff --git a/surfsense_backend/app/agents/new_chat/middleware/context_editing.py b/surfsense_backend/app/agents/new_chat/middleware/context_editing.py index 93ceab8ee..360e3e28f 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/context_editing.py +++ b/surfsense_backend/app/agents/new_chat/middleware/context_editing.py @@ -58,8 +58,7 @@ DEFAULT_SPILL_PREFIX = "/tool_outputs" def _build_spill_placeholder(spill_path: str) -> str: """Build the user-facing placeholder text shown to the model.""" return ( - f"[cleared — full output at {spill_path}; " - f"ask the explore subagent to read it]" + f"[cleared — full output at {spill_path}; ask the explore subagent to read it]" ) @@ -131,7 +130,9 @@ class SpillToBackendEdit(ContextEdit): return candidates = [ - (idx, msg) for idx, msg in enumerate(messages) if isinstance(msg, ToolMessage) + (idx, msg) + for idx, msg in enumerate(messages) + if isinstance(msg, ToolMessage) ] if self.keep >= len(candidates): return diff --git a/surfsense_backend/app/agents/new_chat/middleware/doom_loop.py b/surfsense_backend/app/agents/new_chat/middleware/doom_loop.py index 49ac7dfa8..1dde87752 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/doom_loop.py +++ b/surfsense_backend/app/agents/new_chat/middleware/doom_loop.py @@ -137,16 +137,21 @@ class DoomLoopMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, Respon triggered_call: dict[str, Any] | None = None for call in message.tool_calls: - name = call.get("name") if isinstance(call, dict) else getattr(call, "name", None) - args = call.get("args") if isinstance(call, dict) else getattr(call, "args", {}) + name = ( + call.get("name") + if isinstance(call, dict) + else getattr(call, "name", None) + ) + args = ( + call.get("args") + if isinstance(call, dict) + else getattr(call, "args", {}) + ) if not isinstance(name, str): continue sig = _signature(name, args) window.append(sig) - if ( - len(window) >= self._threshold - and len(set(window)) == 1 - ): + if len(window) >= self._threshold and len(set(window)) == 1: triggered_call = {"name": name, "params": args or {}} break @@ -209,7 +214,9 @@ class DoomLoopMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, Respon # tool call proceeds. The frontend's exact reply names may differ — # we tolerate any shape that contains a string with "reject"/"cancel". if isinstance(decision, dict): - kind = str(decision.get("decision_type") or decision.get("type") or "").lower() + kind = str( + decision.get("decision_type") or decision.get("type") or "" + ).lower() if "reject" in kind or "cancel" in kind: return {"jump_to": "end"} return None diff --git a/surfsense_backend/app/agents/new_chat/middleware/knowledge_search.py b/surfsense_backend/app/agents/new_chat/middleware/knowledge_search.py index f39870df6..08ca8e18b 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/knowledge_search.py +++ b/surfsense_backend/app/agents/new_chat/middleware/knowledge_search.py @@ -552,7 +552,7 @@ def _render_priority_message(priority: list[dict[str, Any]]) -> SystemMessage: for entry in priority: score = entry.get("score") mentioned = entry.get("mentioned") - score_str = f"{score:.3f}" if isinstance(score, (int, float)) else "n/a" + score_str = f"{score:.3f}" if isinstance(score, int | float) else "n/a" mark = " [USER-MENTIONED]" if mentioned else "" lines.append(f"- {entry.get('path', '')} (score={score_str}){mark}") body = "\n".join(lines) @@ -593,7 +593,7 @@ class KnowledgePriorityMiddleware(AgentMiddleware): # type: ignore[type-arg] self.top_k = top_k self.mentioned_document_ids = mentioned_document_ids or [] # Tier 4.2: build the kb-planner private Runnable ONCE here so we - # don't pay the create_agent compile cost (50–200ms) on every turn. + # don't pay the create_agent compile cost (50-200ms) on every turn. # Disabled by default behind ``enable_kb_planner_runnable``; when off # the planner falls back to the legacy ``self.llm.ainvoke`` path. self._planner: Runnable | None = None @@ -617,10 +617,7 @@ class KnowledgePriorityMiddleware(AgentMiddleware): # type: ignore[type-arg] if self.llm is None: return None flags = get_flags() - if ( - not flags.enable_kb_planner_runnable - or flags.disable_new_agent_stack - ): + if not flags.enable_kb_planner_runnable or flags.disable_new_agent_stack: return None from app.agents.new_chat.middleware.retry_after import RetryAfterMiddleware @@ -920,7 +917,7 @@ class KnowledgePriorityMiddleware(AgentMiddleware): # type: ignore[type-arg] chunk_ids = doc.get("matched_chunk_ids") or [] if chunk_ids: matched_chunk_ids[doc_id] = [ - int(cid) for cid in chunk_ids if isinstance(cid, (int, str)) + int(cid) for cid in chunk_ids if isinstance(cid, int | str) ] return priority, matched_chunk_ids diff --git a/surfsense_backend/app/agents/new_chat/middleware/noop_injection.py b/surfsense_backend/app/agents/new_chat/middleware/noop_injection.py index f16084892..8628479c7 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/noop_injection.py +++ b/surfsense_backend/app/agents/new_chat/middleware/noop_injection.py @@ -35,9 +35,7 @@ from langchain_core.tools import tool logger = logging.getLogger(__name__) NOOP_TOOL_NAME = "_noop" -NOOP_TOOL_DESCRIPTION = ( - "Do not call this tool. It exists only for API compatibility." -) +NOOP_TOOL_DESCRIPTION = "Do not call this tool. It exists only for API compatibility." @tool(name_or_callable=NOOP_TOOL_NAME, description=NOOP_TOOL_DESCRIPTION) @@ -78,7 +76,9 @@ def _last_ai_has_tool_calls(messages: list[Any]) -> bool: return False -class NoopInjectionMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, ResponseT]): +class NoopInjectionMiddleware( + AgentMiddleware[AgentState[ResponseT], ContextT, ResponseT] +): """Inject the ``_noop`` tool only when the provider would otherwise 400. The check fires per model call, not at agent build time, because the @@ -116,7 +116,9 @@ class NoopInjectionMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, R async def awrap_model_call( # type: ignore[override] self, request: ModelRequest[ContextT], - handler: Callable[[ModelRequest[ContextT]], Awaitable[ModelResponse[ResponseT]]], + handler: Callable[ + [ModelRequest[ContextT]], Awaitable[ModelResponse[ResponseT]] + ], ) -> Any: if self._should_inject(request): logger.debug("Injecting _noop tool for provider compatibility") diff --git a/surfsense_backend/app/agents/new_chat/middleware/otel_span.py b/surfsense_backend/app/agents/new_chat/middleware/otel_span.py index 5585cf7a2..f51d2f7bb 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/otel_span.py +++ b/surfsense_backend/app/agents/new_chat/middleware/otel_span.py @@ -56,9 +56,7 @@ class OtelSpanMiddleware(AgentMiddleware): async def awrap_model_call( self, request: ModelRequest, - handler: Callable[ - [ModelRequest], Awaitable[ModelResponse | AIMessage | Any] - ], + handler: Callable[[ModelRequest], Awaitable[ModelResponse | AIMessage | Any]], ) -> ModelResponse | AIMessage | Any: if not ot.is_enabled(): return await handler(request) @@ -81,9 +79,7 @@ class OtelSpanMiddleware(AgentMiddleware): async def awrap_tool_call( self, request: ToolCallRequest, - handler: Callable[ - [ToolCallRequest], Awaitable[ToolMessage | Command[Any]] - ], + handler: Callable[[ToolCallRequest], Awaitable[ToolMessage | Command[Any]]], ) -> ToolMessage | Command[Any]: if not ot.is_enabled(): return await handler(request) @@ -187,7 +183,11 @@ def _annotate_model_response(span: Any, result: Any) -> None: def _annotate_tool_result(span: Any, result: Any) -> None: try: if isinstance(result, ToolMessage): - content = result.content if isinstance(result.content, str) else repr(result.content) + content = ( + result.content + if isinstance(result.content, str) + else repr(result.content) + ) span.set_attribute("tool.output.size", len(content)) status = getattr(result, "status", None) if isinstance(status, str): diff --git a/surfsense_backend/app/agents/new_chat/middleware/permission.py b/surfsense_backend/app/agents/new_chat/middleware/permission.py index f59e70bc0..6e1f42baf 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/permission.py +++ b/surfsense_backend/app/agents/new_chat/middleware/permission.py @@ -145,7 +145,9 @@ class PermissionMiddleware(AgentMiddleware): # type: ignore[type-arg] try: patterns = resolver(args or {}) except Exception: - logger.exception("Pattern resolver for %s raised; using bare name", tool_name) + logger.exception( + "Pattern resolver for %s raised; using bare name", tool_name + ) patterns = [tool_name] if not patterns: patterns = [tool_name] @@ -198,11 +200,14 @@ class PermissionMiddleware(AgentMiddleware): # type: ignore[type-arg] # Tier 3b: permission.asked + interrupt.raised spans (no-op when # OTel is disabled). Both fire here so dashboards can correlate # "we asked X" with "interrupt was actually delivered". - with ot.permission_asked_span( - permission=tool_name, - pattern=patterns[0] if patterns else None, - extra={"permission.patterns": list(patterns)}, - ), ot.interrupt_span(interrupt_type="permission_ask"): + with ( + ot.permission_asked_span( + permission=tool_name, + pattern=patterns[0] if patterns else None, + extra={"permission.patterns": list(patterns)}, + ), + ot.interrupt_span(interrupt_type="permission_ask"), + ): decision = interrupt(payload) if isinstance(decision, dict): return decision @@ -211,9 +216,7 @@ class PermissionMiddleware(AgentMiddleware): # type: ignore[type-arg] return {"decision_type": decision} return {"decision_type": "reject"} - def _persist_always( - self, tool_name: str, patterns: list[str] - ) -> None: + def _persist_always(self, tool_name: str, patterns: list[str]) -> None: """Promote ``always`` reply into runtime allow rules. Persistence to ``agent_permission_rules`` is done by the @@ -276,12 +279,16 @@ class PermissionMiddleware(AgentMiddleware): # type: ignore[type-arg] any_change = False for raw in last.tool_calls: - call = dict(raw) if isinstance(raw, dict) else { - "name": getattr(raw, "name", None), - "args": getattr(raw, "args", {}), - "id": getattr(raw, "id", None), - "type": "tool_call", - } + call = ( + dict(raw) + if isinstance(raw, dict) + else { + "name": getattr(raw, "name", None), + "args": getattr(raw, "args", {}), + "id": getattr(raw, "id", None), + "type": "tool_call", + } + ) name = call.get("name") or "" args = call.get("args") or {} action, patterns, rules = self._evaluate(name, args) @@ -307,7 +314,9 @@ class PermissionMiddleware(AgentMiddleware): # type: ignore[type-arg] feedback = decision.get("feedback") if isinstance(feedback, str) and feedback.strip(): raise CorrectedError(feedback, tool=name) - raise RejectedError(tool=name, pattern=patterns[0] if patterns else None) + raise RejectedError( + tool=name, pattern=patterns[0] if patterns else None + ) else: logger.warning( "Unknown permission decision %r; treating as reject", kind diff --git a/surfsense_backend/app/agents/new_chat/middleware/retry_after.py b/surfsense_backend/app/agents/new_chat/middleware/retry_after.py index 82da6a97c..394bb0371 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/retry_after.py +++ b/surfsense_backend/app/agents/new_chat/middleware/retry_after.py @@ -113,7 +113,9 @@ def _exponential_delay( jitter: bool, ) -> float: """Compute an exponential-backoff delay with optional ±25% jitter.""" - delay = initial_delay * (backoff_factor**attempt) if backoff_factor else initial_delay + delay = ( + initial_delay * (backoff_factor**attempt) if backoff_factor else initial_delay + ) delay = min(delay, max_delay) if jitter and delay > 0: delay *= 1 + random.uniform(-0.25, 0.25) @@ -201,7 +203,9 @@ class RetryAfterMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, Resp }, ) except Exception: - logger.debug("dispatch_custom_event failed; suppressed", exc_info=True) + logger.debug( + "dispatch_custom_event failed; suppressed", exc_info=True + ) if delay > 0: time.sleep(delay) # Unreachable @@ -210,7 +214,9 @@ class RetryAfterMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, Resp async def awrap_model_call( # type: ignore[override] self, request: ModelRequest[ContextT], - handler: Callable[[ModelRequest[ContextT]], Awaitable[ModelResponse[ResponseT]]], + handler: Callable[ + [ModelRequest[ContextT]], Awaitable[ModelResponse[ResponseT]] + ], ) -> ModelResponse[ResponseT] | AIMessage: for attempt in range(self.max_retries + 1): try: diff --git a/surfsense_backend/app/agents/new_chat/middleware/skills_backends.py b/surfsense_backend/app/agents/new_chat/middleware/skills_backends.py index 4c3791c87..072d73401 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/skills_backends.py +++ b/surfsense_backend/app/agents/new_chat/middleware/skills_backends.py @@ -29,6 +29,7 @@ gives a clean failure mode if anything tries. from __future__ import annotations +import contextlib import logging from collections.abc import Callable from dataclasses import replace @@ -114,8 +115,10 @@ class BuiltinSkillsBackend(BackendProtocol): infos: list[FileInfo] = [] # Build virtual paths anchored at "/" because CompositeBackend already # stripped the route prefix before calling us. - target_virtual = "/" if target == self.root else ( - "/" + str(target.relative_to(self.root)).replace("\\", "/") + target_virtual = ( + "/" + if target == self.root + else ("/" + str(target.relative_to(self.root)).replace("\\", "/")) ) for child in sorted(target.iterdir()): child_virtual = ( @@ -128,10 +131,8 @@ class BuiltinSkillsBackend(BackendProtocol): "is_dir": child.is_dir(), } if child.is_file(): - try: + with contextlib.suppress(OSError): # pragma: no cover - defensive info["size"] = child.stat().st_size - except OSError: # pragma: no cover - defensive - pass infos.append(info) return infos @@ -163,7 +164,9 @@ class BuiltinSkillsBackend(BackendProtocol): else: content = target.read_bytes() except PermissionError: - responses.append(FileDownloadResponse(path=p, error="permission_denied")) + responses.append( + FileDownloadResponse(path=p, error="permission_denied") + ) continue except OSError as exc: # pragma: no cover - defensive logger.warning("Builtin skill read failed %s: %s", target, exc) @@ -286,6 +289,7 @@ def build_skills_backend_factory( builtin = BuiltinSkillsBackend(builtin_root) if search_space_id is None: + def _factory_builtin_only(runtime: ToolRuntime) -> BackendProtocol: # Default StateBackend is intentionally inert: any path outside the # ``/skills/builtin/`` route resolves to an empty per-runtime state @@ -294,6 +298,7 @@ def build_skills_backend_factory( default=StateBackend(runtime), routes={SKILLS_BUILTIN_PREFIX: builtin}, ) + return _factory_builtin_only def _factory_with_space(runtime: ToolRuntime) -> BackendProtocol: diff --git a/surfsense_backend/app/agents/new_chat/middleware/tool_call_repair.py b/surfsense_backend/app/agents/new_chat/middleware/tool_call_repair.py index 6c3bc674d..54df0cc60 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/tool_call_repair.py +++ b/surfsense_backend/app/agents/new_chat/middleware/tool_call_repair.py @@ -51,13 +51,15 @@ def _coerce_existing_tool_call(call: Any) -> dict[str, Any]: } -class ToolCallNameRepairMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, ResponseT]): +class ToolCallNameRepairMiddleware( + AgentMiddleware[AgentState[ResponseT], ContextT, ResponseT] +): """Two-stage tool-name repair on the most recent ``AIMessage``. Args: registered_tool_names: Set of canonically-registered tool names. ``invalid`` should be in this set so the fallback dispatches. - fuzzy_match_threshold: Optional ``difflib`` ratio (0–1) for the + fuzzy_match_threshold: Optional ``difflib`` ratio (0-1) for the fuzzy-match step that runs *between* lowercase and invalid. Set to ``None`` to disable fuzzy matching (opencode parity). """ @@ -77,9 +79,9 @@ class ToolCallNameRepairMiddleware(AgentMiddleware[AgentState[ResponseT], Contex def _registered_for_runtime(self, runtime: Runtime[ContextT]) -> set[str]: """Allow runtime overrides to expand the set (e.g. dynamic MCP tools).""" ctx_tools = getattr(runtime.context, "registered_tool_names", None) - if isinstance(ctx_tools, (set, frozenset)): + if isinstance(ctx_tools, set | frozenset): return self._registered | set(ctx_tools) - if isinstance(ctx_tools, (list, tuple)): + if isinstance(ctx_tools, list | tuple): return self._registered | set(ctx_tools) return self._registered diff --git a/surfsense_backend/app/agents/new_chat/plugins/year_substituter.py b/surfsense_backend/app/agents/new_chat/plugins/year_substituter.py index 927d533d5..3e2e631d2 100644 --- a/surfsense_backend/app/agents/new_chat/plugins/year_substituter.py +++ b/surfsense_backend/app/agents/new_chat/plugins/year_substituter.py @@ -52,25 +52,26 @@ class _YearSubstituterMiddleware(AgentMiddleware): async def awrap_tool_call( self, request: ToolCallRequest, - handler: Callable[ - [ToolCallRequest], Awaitable[ToolMessage | Command[Any]] - ], + handler: Callable[[ToolCallRequest], Awaitable[ToolMessage | Command[Any]]], ) -> ToolMessage | Command[Any]: result = await handler(request) try: from langchain_core.messages import ToolMessage - if isinstance(result, ToolMessage) and isinstance(result.content, str): - if "{{year}}" in result.content: - new_text = result.content.replace("{{year}}", self._year) - result = ToolMessage( - content=new_text, - tool_call_id=result.tool_call_id, - id=result.id, - name=result.name, - status=result.status, - artifact=result.artifact, - ) + if ( + isinstance(result, ToolMessage) + and isinstance(result.content, str) + and "{{year}}" in result.content + ): + new_text = result.content.replace("{{year}}", self._year) + result = ToolMessage( + content=new_text, + tool_call_id=result.tool_call_id, + id=result.id, + name=result.name, + status=result.status, + artifact=result.artifact, + ) except Exception: # pragma: no cover - defensive logger.exception("year_substituter plugin failed; passing original result") return result diff --git a/surfsense_backend/app/agents/new_chat/prompts/composer.py b/surfsense_backend/app/agents/new_chat/prompts/composer.py index bad033490..77b86aeef 100644 --- a/surfsense_backend/app/agents/new_chat/prompts/composer.py +++ b/surfsense_backend/app/agents/new_chat/prompts/composer.py @@ -62,7 +62,9 @@ ProviderVariant = str # More specific patterns must come first (e.g. ``codex`` before # ``openai_reasoning`` because codex model ids contain ``gpt``). -_OPENAI_CODEX_RE = re.compile(r"\b(gpt-codex|codex-mini|gpt-[\d.]+-codex)\b", re.IGNORECASE) +_OPENAI_CODEX_RE = re.compile( + r"\b(gpt-codex|codex-mini|gpt-[\d.]+-codex)\b", re.IGNORECASE +) _OPENAI_REASONING_RE = re.compile(r"\b(gpt-5|o\d|o-)", re.IGNORECASE) _OPENAI_CLASSIC_RE = re.compile(r"\bgpt-4", re.IGNORECASE) _ANTHROPIC_RE = re.compile(r"\bclaude\b", re.IGNORECASE) @@ -257,9 +259,7 @@ def _build_tools_section( ) if known_disabled: disabled_list = ", ".join( - _format_tool_label(n) - for n in ALL_TOOL_NAMES_ORDERED - if n in known_disabled + _format_tool_label(n) for n in ALL_TOOL_NAMES_ORDERED if n in known_disabled ) parts.append( "\n" diff --git a/surfsense_backend/app/agents/new_chat/subagents/config.py b/surfsense_backend/app/agents/new_chat/subagents/config.py index e20bc06bf..b36d35fa0 100644 --- a/surfsense_backend/app/agents/new_chat/subagents/config.py +++ b/surfsense_backend/app/agents/new_chat/subagents/config.py @@ -279,9 +279,7 @@ def build_explore_subagent( selected_tools = _filter_tools(tools, EXPLORE_READ_TOOLS) deny_rules = _read_only_deny_rules() - permission_mw = _build_permission_middleware( - deny_rules, origin="subagent_explore" - ) + permission_mw = _build_permission_middleware(deny_rules, origin="subagent_explore") spec: dict = { "name": "explore", diff --git a/surfsense_backend/app/agents/new_chat/tools/registry.py b/surfsense_backend/app/agents/new_chat/tools/registry.py index f5ee1a61d..fce1bf872 100644 --- a/surfsense_backend/app/agents/new_chat/tools/registry.py +++ b/surfsense_backend/app/agents/new_chat/tools/registry.py @@ -111,6 +111,8 @@ from .update_memory import create_update_memory_tool, create_update_team_memory_ from .video_presentation import create_generate_video_presentation_tool from .web_search import create_web_search_tool +logger = logging.getLogger(__name__) + # ============================================================================= # Tool Definition # ============================================================================= diff --git a/surfsense_backend/app/observability/otel.py b/surfsense_backend/app/observability/otel.py index 0229524f2..4f2257ab7 100644 --- a/surfsense_backend/app/observability/otel.py +++ b/surfsense_backend/app/observability/otel.py @@ -22,6 +22,7 @@ Goals from __future__ import annotations +import contextlib import logging import os from collections.abc import Iterator @@ -154,18 +155,14 @@ def span( with tracer.start_as_current_span(name) as sp: if attributes: - try: + with contextlib.suppress(Exception): # pragma: no cover — defensive sp.set_attributes(attributes) - except Exception: # pragma: no cover — defensive - pass try: yield sp except BaseException as exc: - try: + with contextlib.suppress(Exception): # pragma: no cover — defensive sp.record_exception(exc) sp.set_status(_OtStatus(_OtStatusCode.ERROR, str(exc))) - except Exception: # pragma: no cover — defensive - pass raise diff --git a/surfsense_backend/app/routes/agent_flags_route.py b/surfsense_backend/app/routes/agent_flags_route.py index d3c90a58d..5732a8dfb 100644 --- a/surfsense_backend/app/routes/agent_flags_route.py +++ b/surfsense_backend/app/routes/agent_flags_route.py @@ -59,7 +59,7 @@ class AgentFeatureFlagsRead(BaseModel): enable_otel: bool @classmethod - def from_flags(cls, flags: AgentFeatureFlags) -> "AgentFeatureFlagsRead": + def from_flags(cls, flags: AgentFeatureFlags) -> AgentFeatureFlagsRead: # asdict() avoids missing-field bugs when AgentFeatureFlags grows. return cls(**asdict(flags)) diff --git a/surfsense_backend/app/routes/agent_permissions_route.py b/surfsense_backend/app/routes/agent_permissions_route.py index e87af29c7..1c76e00e6 100644 --- a/surfsense_backend/app/routes/agent_permissions_route.py +++ b/surfsense_backend/app/routes/agent_permissions_route.py @@ -210,7 +210,7 @@ async def create_rule( session.add(row) try: await session.commit() - except IntegrityError: + except IntegrityError as err: await session.rollback() raise HTTPException( status_code=409, @@ -218,7 +218,7 @@ async def create_rule( "An identical rule already exists for this scope. Update the " "existing rule instead." ), - ) + ) from err await session.refresh(row) return _to_read(row) @@ -248,12 +248,12 @@ async def update_rule( try: await session.commit() - except IntegrityError: + except IntegrityError as err: await session.rollback() raise HTTPException( status_code=409, detail="Update would create a duplicate rule for this scope.", - ) + ) from err await session.refresh(row) return _to_read(row) diff --git a/surfsense_backend/app/routes/agent_revert_route.py b/surfsense_backend/app/routes/agent_revert_route.py index 2f6fe6a32..cbe4e7417 100644 --- a/surfsense_backend/app/routes/agent_revert_route.py +++ b/surfsense_backend/app/routes/agent_revert_route.py @@ -97,10 +97,12 @@ async def revert_agent_action( action=action, requester_user_id=str(user.id) if user is not None else None, ) - except Exception: + except Exception as err: logger.exception("Revert dispatch raised for action_id=%s", action_id) await session.rollback() - raise HTTPException(status_code=500, detail="Internal error during revert.") + raise HTTPException( + status_code=500, detail="Internal error during revert." + ) from err if outcome.status == "ok": await session.commit() diff --git a/surfsense_backend/app/routes/new_chat_routes.py b/surfsense_backend/app/routes/new_chat_routes.py index cbc660222..b5560d90d 100644 --- a/surfsense_backend/app/routes/new_chat_routes.py +++ b/surfsense_backend/app/routes/new_chat_routes.py @@ -1242,7 +1242,9 @@ async def handle_new_chat( await session.close() image_urls = ( - [p.as_data_url() for p in request.user_images] if request.user_images else None + [p.as_data_url() for p in request.user_images] + if request.user_images + else None ) return StreamingResponse( diff --git a/surfsense_backend/app/services/revert_service.py b/surfsense_backend/app/services/revert_service.py index e072f90c6..f3630e0b4 100644 --- a/surfsense_backend/app/services/revert_service.py +++ b/surfsense_backend/app/services/revert_service.py @@ -79,9 +79,7 @@ async def load_action( return result.scalars().first() -async def load_thread( - session: AsyncSession, *, thread_id: int -) -> NewChatThread | None: +async def load_thread(session: AsyncSession, *, thread_id: int) -> NewChatThread | None: stmt = select(NewChatThread).where(NewChatThread.id == thread_id) result = await session.execute(stmt) return result.scalars().first() diff --git a/surfsense_backend/app/utils/user_message_multimodal.py b/surfsense_backend/app/utils/user_message_multimodal.py index 1d0691697..dc9a6fe76 100644 --- a/surfsense_backend/app/utils/user_message_multimodal.py +++ b/surfsense_backend/app/utils/user_message_multimodal.py @@ -7,7 +7,9 @@ import binascii from typing import Any -def build_human_message_content(final_query: str, image_data_urls: list[str]) -> str | list[dict[str, Any]]: +def build_human_message_content( + final_query: str, image_data_urls: list[str] +) -> str | list[dict[str, Any]]: if not image_data_urls: return final_query parts: list[dict[str, Any]] = [{"type": "text", "text": final_query}] diff --git a/surfsense_backend/tests/unit/agents/new_chat/prompts/test_composer.py b/surfsense_backend/tests/unit/agents/new_chat/prompts/test_composer.py index d08bbc8cf..aa0c215b9 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/prompts/test_composer.py +++ b/surfsense_backend/tests/unit/agents/new_chat/prompts/test_composer.py @@ -90,9 +90,7 @@ class TestCompose: assert "" in prompt assert "[citation:chunk_id]" in prompt - def test_team_visibility_uses_team_variants( - self, fixed_today: datetime - ) -> None: + def test_team_visibility_uses_team_variants(self, fixed_today: datetime) -> None: prompt = compose_system_prompt( today=fixed_today, thread_visibility=ChatVisibility.SEARCH_SPACE, @@ -145,9 +143,7 @@ class TestCompose: assert "Generate Image" in prompt assert "Generate Podcast" in prompt - def test_mcp_routing_block_emits_when_provided( - self, fixed_today: datetime - ) -> None: + def test_mcp_routing_block_emits_when_provided(self, fixed_today: datetime) -> None: prompt = compose_system_prompt( today=fixed_today, mcp_connector_tools={"My GitLab": ["gitlab_search", "gitlab_create_mr"]}, @@ -162,9 +158,7 @@ class TestCompose: prompt = compose_system_prompt(today=fixed_today, mcp_connector_tools={}) assert "" not in prompt - def test_provider_block_renders_when_anthropic( - self, fixed_today: datetime - ) -> None: + def test_provider_block_renders_when_anthropic(self, fixed_today: datetime) -> None: prompt = compose_system_prompt( today=fixed_today, model_name="anthropic:claude-3-5-sonnet" ) @@ -267,7 +261,10 @@ class TestStableOrderingForCacheStability: ) b = compose_system_prompt( today=fixed_today, - enabled_tool_names={"scrape_webpage", "web_search"}, # set order shouldn't matter + enabled_tool_names={ + "scrape_webpage", + "web_search", + }, # set order shouldn't matter mcp_connector_tools={"X": ["x_a", "x_b"]}, ) assert a == b diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_action_log.py b/surfsense_backend/tests/unit/agents/new_chat/test_action_log.py index 6834b5be7..aad1524c9 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/test_action_log.py +++ b/surfsense_backend/tests/unit/agents/new_chat/test_action_log.py @@ -83,7 +83,11 @@ class TestActionLogMiddlewareDisabled: async def test_no_op_when_flag_off(self, patch_get_flags) -> None: mw = ActionLogMiddleware(thread_id=1, search_space_id=1, user_id=None) request = _FakeRequest( - tool_call={"name": "make_widget", "args": {"color": "red", "size": 1}, "id": "tc1"} + tool_call={ + "name": "make_widget", + "args": {"color": "red", "size": 1}, + "id": "tc1", + } ) handler = AsyncMock(return_value=ToolMessage(content="ok", tool_call_id="tc1")) with patch_get_flags(_disabled_flags()): @@ -117,13 +121,12 @@ class TestActionLogMiddlewarePersistence: "id": "tc-abc", }, ) - result_msg = ToolMessage( - content="ok", tool_call_id="tc-abc", id="msg-1" - ) + result_msg = ToolMessage(content="ok", tool_call_id="tc-abc", id="msg-1") handler = AsyncMock(return_value=result_msg) - with patch_get_flags(_enabled_flags()), patch( - "app.db.shielded_async_session", side_effect=lambda: factory() + with ( + patch_get_flags(_enabled_flags()), + patch("app.db.shielded_async_session", side_effect=lambda: factory()), ): result = await mw.awrap_tool_call(request, handler) @@ -151,9 +154,11 @@ class TestActionLogMiddlewarePersistence: ) handler = AsyncMock(side_effect=ValueError("boom")) - with patch_get_flags(_enabled_flags()), patch( - "app.db.shielded_async_session", side_effect=lambda: factory() - ), pytest.raises(ValueError, match="boom"): + with ( + patch_get_flags(_enabled_flags()), + patch("app.db.shielded_async_session", side_effect=lambda: factory()), + pytest.raises(ValueError, match="boom"), + ): await mw.awrap_tool_call(request, handler) assert len(captured["rows"]) == 1 @@ -177,8 +182,9 @@ class TestActionLogMiddlewarePersistence: def _exploding_session(): raise RuntimeError("DB is down") - with patch_get_flags(_enabled_flags()), patch( - "app.db.shielded_async_session", side_effect=_exploding_session + with ( + patch_get_flags(_enabled_flags()), + patch("app.db.shielded_async_session", side_effect=_exploding_session), ): result = await mw.awrap_tool_call(request, handler) assert result is result_msg @@ -218,8 +224,9 @@ class TestReverseDescriptor: ) handler = AsyncMock(return_value=result_msg) - with patch_get_flags(_enabled_flags()), patch( - "app.db.shielded_async_session", side_effect=lambda: factory() + with ( + patch_get_flags(_enabled_flags()), + patch("app.db.shielded_async_session", side_effect=lambda: factory()), ): await mw.awrap_tool_call(request, handler) @@ -257,8 +264,9 @@ class TestReverseDescriptor: result_msg = ToolMessage(content="ok", tool_call_id="tc1") handler = AsyncMock(return_value=result_msg) - with patch_get_flags(_enabled_flags()), patch( - "app.db.shielded_async_session", side_effect=lambda: factory() + with ( + patch_get_flags(_enabled_flags()), + patch("app.db.shielded_async_session", side_effect=lambda: factory()), ): await mw.awrap_tool_call(request, handler) @@ -275,11 +283,10 @@ class TestReverseDescriptor: request = _FakeRequest( tool_call={"name": "unknown_tool", "args": {}, "id": "tc1"} ) - handler = AsyncMock( - return_value=ToolMessage(content="ok", tool_call_id="tc1") - ) - with patch_get_flags(_enabled_flags()), patch( - "app.db.shielded_async_session", side_effect=lambda: factory() + handler = AsyncMock(return_value=ToolMessage(content="ok", tool_call_id="tc1")) + with ( + patch_get_flags(_enabled_flags()), + patch("app.db.shielded_async_session", side_effect=lambda: factory()), ): await mw.awrap_tool_call(request, handler) row = captured["rows"][0] @@ -298,11 +305,10 @@ class TestArgsTruncation: request = _FakeRequest( tool_call={"name": "make_widget", "args": {"blob": huge}, "id": "tc1"}, ) - handler = AsyncMock( - return_value=ToolMessage(content="ok", tool_call_id="tc1") - ) - with patch_get_flags(_enabled_flags()), patch( - "app.db.shielded_async_session", side_effect=lambda: factory() + handler = AsyncMock(return_value=ToolMessage(content="ok", tool_call_id="tc1")) + with ( + patch_get_flags(_enabled_flags()), + patch("app.db.shielded_async_session", side_effect=lambda: factory()), ): await mw.awrap_tool_call(request, handler) row = captured["rows"][0] diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_compaction.py b/surfsense_backend/tests/unit/agents/new_chat/test_compaction.py index 4d8d6805c..c6d4cc452 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/test_compaction.py +++ b/surfsense_backend/tests/unit/agents/new_chat/test_compaction.py @@ -26,10 +26,16 @@ class TestIsProtectedSystemMessage: assert _is_protected_system_message(msg) is True def test_unprotected_system_message(self) -> None: - assert _is_protected_system_message(SystemMessage(content="random instructions")) is False + assert ( + _is_protected_system_message(SystemMessage(content="random instructions")) + is False + ) def test_human_message_never_protected(self) -> None: - assert _is_protected_system_message(HumanMessage(content="...")) is False + assert ( + _is_protected_system_message(HumanMessage(content="...")) + is False + ) def test_tolerates_leading_whitespace(self) -> None: msg = SystemMessage(content=" \n\n...") @@ -97,11 +103,17 @@ class TestPartitionMessages: assert protected not in to_summary assert protected in preserved # The non-protected old messages remain in to_summary - assert any(isinstance(m, HumanMessage) and m.content == "old human" for m in to_summary) + assert any( + isinstance(m, HumanMessage) and m.content == "old human" for m in to_summary + ) def test_unprotected_messages_unaffected(self) -> None: partitioner = self._build_partitioner() - msgs = [HumanMessage(content="a"), HumanMessage(content="b"), HumanMessage(content="c")] + msgs = [ + HumanMessage(content="a"), + HumanMessage(content="b"), + HumanMessage(content="c"), + ] to_summary, preserved = partitioner._partition_messages(msgs, 2) assert [m.content for m in to_summary] == ["a", "b"] assert [m.content for m in preserved] == ["c"] diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_context_editing.py b/surfsense_backend/tests/unit/agents/new_chat/test_context_editing.py index 3c31155d4..ba2246413 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/test_context_editing.py +++ b/surfsense_backend/tests/unit/agents/new_chat/test_context_editing.py @@ -70,7 +70,8 @@ class TestSpillEdit: # Earlier ToolMessages should now contain the placeholder text cleared = [ - m for m in tool_messages + m + for m in tool_messages if isinstance(m.content, str) and m.content.startswith("[cleared") ] assert len(cleared) >= 1 diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_dedup_tool_calls.py b/surfsense_backend/tests/unit/agents/new_chat/test_dedup_tool_calls.py index 95017d744..e04f50815 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/test_dedup_tool_calls.py +++ b/surfsense_backend/tests/unit/agents/new_chat/test_dedup_tool_calls.py @@ -46,9 +46,21 @@ def test_callable_dedup_key_takes_priority() -> None: state = { "messages": [ _msg( - {"name": "create_doc", "args": {"parent_id": "x", "title": "y"}, "id": "1"}, - {"name": "create_doc", "args": {"parent_id": "x", "title": "y"}, "id": "2"}, - {"name": "create_doc", "args": {"parent_id": "x", "title": "z"}, "id": "3"}, + { + "name": "create_doc", + "args": {"parent_id": "x", "title": "y"}, + "id": "1", + }, + { + "name": "create_doc", + "args": {"parent_id": "x", "title": "y"}, + "id": "2", + }, + { + "name": "create_doc", + "args": {"parent_id": "x", "title": "z"}, + "id": "3", + }, ) ] } diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_default_permissions_layering.py b/surfsense_backend/tests/unit/agents/new_chat/test_default_permissions_layering.py index d49edbfec..ac6b5d95c 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/test_default_permissions_layering.py +++ b/surfsense_backend/tests/unit/agents/new_chat/test_default_permissions_layering.py @@ -84,9 +84,7 @@ class TestConnectorDenyOverridesDefaultAllow: Rule(permission="linear_create_issue", pattern="*", action="deny") ] ) - rules = evaluate_many( - "linear_create_issue", ["linear_create_issue"], *rulesets - ) + rules = evaluate_many("linear_create_issue", ["linear_create_issue"], *rulesets) assert aggregate_action(rules) == "deny" def test_default_allow_still_applies_to_other_tools(self) -> None: @@ -124,5 +122,7 @@ class TestUserRuleOverridesDefault: rules=[Rule(permission="send_*", pattern="*", action="deny")], origin="user", ) - rules = evaluate_many("send_gmail_email", ["send_gmail_email"], defaults, user_ruleset) + rules = evaluate_many( + "send_gmail_email", ["send_gmail_email"], defaults, user_ruleset + ) assert aggregate_action(rules) == "deny" diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_doom_loop.py b/surfsense_backend/tests/unit/agents/new_chat/test_doom_loop.py index c54163dc3..802129bf6 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/test_doom_loop.py +++ b/surfsense_backend/tests/unit/agents/new_chat/test_doom_loop.py @@ -64,22 +64,17 @@ def test_threshold_triggers_after_n_identical_calls() -> None: runtime, ) name = type(excinfo.value).__name__.lower() - assert ( - "interrupt" in name - or "runtimeerror" in name - ), f"Expected an interrupt-style exception, got {name}" + assert "interrupt" in name or "runtimeerror" in name, ( + f"Expected an interrupt-style exception, got {name}" + ) def test_does_not_trigger_when_args_differ() -> None: mw = DoomLoopMiddleware(threshold=2) runtime = _FakeRuntime() - out = mw.after_model( - {"messages": [_msg_calling("repeat", {"x": 1}, "1")]}, runtime - ) + out = mw.after_model({"messages": [_msg_calling("repeat", {"x": 1}, "1")]}, runtime) assert out is None - out = mw.after_model( - {"messages": [_msg_calling("repeat", {"x": 2}, "2")]}, runtime - ) + out = mw.after_model({"messages": [_msg_calling("repeat", {"x": 2}, "2")]}, runtime) assert out is None diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_noop_injection.py b/surfsense_backend/tests/unit/agents/new_chat/test_noop_injection.py index 8555eea76..346271f4b 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/test_noop_injection.py +++ b/surfsense_backend/tests/unit/agents/new_chat/test_noop_injection.py @@ -91,7 +91,9 @@ class TestShouldInject: mw = NoopInjectionMiddleware() req = _FakeRequest( tools=[object()], - messages=[AIMessage(content="", tool_calls=[{"name": "x", "args": {}, "id": "1"}])], + messages=[ + AIMessage(content="", tool_calls=[{"name": "x", "args": {}, "id": "1"}]) + ], model=_LiteLLMModel(), ) assert mw._should_inject(req) is False @@ -109,7 +111,9 @@ class TestShouldInject: mw = NoopInjectionMiddleware() req = _FakeRequest( tools=[], - messages=[AIMessage(content="", tool_calls=[{"name": "x", "args": {}, "id": "1"}])], + messages=[ + AIMessage(content="", tool_calls=[{"name": "x", "args": {}, "id": "1"}]) + ], model=_OpenAIModel(), ) assert mw._should_inject(req) is False diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_permission_middleware.py b/surfsense_backend/tests/unit/agents/new_chat/test_permission_middleware.py index 194a6eb27..a997c8d61 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/test_permission_middleware.py +++ b/surfsense_backend/tests/unit/agents/new_chat/test_permission_middleware.py @@ -111,6 +111,4 @@ class TestAsk: assert out is None # call kept # Runtime ruleset got the always-allow rule new_rules = [r for r in mw._runtime_ruleset.rules if r.action == "allow"] - assert any( - r.permission == "send_email" for r in new_rules - ) + assert any(r.permission == "send_email" for r in new_rules) diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_plugin_loader.py b/surfsense_backend/tests/unit/agents/new_chat/test_plugin_loader.py index 8d98e1328..c2118c697 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/test_plugin_loader.py +++ b/surfsense_backend/tests/unit/agents/new_chat/test_plugin_loader.py @@ -69,7 +69,9 @@ class TestPluginLoaderBasics: "app.agents.new_chat.plugin_loader.entry_points", return_value=[ep], ): - result = load_plugin_middlewares(_ctx(), allowed_plugin_names=["allowed_only"]) + result = load_plugin_middlewares( + _ctx(), allowed_plugin_names=["allowed_only"] + ) assert result == [] assert not called @@ -135,9 +137,7 @@ class TestPluginLoaderIsolation: _FakeEntryPoint("crashing", crashing_factory), _FakeEntryPoint("ok", year_substituter_factory), ] - with patch( - "app.agents.new_chat.plugin_loader.entry_points", return_value=eps - ): + with patch("app.agents.new_chat.plugin_loader.entry_points", return_value=eps): result = load_plugin_middlewares( _ctx(), allowed_plugin_names={"crashing", "ok"} ) @@ -151,9 +151,7 @@ class TestAllowlistEnv: assert load_allowed_plugin_names_from_env() == set() def test_parses_comma_separated_value(self, monkeypatch) -> None: - monkeypatch.setenv( - "SURFSENSE_ALLOWED_PLUGINS", " year_substituter , noisy , " - ) + monkeypatch.setenv("SURFSENSE_ALLOWED_PLUGINS", " year_substituter , noisy , ") assert load_allowed_plugin_names_from_env() == { "year_substituter", "noisy", diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_retry_after.py b/surfsense_backend/tests/unit/agents/new_chat/test_retry_after.py index 39dd9bf00..d23fd693b 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/test_retry_after.py +++ b/surfsense_backend/tests/unit/agents/new_chat/test_retry_after.py @@ -18,7 +18,7 @@ class _FakeResponse: self.headers = headers -class _FakeRateLimit(Exception): +class _FakeRateLimitError(Exception): def __init__(self, msg: str, headers: dict[str, str] | None = None) -> None: super().__init__(msg) if headers is not None: @@ -27,15 +27,15 @@ class _FakeRateLimit(Exception): class TestExtractRetryAfter: def test_seconds_header(self) -> None: - exc = _FakeRateLimit("rate", {"Retry-After": "30"}) + exc = _FakeRateLimitError("rate", {"Retry-After": "30"}) assert _extract_retry_after_seconds(exc) == 30.0 def test_milliseconds_header_overrides_seconds(self) -> None: - exc = _FakeRateLimit("rate", {"retry-after-ms": "1500"}) + exc = _FakeRateLimitError("rate", {"retry-after-ms": "1500"}) assert _extract_retry_after_seconds(exc) == 1.5 def test_case_insensitive(self) -> None: - exc = _FakeRateLimit("rate", {"RETRY-AFTER": "12"}) + exc = _FakeRateLimitError("rate", {"RETRY-AFTER": "12"}) assert _extract_retry_after_seconds(exc) == 12.0 def test_falls_back_to_message_regex(self) -> None: @@ -67,7 +67,7 @@ class TestIsNonRetryable: class TestDelayCalculation: def test_takes_max_of_backoff_and_header(self) -> None: mw = RetryAfterMiddleware(max_retries=3, initial_delay=1.0, jitter=False) - exc = _FakeRateLimit("rl", {"retry-after": "10"}) + exc = _FakeRateLimitError("rl", {"retry-after": "10"}) delay = mw._delay_for_attempt(0, exc) assert delay == pytest.approx(10.0) diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_specialized_subagents.py b/surfsense_backend/tests/unit/agents/new_chat/test_specialized_subagents.py index 3819b4605..0adb578ce 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/test_specialized_subagents.py +++ b/surfsense_backend/tests/unit/agents/new_chat/test_specialized_subagents.py @@ -122,7 +122,9 @@ class TestExploreSubagent: def test_includes_permission_middleware_with_deny_rules(self) -> None: spec = build_explore_subagent(tools=ALL_TOOLS) permission_mws = [ - m for m in spec["middleware"] if isinstance(m, PermissionMiddleware) # type: ignore[index] + m + for m in spec["middleware"] + if isinstance(m, PermissionMiddleware) # type: ignore[index] ] assert len(permission_mws) == 1 ruleset = permission_mws[0]._static_rulesets[0] @@ -164,7 +166,9 @@ class TestReportWriterSubagent: def test_deny_rules_block_writes_but_allow_generate_report(self) -> None: spec = build_report_writer_subagent(tools=ALL_TOOLS) permission_mws = [ - m for m in spec["middleware"] if isinstance(m, PermissionMiddleware) # type: ignore[index] + m + for m in spec["middleware"] + if isinstance(m, PermissionMiddleware) # type: ignore[index] ] ruleset = permission_mws[0]._static_rulesets[0] deny_patterns = {r.permission for r in ruleset.rules if r.action == "deny"} @@ -194,17 +198,15 @@ class TestConnectorNegotiatorSubagent: def test_deny_ruleset_blocks_mutating_connector_tools(self) -> None: spec = build_connector_negotiator_subagent(tools=ALL_TOOLS) permission_mws = [ - m for m in spec["middleware"] if isinstance(m, PermissionMiddleware) # type: ignore[index] + m + for m in spec["middleware"] + if isinstance(m, PermissionMiddleware) # type: ignore[index] ] ruleset = permission_mws[0]._static_rulesets[0] deny_patterns = {r.permission for r in ruleset.rules if r.action == "deny"} # `linear_create_issue` matches the `*_create` deny pattern. - assert any( - _wildcard_matches(p, "linear_create_issue") for p in deny_patterns - ) - assert any( - _wildcard_matches(p, "slack_send_message") for p in deny_patterns - ) + assert any(_wildcard_matches(p, "linear_create_issue") for p in deny_patterns) + assert any(_wildcard_matches(p, "slack_send_message") for p in deny_patterns) class TestBuildSpecializedSubagents: @@ -242,8 +244,7 @@ class TestBuildSpecializedSubagents: # order: extra → custom → patch → dedup. sentinel_idx = mws.index(sentinel) perm_idx = next( - (i for i, m in enumerate(mws) - if isinstance(m, PermissionMiddleware)), + (i for i, m in enumerate(mws) if isinstance(m, PermissionMiddleware)), None, ) assert perm_idx is not None @@ -259,7 +260,9 @@ class TestFilterToolsWarningSuppression: from app.agents.new_chat.subagents.config import _filter_tools - with caplog.at_level(logging.INFO, logger="app.agents.new_chat.subagents.config"): + with caplog.at_level( + logging.INFO, logger="app.agents.new_chat.subagents.config" + ): # Allowed set asks for two registry tools (one present, one # not) plus a bunch of middleware-provided names. _filter_tools( @@ -275,9 +278,7 @@ class TestFilterToolsWarningSuppression: }, ) - warnings = [ - r.message for r in caplog.records if r.levelno >= logging.INFO - ] + warnings = [r.message for r in caplog.records if r.levelno >= logging.INFO] # Exactly one warning, and it should mention scrape_webpage but not # any middleware-provided name. Inspect the rendered "missing" # list (between the brackets) so we don't false-match substrings diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_tool_call_repair.py b/surfsense_backend/tests/unit/agents/new_chat/test_tool_call_repair.py index f792aef60..e02a04774 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/test_tool_call_repair.py +++ b/surfsense_backend/tests/unit/agents/new_chat/test_tool_call_repair.py @@ -27,9 +27,12 @@ class TestRepair: mw = ToolCallNameRepairMiddleware( registered_tool_names={"echo"}, fuzzy_match_threshold=None ) - msg = AIMessage(content="", tool_calls=[ - {"name": "echo", "args": {}, "id": "1"}, - ]) + msg = AIMessage( + content="", + tool_calls=[ + {"name": "echo", "args": {}, "id": "1"}, + ], + ) out = mw.after_model(_make_state(msg), _FakeRuntime()) assert out is None # no change @@ -37,9 +40,12 @@ class TestRepair: mw = ToolCallNameRepairMiddleware( registered_tool_names={"echo"}, fuzzy_match_threshold=None ) - msg = AIMessage(content="", tool_calls=[ - {"name": "Echo", "args": {"x": 1}, "id": "1"}, - ]) + msg = AIMessage( + content="", + tool_calls=[ + {"name": "Echo", "args": {"x": 1}, "id": "1"}, + ], + ) out = mw.after_model(_make_state(msg), _FakeRuntime()) assert out is not None repaired = out["messages"][0] @@ -50,9 +56,12 @@ class TestRepair: registered_tool_names={"echo", INVALID_TOOL_NAME}, fuzzy_match_threshold=None, ) - msg = AIMessage(content="", tool_calls=[ - {"name": "totally_different_name", "args": {"k": "v"}, "id": "1"}, - ]) + msg = AIMessage( + content="", + tool_calls=[ + {"name": "totally_different_name", "args": {"k": "v"}, "id": "1"}, + ], + ) out = mw.after_model(_make_state(msg), _FakeRuntime()) assert out is not None repaired_call = out["messages"][0].tool_calls[0] @@ -64,9 +73,12 @@ class TestRepair: mw = ToolCallNameRepairMiddleware( registered_tool_names={"echo"}, fuzzy_match_threshold=None ) - msg = AIMessage(content="", tool_calls=[ - {"name": "unknown", "args": {}, "id": "1"}, - ]) + msg = AIMessage( + content="", + tool_calls=[ + {"name": "unknown", "args": {}, "id": "1"}, + ], + ) out = mw.after_model(_make_state(msg), _FakeRuntime()) # No repair available; original returned unchanged (no update) assert out is None @@ -76,9 +88,12 @@ class TestRepair: registered_tool_names={"search_documents"}, fuzzy_match_threshold=0.7, ) - msg = AIMessage(content="", tool_calls=[ - {"name": "search_docments", "args": {}, "id": "1"}, - ]) + msg = AIMessage( + content="", + tool_calls=[ + {"name": "search_docments", "args": {}, "id": "1"}, + ], + ) out = mw.after_model(_make_state(msg), _FakeRuntime()) assert out is not None assert out["messages"][0].tool_calls[0]["name"] == "search_documents" @@ -94,9 +109,12 @@ class TestRepair: mw = ToolCallNameRepairMiddleware( registered_tool_names={"echo"}, fuzzy_match_threshold=None ) - msg = AIMessage(content="", tool_calls=[ - {"name": "DynamicTool", "args": {}, "id": "1"}, - ]) + msg = AIMessage( + content="", + tool_calls=[ + {"name": "DynamicTool", "args": {}, "id": "1"}, + ], + ) runtime = _FakeRuntime(SimpleNamespace(registered_tool_names=["dynamictool"])) out = mw.after_model(_make_state(msg), runtime) assert out is not None diff --git a/surfsense_backend/tests/unit/middleware/test_kb_persistence_filesystem_parity.py b/surfsense_backend/tests/unit/middleware/test_kb_persistence_filesystem_parity.py index 8b464d48d..ef95434bf 100644 --- a/surfsense_backend/tests/unit/middleware/test_kb_persistence_filesystem_parity.py +++ b/surfsense_backend/tests/unit/middleware/test_kb_persistence_filesystem_parity.py @@ -10,7 +10,7 @@ through :class:`KnowledgeBasePersistenceMiddleware` without losing the copy. from __future__ import annotations from typing import Any -from unittest.mock import AsyncMock, MagicMock +from unittest.mock import AsyncMock import numpy as np import pytest diff --git a/surfsense_backend/tests/unit/services/test_revert_service.py b/surfsense_backend/tests/unit/services/test_revert_service.py index cb8443291..e2cbe383a 100644 --- a/surfsense_backend/tests/unit/services/test_revert_service.py +++ b/surfsense_backend/tests/unit/services/test_revert_service.py @@ -16,9 +16,7 @@ class _FakeAction: class TestCanRevert: def test_owner_can_revert_their_own_action(self) -> None: action = _FakeAction(user_id="user-123") - assert can_revert( - requester_user_id="user-123", action=action, is_admin=False - ) + assert can_revert(requester_user_id="user-123", action=action, is_admin=False) def test_other_user_cannot_revert(self) -> None: action = _FakeAction(user_id="user-123") @@ -28,21 +26,15 @@ class TestCanRevert: def test_admin_always_allowed(self) -> None: action = _FakeAction(user_id="user-123") - assert can_revert( - requester_user_id="anybody", action=action, is_admin=True - ) + assert can_revert(requester_user_id="anybody", action=action, is_admin=True) def test_admin_can_revert_anonymous_action(self) -> None: action = _FakeAction(user_id=None) - assert can_revert( - requester_user_id="admin", action=action, is_admin=True - ) + assert can_revert(requester_user_id="admin", action=action, is_admin=True) def test_anonymous_action_blocks_non_admin(self) -> None: action = _FakeAction(user_id=None) - assert not can_revert( - requester_user_id="user-1", action=action, is_admin=False - ) + assert not can_revert(requester_user_id="user-1", action=action, is_admin=False) def test_uuid_string_normalization(self) -> None: """``user_id`` may be a UUID object; comparison should still work.""" @@ -51,6 +43,4 @@ class TestCanRevert: u = uuid.uuid4() action = _FakeAction(user_id=u) # Same UUID, passed as string from the requesting side. - assert can_revert( - requester_user_id=str(u), action=action, is_admin=False - ) + assert can_revert(requester_user_id=str(u), action=action, is_admin=False) From f23be16b351da11130c01d6157cc0fc817cb51b9 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Tue, 28 Apr 2026 23:25:26 -0700 Subject: [PATCH 7/8] refactor: citation viewer --- surfsense_web/app/globals.css | 21 - .../atoms/citation/citation-panel.atom.ts | 40 ++ .../pending-chunk-highlight.atom.ts | 19 - .../atoms/layout/right-panel.atom.ts | 2 +- .../assistant-ui/inline-citation.tsx | 80 +--- .../citation-panel/citation-panel.tsx | 230 ++++++++++ .../components/editor-panel/editor-panel.tsx | 407 +----------------- .../components/editor/plate-editor.tsx | 30 +- surfsense_web/components/editor/presets.ts | 28 -- .../layout/ui/right-panel/RightPanel.tsx | 72 +++- .../components/ui/search-highlight-node.tsx | 45 -- surfsense_web/lib/citation-search.ts | 125 ------ surfsense_web/package.json | 1 - surfsense_web/pnpm-lock.yaml | 17 - 14 files changed, 362 insertions(+), 755 deletions(-) create mode 100644 surfsense_web/atoms/citation/citation-panel.atom.ts delete mode 100644 surfsense_web/atoms/document-viewer/pending-chunk-highlight.atom.ts create mode 100644 surfsense_web/components/citation-panel/citation-panel.tsx delete mode 100644 surfsense_web/components/ui/search-highlight-node.tsx delete mode 100644 surfsense_web/lib/citation-search.ts diff --git a/surfsense_web/app/globals.css b/surfsense_web/app/globals.css index f54bc2197..a37ddb8f3 100644 --- a/surfsense_web/app/globals.css +++ b/surfsense_web/app/globals.css @@ -210,27 +210,6 @@ button { } } -/* Citation-jump highlight — entrance pulse only. The `SearchHighlightLeaf` - (see components/ui/search-highlight-node.tsx) is otherwise statically - tinted; this animation runs once on mount to draw the eye to the cited - text after `scrollIntoView` lands. The highlight itself is permanent - until the user clicks inside the editor (or another dismissal trigger - fires in `EditorPanelContent`). */ -@keyframes citation-flash-in { - 0% { - background-color: transparent; - box-shadow: 0 0 0 0 transparent; - } - 40% { - background-color: color-mix(in oklab, var(--primary) 30%, transparent); - box-shadow: 0 0 0 3px color-mix(in oklab, var(--primary) 25%, transparent); - } - 100% { - background-color: color-mix(in oklab, var(--primary) 15%, transparent); - box-shadow: 0 0 0 1px color-mix(in oklab, var(--primary) 40%, transparent); - } -} - /* Human-in-the-loop approval card animations */ @keyframes pulse-subtle { 0%, diff --git a/surfsense_web/atoms/citation/citation-panel.atom.ts b/surfsense_web/atoms/citation/citation-panel.atom.ts new file mode 100644 index 000000000..ca7312857 --- /dev/null +++ b/surfsense_web/atoms/citation/citation-panel.atom.ts @@ -0,0 +1,40 @@ +import { atom } from "jotai"; +import { rightPanelCollapsedAtom, rightPanelTabAtom } from "@/atoms/layout/right-panel.atom"; + +interface CitationPanelState { + isOpen: boolean; + chunkId: number | null; +} + +const initialState: CitationPanelState = { + isOpen: false, + chunkId: null, +}; + +export const citationPanelAtom = atom(initialState); + +export const citationPanelOpenAtom = atom((get) => get(citationPanelAtom).isOpen); + +const preCitationCollapsedAtom = atom(null); + +export const openCitationPanelAtom = atom(null, (get, set, payload: { chunkId: number }) => { + if (!get(citationPanelAtom).isOpen) { + set(preCitationCollapsedAtom, get(rightPanelCollapsedAtom)); + } + set(citationPanelAtom, { + isOpen: true, + chunkId: payload.chunkId, + }); + set(rightPanelTabAtom, "citation"); + set(rightPanelCollapsedAtom, false); +}); + +export const closeCitationPanelAtom = atom(null, (get, set) => { + set(citationPanelAtom, initialState); + set(rightPanelTabAtom, "sources"); + const prev = get(preCitationCollapsedAtom); + if (prev !== null) { + set(rightPanelCollapsedAtom, prev); + set(preCitationCollapsedAtom, null); + } +}); diff --git a/surfsense_web/atoms/document-viewer/pending-chunk-highlight.atom.ts b/surfsense_web/atoms/document-viewer/pending-chunk-highlight.atom.ts deleted file mode 100644 index a3f8357e8..000000000 --- a/surfsense_web/atoms/document-viewer/pending-chunk-highlight.atom.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { atom } from "jotai"; - -/** - * Cross-component handoff for citation jumps. Set by `InlineCitation` when a - * numeric chunk badge is clicked (after the document has been resolved); read - * by `DocumentTabContent` once the matching document tab mounts so it can - * scroll to and softly highlight the cited chunk inside the rendered markdown. - * - * Cleared by `DocumentTabContent` only after a terminal state — exact / - * approximate / miss — has been reached, so that an escalation refetch (2MB - * preview → 16MB) keeps the pending intent alive across the re-render. - */ -export interface PendingChunkHighlight { - documentId: number; - chunkId: number; - chunkText: string; -} - -export const pendingChunkHighlightAtom = atom(null); diff --git a/surfsense_web/atoms/layout/right-panel.atom.ts b/surfsense_web/atoms/layout/right-panel.atom.ts index e06500113..d296587ed 100644 --- a/surfsense_web/atoms/layout/right-panel.atom.ts +++ b/surfsense_web/atoms/layout/right-panel.atom.ts @@ -1,6 +1,6 @@ import { atom } from "jotai"; -export type RightPanelTab = "sources" | "report" | "editor" | "hitl-edit"; +export type RightPanelTab = "sources" | "report" | "editor" | "hitl-edit" | "citation"; export const rightPanelTabAtom = atom("sources"); diff --git a/surfsense_web/components/assistant-ui/inline-citation.tsx b/surfsense_web/components/assistant-ui/inline-citation.tsx index ae8d434a8..2aeba89ca 100644 --- a/surfsense_web/components/assistant-ui/inline-citation.tsx +++ b/surfsense_web/components/assistant-ui/inline-citation.tsx @@ -1,13 +1,11 @@ "use client"; -import { useQuery, useQueryClient } from "@tanstack/react-query"; +import { useQuery } from "@tanstack/react-query"; import { useSetAtom } from "jotai"; import { ExternalLink, FileText } from "lucide-react"; import type { FC } from "react"; import { useCallback, useEffect, useRef, useState } from "react"; -import { toast } from "sonner"; -import { pendingChunkHighlightAtom } from "@/atoms/document-viewer/pending-chunk-highlight.atom"; -import { openEditorPanelAtom } from "@/atoms/editor/editor-panel.atom"; +import { openCitationPanelAtom } from "@/atoms/citation/citation-panel.atom"; import { useCitationMetadata } from "@/components/assistant-ui/citation-metadata-context"; import { MarkdownViewer } from "@/components/markdown-viewer"; import { Citation } from "@/components/tool-ui/citation"; @@ -29,11 +27,11 @@ const POPOVER_HOVER_CLOSE_DELAY_MS = 150; * Surfsense documentation chunks (`isDocsChunk`). Negative chunk IDs render as * a static "doc" pill (anonymous/synthetic uploads). * - * Numeric KB chunks: clicking resolves the parent document via - * `getDocumentByChunk`, opens the document in the right side panel (alongside - * the chat — does not replace it), and stages the cited chunk text in - * `pendingChunkHighlightAtom` so `EditorPanelContent` can scroll to and softly - * highlight it inside the rendered markdown. + * Numeric KB chunks: clicking opens the citation panel in the right + * sidebar (alongside the chat — does not replace it). The panel shows + * the cited chunk surrounded by adjacent chunks (via the API's + * `chunk_window`), with the cited one highlighted and an option to + * expand the window or jump into the full document via the editor panel. * * Surfsense docs chunks: rendered as a hover-controlled shadcn Popover that * lazily fetches and previews the cited chunk inline, since those docs aren't @@ -65,71 +63,17 @@ export const InlineCitation: FC = ({ chunkId, isDocsChunk = }; const NumericChunkCitation: FC<{ chunkId: number }> = ({ chunkId }) => { - const queryClient = useQueryClient(); - const setPendingHighlight = useSetAtom(pendingChunkHighlightAtom); - const openEditorPanel = useSetAtom(openEditorPanelAtom); - const [resolving, setResolving] = useState(false); - - const handleClick = useCallback(async () => { - if (resolving) return; - setResolving(true); - console.log("[citation:click] start", { chunkId }); - try { - const data = await queryClient.fetchQuery({ - // Local key with explicit window. The shared `cacheKeys.documents.byChunk` - // is window-agnostic (latent footgun); namespace the call to avoid - // reusing a different-window cached result. - queryKey: ["documents", "by-chunk", chunkId, "w0"] as const, - queryFn: () => - documentsApiService.getDocumentByChunk({ chunk_id: chunkId, chunk_window: 0 }), - staleTime: 5 * 60 * 1000, - }); - const cited = data.chunks.find((c) => c.id === chunkId) ?? data.chunks[0]; - console.log("[citation:click] fetched doc-by-chunk", { - docId: data.id, - docTitle: data.title, - chunksReturned: data.chunks.length, - citedChunkId: cited?.id, - citedChunkContentLen: cited?.content?.length ?? 0, - citedChunkPreview: - cited?.content && cited.content.length > 120 - ? `${cited.content.slice(0, 120)}…(+${cited.content.length - 120})` - : (cited?.content ?? ""), - }); - // Stage the highlight BEFORE opening the panel so `EditorPanelContent` - // already sees the pending intent on its very first render — avoids a - // "fetch → render → no-pending → next-tick render with pending" race. - setPendingHighlight({ - documentId: data.id, - chunkId, - chunkText: cited?.content ?? "", - }); - openEditorPanel({ - documentId: data.id, - searchSpaceId: data.search_space_id, - title: data.title, - }); - console.log("[citation:click] staged highlight + opened editor panel", { - documentId: data.id, - }); - } catch (err) { - console.warn("[citation:click] failed", err); - toast.error(err instanceof Error ? err.message : "Couldn't open cited document"); - } finally { - setResolving(false); - } - }, [chunkId, openEditorPanel, queryClient, resolving, setPendingHighlight]); + const openCitationPanel = useSetAtom(openCitationPanelAtom); return ( ); }; diff --git a/surfsense_web/components/citation-panel/citation-panel.tsx b/surfsense_web/components/citation-panel/citation-panel.tsx new file mode 100644 index 000000000..cec07b9cf --- /dev/null +++ b/surfsense_web/components/citation-panel/citation-panel.tsx @@ -0,0 +1,230 @@ +"use client"; + +import { useQuery } from "@tanstack/react-query"; +import { useSetAtom } from "jotai"; +import { ChevronDown, ChevronUp, ExternalLink, XIcon } from "lucide-react"; +import type { FC } from "react"; +import { useEffect, useMemo, useRef, useState } from "react"; +import { openEditorPanelAtom } from "@/atoms/editor/editor-panel.atom"; +import { MarkdownViewer } from "@/components/markdown-viewer"; +import { Button } from "@/components/ui/button"; +import { Spinner } from "@/components/ui/spinner"; +import { documentsApiService } from "@/lib/apis/documents-api.service"; + +const DEFAULT_CHUNK_WINDOW = 5; +const EXPANDED_CHUNK_WINDOW = 50; + +interface CitationPanelContentProps { + chunkId: number; + onClose?: () => void; +} + +/** + * Right-panel citation viewer. Shows the cited chunk surrounded by + * adjacent chunks (±N chunks via the API's `chunk_window` parameter), + * with the cited one visually highlighted and auto-scrolled into view. + * The window can be expanded to a wider range, or the user can jump to + * the full document via the editor panel. + */ +export const CitationPanelContent: FC = ({ chunkId, onClose }) => { + const openEditorPanel = useSetAtom(openEditorPanelAtom); + const [expanded, setExpanded] = useState(false); + + useEffect(() => { + setExpanded(false); + }, []); + + const chunkWindow = expanded ? EXPANDED_CHUNK_WINDOW : DEFAULT_CHUNK_WINDOW; + + const { data, isLoading, error } = useQuery({ + queryKey: ["citation-panel", chunkId, chunkWindow] as const, + queryFn: () => + documentsApiService.getDocumentByChunk({ + chunk_id: chunkId, + chunk_window: chunkWindow, + }), + staleTime: 5 * 60 * 1000, + }); + + const cited = useMemo(() => data?.chunks.find((c) => c.id === chunkId) ?? null, [data, chunkId]); + + const totalChunks = data?.total_chunks ?? data?.chunks.length ?? 0; + const startIndex = data?.chunk_start_index ?? 0; + const citedIndexInWindow = data + ? Math.max( + 0, + data.chunks.findIndex((c) => c.id === chunkId) + ) + : 0; + const shownAbove = citedIndexInWindow; + const shownBelow = data ? Math.max(0, data.chunks.length - 1 - citedIndexInWindow) : 0; + const hasMoreAbove = startIndex > 0; + const hasMoreBelow = data ? startIndex + data.chunks.length < totalChunks : false; + + // Scroll the cited chunk into view inside the panel's scroll container + // (not the page). We anchor the scroll to the panel's scroll element + // so opening the citation doesn't yank the chat scroll on the left. + const scrollContainerRef = useRef(null); + const citedRef = useRef(null); + useEffect(() => { + if (!cited) return; + const id = requestAnimationFrame(() => { + const container = scrollContainerRef.current; + const target = citedRef.current; + if (!container || !target) return; + const containerRect = container.getBoundingClientRect(); + const targetRect = target.getBoundingClientRect(); + const offset = targetRect.top - containerRect.top + container.scrollTop; + container.scrollTo({ + top: Math.max(0, offset - 16), + behavior: "smooth", + }); + }); + return () => cancelAnimationFrame(id); + }, [cited]); + + const handleOpenFullDocument = () => { + if (!data) return; + openEditorPanel({ + documentId: data.id, + searchSpaceId: data.search_space_id, + title: data.title, + }); + }; + + return ( + <> +
+
+

Citation

+
+ {onClose && ( + + )} +
+
+
+
+

+ {data?.title ?? (isLoading ? "Loading…" : `Chunk #${chunkId}`)} +

+
+
+ Chunk #{chunkId} + {totalChunks > 0 && · {totalChunks} chunks} +
+
+
+ +
+ {isLoading && ( +
+ + Loading citation… +
+ )} + + {error && ( +

+ {error instanceof Error ? error.message : "Failed to load citation"} +

+ )} + + {!isLoading && !error && data && ( + <> + {hasMoreAbove && ( +

+ … {startIndex} earlier chunk{startIndex === 1 ? "" : "s"} not shown +

+ )} +
+ {data.chunks.map((chunk) => { + const isCited = chunk.id === chunkId; + return ( +
+
+ + {isCited ? "Cited chunk" : `Chunk #${chunk.id}`} + + {isCited && ( + #{chunk.id} + )} +
+
+ +
+
+ ); + })} +
+ {hasMoreBelow && ( +

+ … {totalChunks - (startIndex + data.chunks.length)} later chunk + {totalChunks - (startIndex + data.chunks.length) === 1 ? "" : "s"} not shown +

+ )} + + )} +
+ + {!isLoading && !error && data && ( +
+
+ Showing {shownAbove} above · cited · {shownBelow} below +
+
+ {(hasMoreAbove || hasMoreBelow) && !expanded && ( + + )} + {expanded && ( + + )} + +
+
+ )} + + ); +}; diff --git a/surfsense_web/components/editor-panel/editor-panel.tsx b/surfsense_web/components/editor-panel/editor-panel.tsx index 0c4e9485b..df138e97e 100644 --- a/surfsense_web/components/editor-panel/editor-panel.tsx +++ b/surfsense_web/components/editor-panel/editor-panel.tsx @@ -1,6 +1,5 @@ "use client"; -import { FindReplacePlugin } from "@platejs/find-replace"; import { useAtomValue, useSetAtom } from "jotai"; import { Check, @@ -15,21 +14,17 @@ import { import dynamic from "next/dynamic"; import { useCallback, useEffect, useRef, useState } from "react"; import { toast } from "sonner"; -import { pendingChunkHighlightAtom } from "@/atoms/document-viewer/pending-chunk-highlight.atom"; import { closeEditorPanelAtom, editorPanelAtom } from "@/atoms/editor/editor-panel.atom"; import { VersionHistoryButton } from "@/components/documents/version-history"; -import type { PlateEditorInstance } from "@/components/editor/plate-editor"; import { SourceCodeEditor } from "@/components/editor/source-code-editor"; import { MarkdownViewer } from "@/components/markdown-viewer"; import { Alert, AlertDescription } from "@/components/ui/alert"; import { Button } from "@/components/ui/button"; import { Drawer, DrawerContent, DrawerHandle, DrawerTitle } from "@/components/ui/drawer"; -import { CITATION_HIGHLIGHT_CLASS } from "@/components/ui/search-highlight-node"; import { Spinner } from "@/components/ui/spinner"; import { useMediaQuery } from "@/hooks/use-media-query"; import { useElectronAPI } from "@/hooks/use-platform"; import { authenticatedFetch, getBearerToken, redirectToLogin } from "@/lib/auth-utils"; -import { buildCitationSearchCandidates } from "@/lib/citation-search"; import { inferMonacoLanguageFromPath } from "@/lib/editor-language"; const PlateEditor = dynamic( @@ -37,10 +32,7 @@ const PlateEditor = dynamic( { ssr: false, loading: () => } ); -type CitationHighlightStatus = "exact" | "miss"; - const LARGE_DOCUMENT_THRESHOLD = 2 * 1024 * 1024; // 2MB -const CITATION_MAX_LENGTH = 16 * 1024 * 1024; // 16MB on-demand cap for citation jumps interface EditorContent { document_id: number; @@ -145,60 +137,6 @@ export function EditorPanelContent({ const isLocalFileMode = kind === "local_file"; const editorRenderMode: EditorRenderMode = isLocalFileMode ? "source_code" : "rich_markdown"; - // --- Citation-jump highlight wiring ---------------------------------- - // `EditorPanelContent` is the consumer of `pendingChunkHighlightAtom`: when - // a citation badge is clicked, the badge stages `{documentId, chunkId, - // chunkText}` and opens this panel. We drive Plate's `FindReplacePlugin` - // (registered in every preset) to highlight the cited text natively via - // Slate decorations — no DOM walking, no Range gymnastics. The state - // machine below escalates the document fetch from 2MB → 16MB once if no - // candidate snippet matched in the preview, and surfaces miss outcomes - // via an inline alert. - const pending = useAtomValue(pendingChunkHighlightAtom); - const setPendingHighlight = useSetAtom(pendingChunkHighlightAtom); - const [fetchKey, setFetchKey] = useState(0); - const [maxLengthOverride, setMaxLengthOverride] = useState(null); - const [highlightResult, setHighlightResult] = useState(null); - const editorRef = useRef(null); - const escalatedForRef = useRef(null); - const lastAppliedChunkIdRef = useRef(null); - // Tracks whether a citation highlight is currently decorated in the - // editor. We use a ref (not state) because the click-to-dismiss handler - // runs in a stable callback that would otherwise close over stale state. - const isHighlightActiveRef = useRef(false); - // Once a citation jump targets this doc we have to keep `PlateEditor` - // mounted for the *rest of the doc session* — even after the highlight - // effect clears `pendingChunkHighlightAtom` (which it does as soon as - // the decoration is applied, so a follow-up citation on the same chunk - // can re-trigger). Without this latch, non-editable docs would re-render - // back into `MarkdownViewer` the instant `pending` is released, tearing - // down the Plate decorations and dropping the highlight after a frame. - const [stickyPlateMode, setStickyPlateMode] = useState(false); - - const clearCitationSearch = useCallback(() => { - isHighlightActiveRef.current = false; - const editor = editorRef.current; - if (!editor) return; - try { - editor.setOption(FindReplacePlugin, "search", ""); - editor.api.redecorate(); - } catch (err) { - console.warn("[EditorPanelContent] clearCitationSearch failed:", err); - } - }, []); - - // Dismiss the highlight when the user interacts with the editor surface. - // `onPointerDown` fires before focus / selection changes so the click - // itself feels responsive — the highlight clears in the same event tick - // that places the cursor. No-op when nothing is highlighted, so we don't - // thrash `redecorate` on every click in normal editing. - const handleEditorPointerDown = useCallback(() => { - if (!isHighlightActiveRef.current) return; - clearCitationSearch(); - setHighlightResult(null); - }, [clearCitationSearch]); - - const isCitationTarget = !!pending && !isLocalFileMode && pending.documentId === documentId; const resolveLocalVirtualPath = useCallback( async (candidatePath: string): Promise => { if (!electronAPI?.getAgentFilesystemMounts) { @@ -218,8 +156,6 @@ export function EditorPanelContent({ const isLargeDocument = (editorDoc?.content_size_bytes ?? 0) > LARGE_DOCUMENT_THRESHOLD; - // `fetchKey` is an explicit re-fetch trigger (escalation bumps it to force - // a new request even when documentId/searchSpaceId haven't changed). useEffect(() => { const controller = new AbortController(); setIsLoading(true); @@ -231,12 +167,6 @@ export function EditorPanelContent({ setIsEditing(false); initialLoadDone.current = false; changeCountRef.current = 0; - // Clear any in-flight FindReplacePlugin search before the editor - // re-mounts on new content (a fresh editor key is generated below - // from documentId + isEditing, so the previous editor + its - // decorations are about to be discarded anyway, but we belt-and- - // brace here for the case where only `fetchKey` changed). - clearCitationSearch(); const doFetch = async () => { try { @@ -281,11 +211,7 @@ export function EditorPanelContent({ const url = new URL( `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-spaces/${searchSpaceId}/documents/${documentId}/editor-content` ); - url.searchParams.set("max_length", String(maxLengthOverride ?? LARGE_DOCUMENT_THRESHOLD)); - // `fetchKey` participates here so biome's noUnusedVariables sees it - // as consumed; bumping it forces a fresh request even when the URL - // is otherwise identical. - if (fetchKey > 0) url.searchParams.set("_n", String(fetchKey)); + url.searchParams.set("max_length", String(LARGE_DOCUMENT_THRESHOLD)); const response = await authenticatedFetch(url.toString(), { method: "GET" }); @@ -331,259 +257,8 @@ export function EditorPanelContent({ resolveLocalVirtualPath, searchSpaceId, title, - fetchKey, - maxLengthOverride, - clearCitationSearch, ]); - // Reset citation-jump bookkeeping whenever the panel switches to a different - // document (or local file). Body only writes setters — the deps are the - // real triggers we want to react to. - // biome-ignore lint/correctness/useExhaustiveDependencies: documentId/localFilePath are intentional triggers. - useEffect(() => { - clearCitationSearch(); - escalatedForRef.current = null; - lastAppliedChunkIdRef.current = null; - setHighlightResult(null); - setMaxLengthOverride(null); - setFetchKey(0); - // Drop sticky Plate mode when the panel moves to a different doc - // — the next doc starts in its preferred render mode (Plate for - // editable, MarkdownViewer for everything else) until/unless a - // citation jump targets it. - setStickyPlateMode(false); - }, [documentId, localFilePath, clearCitationSearch]); - - // Latch sticky Plate mode the first time a citation jump targets this - // doc. We keep it sticky for the remainder of this doc session so the - // highlight effect's `setPendingHighlight(null)` doesn't unmount the - // editor mid-flight (see comment on `stickyPlateMode` declaration). - useEffect(() => { - if (isCitationTarget) setStickyPlateMode(true); - }, [isCitationTarget]); - - // `isEditorReady` is what `useEffect` actually depends on — `editorRef` - // is a ref so changes don't trigger re-runs. We flip this to `true` once - // `PlateEditor` calls back with its live editor instance (its - // `usePlateEditor` value-init runs synchronously, so by the time this - // flips true the markdown is already deserialized into the Slate tree). - const [isEditorReady, setIsEditorReady] = useState(false); - const handleEditorReady = useCallback((editor: PlateEditorInstance | null) => { - console.log("[citation:editor] handleEditorReady", { ready: !!editor }); - editorRef.current = editor; - setIsEditorReady(!!editor); - }, []); - - // --- Citation jump highlight effect ----------------------------------- - // Drives Plate's FindReplacePlugin to highlight the cited chunk: - // 1. Build candidate snippets from the chunk text (first sentence, - // first 8 words, full chunk if short). Plate's decorate runs per- - // block and won't cross block boundaries, so the shorter - // candidates exist to give us something that fits in one - // paragraph / heading. - // 2. For each candidate: setOption('search', ...) → redecorate → - // wait two animation frames for React to flush → query the editor - // DOM for `.${CITATION_HIGHLIGHT_CLASS}`. First hit wins. - // - // Why a className and not a `data-*` attribute? Plate's - // `PlateLeaf` runs its props through `useNodeAttributes`, which - // only forwards `attributes`, `className`, `ref`, and `style` — - // arbitrary `data-*` attributes are silently dropped. `className` - // is the only escape hatch guaranteed to survive into the DOM. - // 3. On hit: smooth-scroll the first match into view, mark the - // highlight active (so a click inside the editor can dismiss it), - // release the pending atom. - // 4. On terminal miss: if the doc was truncated and we haven't - // escalated yet, bump the fetch's `max_length` to the citation - // cap and re-fetch — the post-refetch render will re-run this - // effect against the larger preview. Otherwise, release the - // atom and show the miss alert. - useEffect(() => { - console.log("[citation:effect] fired", { - isCitationTarget, - pendingDocId: pending?.documentId, - pendingChunkId: pending?.chunkId, - pendingChunkTextLen: pending?.chunkText?.length, - documentId, - isLocalFileMode, - isEditing, - hasMarkdown: !!editorDoc?.source_markdown, - markdownLen: editorDoc?.source_markdown?.length, - truncated: editorDoc?.truncated, - isEditorReady, - editorRefSet: !!editorRef.current, - maxLengthOverride, - }); - if (!isCitationTarget || !pending) { - console.log("[citation:effect] guard ✗ no citation target / no pending"); - return; - } - if (isLocalFileMode || isEditing) { - console.log("[citation:effect] guard ✗ localFileMode/editing"); - return; - } - if (!editorDoc?.source_markdown) { - console.log("[citation:effect] guard ✗ source_markdown not ready"); - return; - } - if (!isEditorReady) { - console.log("[citation:effect] guard ✗ editor not ready yet"); - return; - } - const editor = editorRef.current; - if (!editor) { - console.log("[citation:effect] guard ✗ editorRef.current is null"); - return; - } - - if (lastAppliedChunkIdRef.current !== pending.chunkId) { - lastAppliedChunkIdRef.current = pending.chunkId; - } - - let cancelled = false; - - const finishMiss = () => { - console.log("[citation:effect] terminal miss — no candidate matched"); - try { - editor.setOption(FindReplacePlugin, "search", ""); - editor.api.redecorate(); - } catch (err) { - console.warn("[EditorPanelContent] reset search after miss failed:", err); - } - const canEscalate = - editorDoc.truncated === true && - (maxLengthOverride ?? LARGE_DOCUMENT_THRESHOLD) < CITATION_MAX_LENGTH && - escalatedForRef.current !== pending.chunkId; - console.log("[citation:effect] miss decision", { - truncated: editorDoc.truncated, - currentMaxLength: maxLengthOverride ?? LARGE_DOCUMENT_THRESHOLD, - canEscalate, - }); - if (canEscalate) { - escalatedForRef.current = pending.chunkId; - setMaxLengthOverride(CITATION_MAX_LENGTH); - setFetchKey((k) => k + 1); - // Keep the atom set so the post-refetch render re-runs. - return; - } - setHighlightResult("miss"); - setPendingHighlight(null); - }; - - const tryCandidates = async () => { - const candidates = buildCitationSearchCandidates(pending.chunkText); - console.log("[citation:effect] candidates built", { - count: candidates.length, - previews: candidates.map((c) => c.slice(0, 60)), - }); - if (candidates.length === 0) { - if (!cancelled) finishMiss(); - return; - } - // Resolve the editor's rendered DOM root via Slate's stable - // `[data-slate-editor="true"]` attribute (set by slate-react's - // ``). Scoping queries to this root prevents - // `` elements rendered elsewhere on the page (e.g. chat - // search-highlight leaves in another mounted PlateEditor) from - // being mistaken for citation hits. - const editorRoot = document.querySelector('[data-slate-editor="true"]'); - console.log("[citation:effect] editor root", { - hasRoot: !!editorRoot, - }); - const root: ParentNode = editorRoot ?? document; - - for (let i = 0; i < candidates.length; i++) { - const candidate = candidates[i]; - if (cancelled) return; - try { - editor.setOption(FindReplacePlugin, "search", candidate); - editor.api.redecorate(); - console.log(`[citation:effect] try #${i} setOption + redecorate`, { - len: candidate.length, - preview: candidate.slice(0, 80), - }); - } catch (err) { - console.warn("[EditorPanelContent] setOption/redecorate failed:", err); - continue; - } - // Two rAFs: first lets Slate flush its onChange, second lets - // React commit the decoration leaves into the DOM. - await new Promise((resolve) => - requestAnimationFrame(() => requestAnimationFrame(() => resolve())) - ); - if (cancelled) return; - // Primary probe: by our stable class on the rendered . - let el = root.querySelector(`.${CITATION_HIGHLIGHT_CLASS}`); - const classMarkCount = root.querySelectorAll(`.${CITATION_HIGHLIGHT_CLASS}`).length; - // Diagnostic fallback: any inside the editor root. - // If we ever see allMarks > 0 but classMarkCount === 0, - // the className was stripped again and we need to revisit - // `useNodeAttributes` filtering. - const allMarkCount = root.querySelectorAll("mark").length; - if (!el && allMarkCount > 0) { - el = root.querySelector("mark"); - } - console.log(`[citation:effect] try #${i} DOM probe`, { - foundEl: !!el, - classMarkCount, - allMarkCount, - usedFallback: !!el && classMarkCount === 0, - }); - if (el) { - try { - el.scrollIntoView({ block: "center", behavior: "smooth" }); - } catch { - el.scrollIntoView(); - } - isHighlightActiveRef.current = true; - setHighlightResult("exact"); - console.log(`[citation:effect] ✓ exact via candidate #${i} — atom released`); - // No auto-clear timer — the highlight is intentionally - // permanent until the user clicks inside the editor (see - // `handleEditorPointerDown`) or another dismissal trigger - // fires (doc switch, edit-mode toggle, panel unmount, - // next citation jump). Sticky Plate mode keeps the - // editor mounted after the atom clears. - setPendingHighlight(null); - return; - } - } - if (!cancelled) finishMiss(); - }; - - void tryCandidates(); - - return () => { - cancelled = true; - }; - }, [ - isCitationTarget, - pending, - documentId, - editorDoc?.source_markdown, - editorDoc?.truncated, - isLocalFileMode, - isEditing, - isEditorReady, - maxLengthOverride, - clearCitationSearch, - setPendingHighlight, - ]); - - // Cleanup any active highlight on unmount. - useEffect(() => { - return () => clearCitationSearch(); - }, [clearCitationSearch]); - - // Toggling into edit mode swaps Plate out of readOnly. Clear the citation - // search so stale leaves don't linger in the editing surface. - useEffect(() => { - if (isEditing) { - clearCitationSearch(); - setHighlightResult(null); - } - }, [isEditing, clearCitationSearch]); - useEffect(() => { return () => { if (copyResetTimeoutRef.current) { @@ -617,7 +292,7 @@ export function EditorPanelContent({ }, [editorDoc?.source_markdown]); const handleSave = useCallback( - async (_options?: { silent?: boolean }) => { + async (options?: { silent?: boolean }) => { setSaving(true); try { if (isLocalFileMode) { @@ -668,11 +343,15 @@ export function EditorPanelContent({ setEditorDoc((prev) => (prev ? { ...prev, source_markdown: markdownRef.current } : prev)); setEditedMarkdown(null); - toast.success("Document saved! Reindexing in background..."); + if (!options?.silent) { + toast.success("Document saved! Reindexing in background..."); + } return true; } catch (err) { console.error("Error saving document:", err); - toast.error(err instanceof Error ? err.message : "Failed to save document"); + if (!options?.silent) { + toast.error(err instanceof Error ? err.message : "Failed to save document"); + } return false; } finally { setSaving(false); @@ -693,15 +372,11 @@ export function EditorPanelContent({ EDITABLE_DOCUMENT_TYPES.has(editorDoc.document_type ?? "")) && !isLargeDocument : false; - // Use PlateEditor for any of: - // - Editable doc types (FILE/NOTE) — existing editing UX. - // - Active citation jump in flight (`isCitationTarget`) — covers the - // mount in the very first render where the atom is set but the - // sticky effect hasn't fired yet. - // - Sticky Plate mode latched on a previous citation jump — keeps - // the editor mounted (with its decorations) after the highlight - // effect clears the atom. Resets when the doc changes. - const renderInPlateEditor = isEditableType || isCitationTarget || stickyPlateMode; + // Render through PlateEditor for editable doc types (FILE/NOTE). + // Everything else (large docs, non-editable types) falls back to the + // lightweight `MarkdownViewer` — Plate is heavy on multi-MB docs and + // non-editable types don't benefit from its editing UX. + const renderInPlateEditor = isEditableType; const hasUnsavedChanges = editedMarkdown !== null; const showDesktopHeader = !!onClose; const showEditingActions = isEditableType && isEditing; @@ -744,36 +419,6 @@ export function EditorPanelContent({ } }, [documentId, editorDoc?.title, searchSpaceId]); - // We no longer surface an "approximate" status — Plate's FindReplacePlugin - // either decorates an exact match or it doesn't, and the candidate snippet - // strategy (first sentence → first 8 words → full chunk) means we either - // land on the citation start or fall through to the miss alert. - const showMissAlert = isCitationTarget && highlightResult === "miss"; - - const citationAlerts = showMissAlert && ( - - - - Cited section couldn't be located in this view. - {editorDoc?.truncated && ( - - )} - - - ); - const largeDocAlert = isLargeDocument && !isLocalFileMode && editorDoc && ( @@ -1002,30 +647,17 @@ export function EditorPanelContent({ }} />
- ) : isLargeDocument && !isLocalFileMode && !isCitationTarget ? ( - // Large doc, no active citation — fast Streamdown preview - // + download CTA. We only fall back to MarkdownViewer here - // because Plate is heavy on multi-MB docs and the user - // isn't waiting on a specific citation to render. + ) : isLargeDocument && !isLocalFileMode ? ( + // Large doc — fast Streamdown preview + download CTA. + // Plate is heavy on multi-MB docs.
{largeDocAlert}
) : renderInPlateEditor ? ( - // Editable doc (FILE/NOTE) OR active citation jump (any - // doc type). The citation path uses Plate's - // FindReplacePlugin for native, decoration-based - // highlighting — see the citation-jump highlight effect - // above for how `editorRef` and `handleEditorReady` are - // wired. + // Editable doc (FILE/NOTE) — Plate editing UX.
- {(citationAlerts || (isLargeDocument && isCitationTarget && !isLocalFileMode)) && ( -
- {isLargeDocument && isCitationTarget && largeDocAlert} - {citationAlerts} -
- )} -
+
diff --git a/surfsense_web/components/editor/plate-editor.tsx b/surfsense_web/components/editor/plate-editor.tsx index eef18ef6a..7f12d3cae 100644 --- a/surfsense_web/components/editor/plate-editor.tsx +++ b/surfsense_web/components/editor/plate-editor.tsx @@ -12,10 +12,7 @@ import { type EditorPreset, presetMap } from "@/components/editor/presets"; import { escapeMdxExpressions } from "@/components/editor/utils/escape-mdx"; import { Editor, EditorContainer } from "@/components/ui/editor"; -/** Live editor instance returned by `usePlateEditor`. Exposed via the - * `onEditorReady` prop so callers (e.g. `EditorPanelContent`) can drive - * plugin options imperatively — most notably setting - * `FindReplacePlugin`'s `search` option for citation-jump highlights. */ +/** Live editor instance returned by `usePlateEditor`. */ export type PlateEditorInstance = ReturnType; export interface PlateEditorProps { @@ -68,15 +65,6 @@ export interface PlateEditorProps { * without modifying the core editor component. */ extraPlugins?: AnyPluginConfig[]; - /** - * Called whenever the live editor instance (re)mounts, with `null` on - * unmount. Used by callers that need to drive plugin options imperatively - * — e.g. `EditorPanelContent` setting `FindReplacePlugin`'s `search` - * option for citation-jump highlights. The callback is invoked exactly - * once per editor lifetime (the parent's `key` prop forces a fresh - * editor when needed, e.g. on edit-mode toggle). - */ - onEditorReady?: (editor: PlateEditorInstance | null) => void; } function PlateEditorContent({ @@ -115,7 +103,6 @@ export function PlateEditor({ defaultEditing = false, preset = "full", extraPlugins = [], - onEditorReady, }: PlateEditorProps) { const lastMarkdownRef = useRef(markdown); const lastHtmlRef = useRef(html); @@ -172,21 +159,6 @@ export function PlateEditor({ : undefined, }); - // Expose the live editor instance to imperative callers (e.g. citation - // jump highlights). We deliberately don't depend on `onEditorReady` - // itself in the cleanup closure — callers commonly pass an arrow that - // closes over a stable ref setter, but if they pass a freshly-bound - // callback per render, the `onEditorReady?.(editor)` re-fires which is - // idempotent for ref-style setters. - const onEditorReadyRef = useRef(onEditorReady); - useEffect(() => { - onEditorReadyRef.current = onEditorReady; - }, [onEditorReady]); - useEffect(() => { - onEditorReadyRef.current?.(editor); - return () => onEditorReadyRef.current?.(null); - }, [editor]); - // Update editor content when html prop changes externally useEffect(() => { if (html !== undefined && html !== lastHtmlRef.current) { diff --git a/surfsense_web/components/editor/presets.ts b/surfsense_web/components/editor/presets.ts index 49f53ecf1..c207b5e56 100644 --- a/surfsense_web/components/editor/presets.ts +++ b/surfsense_web/components/editor/presets.ts @@ -1,6 +1,5 @@ "use client"; -import { FindReplacePlugin } from "@platejs/find-replace"; import type { AnyPluginConfig } from "platejs"; import { TrailingBlockPlugin } from "platejs"; @@ -18,30 +17,6 @@ import { SelectionKit } from "@/components/editor/plugins/selection-kit"; import { SlashCommandKit } from "@/components/editor/plugins/slash-command-kit"; import { TableKit } from "@/components/editor/plugins/table-kit"; import { ToggleKit } from "@/components/editor/plugins/toggle-kit"; -import { SearchHighlightLeaf } from "@/components/ui/search-highlight-node"; - -/** - * Citation-jump highlighter. Re-uses Plate's built-in `FindReplacePlugin` - * (decorate-only, no editing surface) to drive the "scroll-to-cited-text" - * UX in `EditorPanelContent`. We register it in every preset because: - * - Decorate is a no-op when `search` is empty (single getOptions() check - * per block), so cost is effectively zero for non-citation viewers. - * - Keeping it preset-agnostic means citations work whether the doc is - * opened in editable (`full`) or pure-viewer (`readonly`) modes. - * - * The parent component drives `setOption(FindReplacePlugin, 'search', ...)` - * + `editor.api.redecorate()` to trigger highlights, then queries the - * editor DOM for `.citation-highlight-leaf` to scroll the first match - * into view. (We can't use a `data-*` attribute here — Plate's - * `PlateLeaf` runs props through `useNodeAttributes`, which only forwards - * `attributes`, `className`, `ref`, `style`; arbitrary `data-*` props are - * silently dropped.) See `components/ui/search-highlight-node.tsx` for - * the leaf component and `CITATION_HIGHLIGHT_CLASS` constant. - */ -const CitationFindReplacePlugin = FindReplacePlugin.configure({ - options: { search: "" }, - render: { node: SearchHighlightLeaf }, -}); /** * Full preset – every plugin kit enabled. @@ -63,7 +38,6 @@ export const fullPreset: AnyPluginConfig[] = [ ...AutoformatKit, ...DndKit, TrailingBlockPlugin, - CitationFindReplacePlugin, ]; /** @@ -78,7 +52,6 @@ export const minimalPreset: AnyPluginConfig[] = [ ...LinkKit, ...AutoformatKit, TrailingBlockPlugin, - CitationFindReplacePlugin, ]; /** @@ -95,7 +68,6 @@ export const readonlyPreset: AnyPluginConfig[] = [ ...CalloutKit, ...ToggleKit, ...MathKit, - CitationFindReplacePlugin, ]; /** All available preset names */ diff --git a/surfsense_web/components/layout/ui/right-panel/RightPanel.tsx b/surfsense_web/components/layout/ui/right-panel/RightPanel.tsx index 04bae010c..3481eec28 100644 --- a/surfsense_web/components/layout/ui/right-panel/RightPanel.tsx +++ b/surfsense_web/components/layout/ui/right-panel/RightPanel.tsx @@ -6,6 +6,7 @@ import dynamic from "next/dynamic"; import { startTransition, useEffect } from "react"; import { closeHitlEditPanelAtom, hitlEditPanelAtom } from "@/atoms/chat/hitl-edit-panel.atom"; import { closeReportPanelAtom, reportPanelAtom } from "@/atoms/chat/report-panel.atom"; +import { citationPanelAtom, closeCitationPanelAtom } from "@/atoms/citation/citation-panel.atom"; import { documentsSidebarOpenAtom } from "@/atoms/documents/ui.atoms"; import { closeEditorPanelAtom, editorPanelAtom } from "@/atoms/editor/editor-panel.atom"; import { rightPanelCollapsedAtom, rightPanelTabAtom } from "@/atoms/layout/right-panel.atom"; @@ -21,6 +22,14 @@ const EditorPanelContent = dynamic( { ssr: false, loading: () => null } ); +const CitationPanelContent = dynamic( + () => + import("@/components/citation-panel/citation-panel").then((m) => ({ + default: m.CitationPanelContent, + })), + { ssr: false, loading: () => null } +); + const HitlEditPanelContent = dynamic( () => import("@/components/hitl-edit-panel/hitl-edit-panel").then((m) => ({ @@ -69,12 +78,14 @@ export function RightPanelExpandButton() { const reportState = useAtomValue(reportPanelAtom); const editorState = useAtomValue(editorPanelAtom); const hitlEditState = useAtomValue(hitlEditPanelAtom); + const citationState = useAtomValue(citationPanelAtom); const reportOpen = reportState.isOpen && !!reportState.reportId; const editorOpen = editorState.isOpen && (editorState.kind === "document" ? !!editorState.documentId : !!editorState.localFilePath); const hitlEditOpen = hitlEditState.isOpen && !!hitlEditState.onSave; - const hasContent = documentsOpen || reportOpen || editorOpen || hitlEditOpen; + const citationOpen = citationState.isOpen && citationState.chunkId != null; + const hasContent = documentsOpen || reportOpen || editorOpen || hitlEditOpen || citationOpen; if (!collapsed || !hasContent) return null; @@ -98,7 +109,13 @@ export function RightPanelExpandButton() { ); } -const PANEL_WIDTHS = { sources: 420, report: 640, editor: 640, "hitl-edit": 640 } as const; +const PANEL_WIDTHS = { + sources: 420, + report: 640, + editor: 640, + "hitl-edit": 640, + citation: 560, +} as const; export function RightPanel({ documentsPanel }: RightPanelProps) { const [activeTab] = useAtom(rightPanelTabAtom); @@ -108,6 +125,8 @@ export function RightPanel({ documentsPanel }: RightPanelProps) { const closeEditor = useSetAtom(closeEditorPanelAtom); const hitlEditState = useAtomValue(hitlEditPanelAtom); const closeHitlEdit = useSetAtom(closeHitlEditPanelAtom); + const citationState = useAtomValue(citationPanelAtom); + const closeCitation = useSetAtom(closeCitationPanelAtom); const [collapsed, setCollapsed] = useAtom(rightPanelCollapsedAtom); const documentsOpen = documentsPanel?.open ?? false; @@ -116,37 +135,59 @@ export function RightPanel({ documentsPanel }: RightPanelProps) { editorState.isOpen && (editorState.kind === "document" ? !!editorState.documentId : !!editorState.localFilePath); const hitlEditOpen = hitlEditState.isOpen && !!hitlEditState.onSave; + const citationOpen = citationState.isOpen && citationState.chunkId != null; useEffect(() => { - if (!reportOpen && !editorOpen && !hitlEditOpen) return; + if (!reportOpen && !editorOpen && !hitlEditOpen && !citationOpen) return; const handleKeyDown = (e: KeyboardEvent) => { if (e.key === "Escape") { if (hitlEditOpen) closeHitlEdit(); + else if (citationOpen) closeCitation(); else if (editorOpen) closeEditor(); else if (reportOpen) closeReport(); } }; document.addEventListener("keydown", handleKeyDown); return () => document.removeEventListener("keydown", handleKeyDown); - }, [reportOpen, editorOpen, hitlEditOpen, closeReport, closeEditor, closeHitlEdit]); + }, [ + reportOpen, + editorOpen, + hitlEditOpen, + citationOpen, + closeReport, + closeEditor, + closeHitlEdit, + closeCitation, + ]); - const isVisible = (documentsOpen || reportOpen || editorOpen || hitlEditOpen) && !collapsed; + const isVisible = + (documentsOpen || reportOpen || editorOpen || hitlEditOpen || citationOpen) && !collapsed; let effectiveTab = activeTab; if (effectiveTab === "hitl-edit" && !hitlEditOpen) { - effectiveTab = editorOpen ? "editor" : reportOpen ? "report" : "sources"; - } else if (effectiveTab === "editor" && !editorOpen) { - effectiveTab = reportOpen ? "report" : "sources"; - } else if (effectiveTab === "report" && !reportOpen) { - effectiveTab = editorOpen ? "editor" : "sources"; - } else if (effectiveTab === "sources" && !documentsOpen) { - effectiveTab = hitlEditOpen - ? "hitl-edit" + effectiveTab = citationOpen + ? "citation" : editorOpen ? "editor" : reportOpen ? "report" : "sources"; + } else if (effectiveTab === "citation" && !citationOpen) { + effectiveTab = editorOpen ? "editor" : reportOpen ? "report" : "sources"; + } else if (effectiveTab === "editor" && !editorOpen) { + effectiveTab = citationOpen ? "citation" : reportOpen ? "report" : "sources"; + } else if (effectiveTab === "report" && !reportOpen) { + effectiveTab = citationOpen ? "citation" : editorOpen ? "editor" : "sources"; + } else if (effectiveTab === "sources" && !documentsOpen) { + effectiveTab = hitlEditOpen + ? "hitl-edit" + : citationOpen + ? "citation" + : editorOpen + ? "editor" + : reportOpen + ? "report" + : "sources"; } const targetWidth = PANEL_WIDTHS[effectiveTab]; @@ -205,6 +246,11 @@ export function RightPanel({ documentsPanel }: RightPanelProps) { />
)} + {effectiveTab === "citation" && citationOpen && citationState.chunkId != null && ( +
+ +
+ )}
); diff --git a/surfsense_web/components/ui/search-highlight-node.tsx b/surfsense_web/components/ui/search-highlight-node.tsx deleted file mode 100644 index e3f316cce..000000000 --- a/surfsense_web/components/ui/search-highlight-node.tsx +++ /dev/null @@ -1,45 +0,0 @@ -"use client"; - -import type { PlateLeafProps } from "platejs/react"; -import { PlateLeaf } from "platejs/react"; - -/** - * Stable class name used to identify Plate-rendered citation highlight - * leaves in the DOM. We can't use a `data-*` attribute here — Plate's - * `PlateLeaf` runs its props through `useNodeAttributes`, which only - * forwards `attributes`, `className`, `ref`, and `style` to the rendered - * element; arbitrary `data-*` props are silently dropped (verified - * against `@platejs/core/dist/react/index.js` v52). So `className` is - * the only escape hatch that's guaranteed to survive into the DOM. - */ -export const CITATION_HIGHLIGHT_CLASS = "citation-highlight-leaf"; - -/** - * Leaf rendered for ranges decorated by `@platejs/find-replace`'s - * `FindReplacePlugin`. We re-purpose that plugin to drive the citation-jump - * highlight: when a citation is staged, the parent sets the plugin's `search` - * option to a snippet of the chunk text and Plate decorates every match with - * `searchHighlight: true`. This component renders those decorations as a - * `` tagged with `CITATION_HIGHLIGHT_CLASS` so the parent can: - * 1. Query the first match in DOM order to scroll it into view. - * 2. Detect the active-highlight state without a separate React ref. - * - * The highlight is **persistent** — it does not auto-fade. The parent in - * `EditorPanelContent` clears it by setting the plugin's `search` option - * back to "" when one of: (a) the user clicks anywhere inside the editor, - * (b) the panel switches to a different document, (c) the user toggles - * into edit mode, (d) another citation jump is staged, (e) the panel - * unmounts. We use a brief entrance pulse (`citation-flash-in`, see - * `globals.css`) purely to draw the eye after `scrollIntoView` lands. - */ -export function SearchHighlightLeaf(props: PlateLeafProps) { - return ( - - {props.children} - - ); -} diff --git a/surfsense_web/lib/citation-search.ts b/surfsense_web/lib/citation-search.ts deleted file mode 100644 index f80f13076..000000000 --- a/surfsense_web/lib/citation-search.ts +++ /dev/null @@ -1,125 +0,0 @@ -/** - * Snippet generation for the citation-jump highlight, driven by Plate's - * `FindReplacePlugin`. The plugin runs `decorate` per-block and only matches - * within blocks whose children are all `Text` nodes (so it crosses inline - * marks like bold/italic but **not** block boundaries, and a block that - * contains even one inline element such as a link is silently skipped). - * That means a full chunk that spans heading + paragraph won't match as a - * single string — we have to pick a shorter snippet that fits inside one - * rendered block. - * - * `buildCitationSearchCandidates` returns search strings ordered from - * "most-specific anchor" to "broadest fallback": - * 1. First sentence of the chunk (capped at `FIRST_SENTENCE_MAX`). - * 2. First `FIRST_PHRASE_WORDS` words. - * 3. Each non-trivial line of the chunk, in source order — gives us a - * separate attempt for each rendered block, so a heading line with - * an inline link doesn't doom the whole jump. - * 4. Full chunk (only if it's already short enough to plausibly fit - * inside one block). - * - * The caller tries each candidate in turn — set the plugin's `search` - * option, `editor.api.redecorate()`, then check the editor DOM for a - * `.citation-highlight-leaf` element. First candidate that produces one - * wins; subsequent candidates are skipped. - */ - -const FIRST_SENTENCE_MAX = 120; -const FIRST_PHRASE_WORDS = 8; -const MIN_SNIPPET_LENGTH = 6; -const FULL_CHUNK_MAX = FIRST_SENTENCE_MAX * 2; -const MAX_LINE_CANDIDATES = 6; -const LINE_CANDIDATE_MAX = FIRST_SENTENCE_MAX; - -function normalizeWhitespace(input: string): string { - return input.replace(/\s+/g, " ").trim(); -} - -/** - * Strip the markdown syntax that won't survive into the rendered editor's - * plain text, so the chunk text (which comes back from the indexer as raw - * source markdown) can be matched against the literal text values stored - * in Plate's Slate tree. - * - * Order matters: handle multi-char and "container" syntax before single- - * char emphasis, otherwise `**text**` collapses to `*text*` first. - * - * Heuristic only — we don't aim to be a full markdown parser, just to - * remove the common markers (`**bold**`, `[text](url)`, `# headings`, - * `- list`, etc.) that show up in connector-doc chunks and would break - * literal substring search. - */ -export function stripMarkdownForMatch(input: string): string { - let s = input; - s = s.replace(/```[a-z0-9_+-]*\n?([\s\S]*?)```/gi, (_, body: string) => body); - s = s.replace(//g, " "); - s = s.replace(/!\[([^\]]*)\]\([^)]*\)/g, "$1"); - s = s.replace(/!\[([^\]]*)\]\[[^\]]*\]/g, "$1"); - s = s.replace(/\[([^\]]+)\]\([^)]*\)/g, "$1"); - s = s.replace(/\[([^\]]+)\]\[[^\]]*\]/g, "$1"); - s = s.replace(/<((?:https?|mailto):[^>\s]+)>/g, "$1"); - s = s.replace(/`+([^`\n]+?)`+/g, "$1"); - s = s.replace(/(\*\*|__)([\s\S]+?)\1/g, "$2"); - s = s.replace(/(?+[ \t]?/gm, ""); - s = s.replace(/^[ \t]*[-*+][ \t]+/gm, ""); - s = s.replace(/^[ \t]*\d+\.[ \t]+/gm, ""); - s = s.replace(/^[ \t]{0,3}(?:[-*_])(?:[ \t]*[-*_]){2,}[ \t]*$/gm, ""); - s = s.replace(/^[ \t]*\|?(?:[ \t]*:?-+:?[ \t]*\|)+[ \t]*:?-+:?[ \t]*\|?[ \t]*$/gm, ""); - s = s.replace(/\\([\\`*_{}[\]()#+\-.!~>])/g, "$1"); - return s; -} - -export function buildCitationSearchCandidates(rawText: string): string[] { - if (!rawText) return []; - const stripped = stripMarkdownForMatch(rawText); - const normalized = normalizeWhitespace(stripped); - if (normalized.length < MIN_SNIPPET_LENGTH) return []; - - const out: string[] = []; - const seen = new Set(); - const push = (s: string) => { - const t = normalizeWhitespace(s); - if (t.length >= MIN_SNIPPET_LENGTH && !seen.has(t)) { - out.push(t); - seen.add(t); - } - }; - - const sentenceMatch = normalized.match(/^[^.!?]+[.!?]/); - if (sentenceMatch) { - const sentence = sentenceMatch[0]; - push(sentence.length > FIRST_SENTENCE_MAX ? sentence.slice(0, FIRST_SENTENCE_MAX) : sentence); - } else if (normalized.length > FIRST_SENTENCE_MAX) { - push(normalized.slice(0, FIRST_SENTENCE_MAX)); - } - - const words = normalized.split(" ").filter(Boolean); - if (words.length > FIRST_PHRASE_WORDS) { - push(words.slice(0, FIRST_PHRASE_WORDS).join(" ")); - } - - // Per-line candidates: each chunk line is roughly one block in the - // rendered editor. Trying them in order gives us a separate decorate - // attempt for each block, which matters when the first line is a - // heading containing a link (Plate's `FindReplacePlugin` will skip - // any block whose children aren't all text nodes). - const rawLines = stripped.split(/\r?\n/); - let lineCount = 0; - for (const line of rawLines) { - if (lineCount >= MAX_LINE_CANDIDATES) break; - const trimmed = normalizeWhitespace(line); - if (trimmed.length < MIN_SNIPPET_LENGTH) continue; - push(trimmed.length > LINE_CANDIDATE_MAX ? trimmed.slice(0, LINE_CANDIDATE_MAX) : trimmed); - lineCount++; - } - - if (normalized.length <= FULL_CHUNK_MAX) { - push(normalized); - } - - return out; -} diff --git a/surfsense_web/package.json b/surfsense_web/package.json index 665490e4f..41175daeb 100644 --- a/surfsense_web/package.json +++ b/surfsense_web/package.json @@ -36,7 +36,6 @@ "@platejs/code-block": "^52.0.11", "@platejs/combobox": "^52.0.15", "@platejs/dnd": "^52.0.11", - "@platejs/find-replace": "^52.3.10", "@platejs/floating": "^52.0.11", "@platejs/indent": "^52.0.11", "@platejs/link": "^52.0.11", diff --git a/surfsense_web/pnpm-lock.yaml b/surfsense_web/pnpm-lock.yaml index a1a7bea12..b1730e842 100644 --- a/surfsense_web/pnpm-lock.yaml +++ b/surfsense_web/pnpm-lock.yaml @@ -53,9 +53,6 @@ importers: '@platejs/dnd': specifier: ^52.0.11 version: 52.0.11(platejs@52.0.17(@types/react@19.2.14)(immer@10.2.0)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(scheduler@0.27.0)(use-sync-external-store@1.6.0(react@19.2.4)))(react-dnd-html5-backend@16.0.1)(react-dnd@16.0.1(@types/node@20.19.33)(@types/react@19.2.14)(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4) - '@platejs/find-replace': - specifier: ^52.3.10 - version: 52.3.10(platejs@52.0.17(@types/react@19.2.14)(immer@10.2.0)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(scheduler@0.27.0)(use-sync-external-store@1.6.0(react@19.2.4)))(react-dom@19.2.4(react@19.2.4))(react@19.2.4) '@platejs/floating': specifier: ^52.0.11 version: 52.0.11(platejs@52.0.17(@types/react@19.2.14)(immer@10.2.0)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(scheduler@0.27.0)(use-sync-external-store@1.6.0(react@19.2.4)))(react-dom@19.2.4(react@19.2.4))(react@19.2.4) @@ -2830,13 +2827,6 @@ packages: react-dnd-html5-backend: '>=14.0.0' react-dom: '>=18.0.0' - '@platejs/find-replace@52.3.10': - resolution: {integrity: sha512-V/MOMMUYxHfEn/skd2+YO213xSATFDVsl8FzVzVRV/XaxwwVefH2EPD1lAVIvmYjennTVTTsHHtEI9K9iOsEaA==} - peerDependencies: - platejs: '>=52.0.11' - react: '>=18.0.0' - react-dom: '>=18.0.0' - '@platejs/floating@52.0.11': resolution: {integrity: sha512-ApNpw4KWml+kuK+XTTpji+f/7GxTR4nRzlnfJMvGBrJpLPQ4elS5MABm3oUi81DZn+aub5HvsyH7UqCw7F76IA==} peerDependencies: @@ -11115,13 +11105,6 @@ snapshots: react-dnd-html5-backend: 16.0.1 react-dom: 19.2.4(react@19.2.4) - '@platejs/find-replace@52.3.10(platejs@52.0.17(@types/react@19.2.14)(immer@10.2.0)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(scheduler@0.27.0)(use-sync-external-store@1.6.0(react@19.2.4)))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': - dependencies: - platejs: 52.0.17(@types/react@19.2.14)(immer@10.2.0)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(scheduler@0.27.0)(use-sync-external-store@1.6.0(react@19.2.4)) - react: 19.2.4 - react-compiler-runtime: 1.0.0(react@19.2.4) - react-dom: 19.2.4(react@19.2.4) - '@platejs/floating@52.0.11(platejs@52.0.17(@types/react@19.2.14)(immer@10.2.0)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(scheduler@0.27.0)(use-sync-external-store@1.6.0(react@19.2.4)))(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': dependencies: '@floating-ui/core': 1.7.4 From f9b5367754c5e07a586b5a318ac06245b3d10846 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Tue, 28 Apr 2026 23:52:37 -0700 Subject: [PATCH 8/8] chore: cleaned comments slop --- surfsense_backend/.env.example | 14 +- .../versions/130_add_agent_action_log.py | 6 +- .../versions/131_add_document_revisions.py | 2 +- .../132_add_agent_permission_rules.py | 9 +- .../app/agents/new_chat/chat_deepagent.py | 125 +++++++++--------- .../app/agents/new_chat/errors.py | 8 +- .../app/agents/new_chat/feature_flags.py | 31 ++--- .../agents/new_chat/middleware/busy_mutex.py | 15 ++- .../agents/new_chat/middleware/compaction.py | 19 +-- .../new_chat/middleware/context_editing.py | 18 +-- .../new_chat/middleware/dedup_tool_calls.py | 10 +- .../agents/new_chat/middleware/doom_loop.py | 22 +-- .../new_chat/middleware/knowledge_search.py | 21 +-- .../new_chat/middleware/noop_injection.py | 28 ++-- .../agents/new_chat/middleware/otel_span.py | 6 +- .../agents/new_chat/middleware/permission.py | 25 ++-- .../agents/new_chat/middleware/retry_after.py | 14 +- .../new_chat/middleware/tool_call_repair.py | 19 +-- .../app/agents/new_chat/permissions.py | 9 +- .../app/agents/new_chat/plugin_loader.py | 9 +- .../new_chat/plugins/year_substituter.py | 10 +- .../app/agents/new_chat/prompts/composer.py | 21 ++- .../app/agents/new_chat/subagents/__init__.py | 15 ++- .../app/agents/new_chat/system_prompt.py | 15 ++- .../app/agents/new_chat/tools/invalid_tool.py | 5 +- .../app/agents/new_chat/tools/registry.py | 6 +- surfsense_backend/app/observability/otel.py | 4 +- .../app/routes/agent_revert_route.py | 8 +- .../agents/new_chat/prompts/test_composer.py | 2 +- .../unit/agents/new_chat/test_otel_span.py | 2 +- .../unit/agents/new_chat/test_permissions.py | 2 +- .../agents/new_chat/test_plugin_loader.py | 2 +- .../tests/unit/observability/test_otel.py | 2 +- .../unit/services/test_revert_service.py | 2 +- 34 files changed, 274 insertions(+), 232 deletions(-) diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index e133a2bc5..c1bfcc538 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -250,12 +250,12 @@ LANGSMITH_PROJECT=surfsense # ============================================================================= -# OPTIONAL: New-chat agent feature flags (OpenCode-port) +# OPTIONAL: New-chat agent feature flags # ============================================================================= # Master kill-switch — when true, every flag below is forced OFF. # SURFSENSE_DISABLE_NEW_AGENT_STACK=false -# Tier 1 — Agent quality +# Agent quality # SURFSENSE_ENABLE_CONTEXT_EDITING=false # SURFSENSE_ENABLE_COMPACTION_V2=false # SURFSENSE_ENABLE_RETRY_AFTER=false @@ -265,24 +265,24 @@ LANGSMITH_PROJECT=surfsense # SURFSENSE_ENABLE_TOOL_CALL_REPAIR=false # SURFSENSE_ENABLE_DOOM_LOOP=false # leave OFF until UI handles permission='doom_loop' -# Tier 2 — Safety +# Safety # SURFSENSE_ENABLE_PERMISSION=false # SURFSENSE_ENABLE_BUSY_MUTEX=false # SURFSENSE_ENABLE_LLM_TOOL_SELECTOR=false # adds a per-turn LLM call -# Tier 3b — Observability (also requires OTEL_EXPORTER_OTLP_ENDPOINT) +# Observability — OTel (also requires OTEL_EXPORTER_OTLP_ENDPOINT) # SURFSENSE_ENABLE_OTEL=false -# Tier 4 — Skills + subagents +# Skills + subagents # SURFSENSE_ENABLE_SKILLS=false # SURFSENSE_ENABLE_SPECIALIZED_SUBAGENTS=false # SURFSENSE_ENABLE_KB_PLANNER_RUNNABLE=false -# Tier 5 — Snapshot / revert +# Snapshot / revert # SURFSENSE_ENABLE_ACTION_LOG=false # SURFSENSE_ENABLE_REVERT_ROUTE=false # Backend-only; flip when UI ships -# Tier 6 — Plugins +# Plugins # SURFSENSE_ENABLE_PLUGIN_LOADER=false # Comma-separated allowlist of plugin entry-point names # SURFSENSE_ALLOWED_PLUGINS=year_substituter diff --git a/surfsense_backend/alembic/versions/130_add_agent_action_log.py b/surfsense_backend/alembic/versions/130_add_agent_action_log.py index 2f06b8ddd..f86a8a3b5 100644 --- a/surfsense_backend/alembic/versions/130_add_agent_action_log.py +++ b/surfsense_backend/alembic/versions/130_add_agent_action_log.py @@ -4,8 +4,10 @@ Revision ID: 130 Revises: 129 Create Date: 2026-04-28 -Tier 5.2 in the OpenCode-port plan. Adds the append-only ``agent_action_log`` -table that :class:`ActionLogMiddleware` writes to after every tool call. +Adds the append-only ``agent_action_log`` table that +:class:`ActionLogMiddleware` writes to after every tool call. Each row +optionally carries a ``reverse_descriptor`` payload used by +``POST /api/threads/{thread_id}/revert/{action_id}`` to undo the action. """ from __future__ import annotations diff --git a/surfsense_backend/alembic/versions/131_add_document_revisions.py b/surfsense_backend/alembic/versions/131_add_document_revisions.py index 46c6991b6..95ce0e032 100644 --- a/surfsense_backend/alembic/versions/131_add_document_revisions.py +++ b/surfsense_backend/alembic/versions/131_add_document_revisions.py @@ -4,7 +4,7 @@ Revision ID: 131 Revises: 130 Create Date: 2026-04-28 -Tier 5.1 in the OpenCode-port plan. Adds two snapshot tables: +Adds two snapshot tables that back the per-action revert flow: * ``document_revisions``: pre-mutation snapshot of NOTE/FILE/EXTENSION docs. * ``folder_revisions``: pre-mutation snapshot of folder mkdir/move/delete. diff --git a/surfsense_backend/alembic/versions/132_add_agent_permission_rules.py b/surfsense_backend/alembic/versions/132_add_agent_permission_rules.py index 0e81eacb5..ff5b52e18 100644 --- a/surfsense_backend/alembic/versions/132_add_agent_permission_rules.py +++ b/surfsense_backend/alembic/versions/132_add_agent_permission_rules.py @@ -4,11 +4,10 @@ Revision ID: 132 Revises: 131 Create Date: 2026-04-28 -Tier 2.1 in the OpenCode-port plan. Adds the persistent ``agent_permission_rules`` -table consumed by :class:`PermissionMiddleware` at agent build time. Rules -can be scoped at search-space (``user_id`` / ``thread_id`` NULL), -user-wide (``user_id`` set, ``thread_id`` NULL), or per-thread -(``thread_id`` set). +Adds the persistent ``agent_permission_rules`` table consumed by +:class:`PermissionMiddleware` at agent build time. Rules can be scoped +at search-space (``user_id`` / ``thread_id`` NULL), user-wide +(``user_id`` set, ``thread_id`` NULL), or per-thread (``thread_id`` set). """ from __future__ import annotations diff --git a/surfsense_backend/app/agents/new_chat/chat_deepagent.py b/surfsense_backend/app/agents/new_chat/chat_deepagent.py index 3ca44dd4f..bfb94ba2d 100644 --- a/surfsense_backend/app/agents/new_chat/chat_deepagent.py +++ b/surfsense_backend/app/agents/new_chat/chat_deepagent.py @@ -353,11 +353,12 @@ async def create_surfsense_deep_agent( additional_tools=list(additional_tools) if additional_tools else None, ) - # Tier 1.6: register `invalid` tool. It is dispatched only when - # ToolCallNameRepairMiddleware rewrites a malformed call. We - # intentionally append it AFTER ``build_tools_async`` so it never - # appears in the system-prompt tool list (which is built from the - # registry, not the bound tool list). + # Register the ``invalid`` tool only when tool-call repair is on. It + # is dispatched only when :class:`ToolCallNameRepairMiddleware` + # rewrites a malformed call. We intentionally append it AFTER + # ``build_tools_async`` so it never appears in the system-prompt + # tool list (which is built from the registry, not the bound tool + # list). _flags: AgentFeatureFlags = get_flags() if _flags.enable_tool_call_repair and INVALID_TOOL_NAME not in { t.name for t in tools @@ -455,10 +456,10 @@ async def create_surfsense_deep_agent( return agent -# Tier 1.1: tools whose output is too costly / lossy to discard. Keep -# this conservative — anything listed here is *never* pruned by -# ContextEditingMiddleware. The list is filtered against actually-bound -# tool names so disabled connectors don't show up here. +# Tools whose output is too costly / lossy to discard. Keep this +# conservative — anything listed here is *never* pruned by +# :class:`ContextEditingMiddleware`. The list is filtered against +# actually-bound tool names so disabled connectors don't show up here. _PRUNE_PROTECTED_TOOL_NAMES: frozenset[str] = frozenset( { "generate_report", @@ -485,11 +486,12 @@ def _safe_exclude_tools(tools: Sequence[BaseTool]) -> tuple[str, ...]: return tuple(name for name in _PRUNE_PROTECTED_TOOL_NAMES if name in enabled) -# Tier 2.1 / cleanup: opencode `Permission.disabled` parity. Replaces the -# legacy binary ``_CONNECTOR_TYPE_TO_SEARCHABLE``-based gating with a -# declarative pass over :data:`BUILTIN_TOOLS`. Each tool that declares a -# ``required_connector`` not present in ``available_connectors`` gets a -# deny rule so any execution attempt short-circuits with permission_denied. +# Connector gating: any tool whose ``ToolDefinition.required_connector`` +# isn't actually wired up gets a synthesized permission deny rule so +# execution attempts short-circuit with ``permission_denied`` instead of +# bubbling up provider-specific 401/404 errors. Mirrors OpenCode's +# ``Permission.disabled`` (declarative, per-tool gating) — replaces the +# legacy binary ``_CONNECTOR_TYPE_TO_SEARCHABLE`` substring-heuristic. def _synthesize_connector_deny_rules( *, available_connectors: list[str] | None, @@ -503,11 +505,6 @@ def _synthesize_connector_deny_rules( 1. It is currently bound (``enabled_tool_names``). 2. It declares a ``required_connector``. 3. That connector is *not* in ``available_connectors``. - - This expresses the OpenCode ``Permission.disabled`` semantics - declaratively, replacing the substring-heuristic binary gating - that used to consult the hardcoded ``_CONNECTOR_TYPE_TO_SEARCHABLE`` - map. """ available = set(available_connectors or []) deny: list[Rule] = [] @@ -581,7 +578,7 @@ def _build_compiled_agent_blocking( "middleware": gp_middleware, } - # Tier 4.3: specialized user-facing subagents (explore, report_writer, + # Specialized user-facing subagents (explore, report_writer, # connector_negotiator). Registered through SubAgentMiddleware alongside # the general-purpose spec so the parent's `task` tool can address them # by name. Off by default until the flag flips so existing deployments @@ -629,14 +626,13 @@ def _build_compiled_agent_blocking( # ``wrap_model_call`` ordering: the FIRST middleware in the list is the # OUTERMOST wrapper. To ensure prune executes before summarization, # place ``SpillingContextEditingMiddleware`` before - # ``SurfSenseCompactionMiddleware`` (Tier 1.1 + 1.3). - # Compaction is the canonical token-budget defense after the - # cleanup tier removed ``SafeSummarizationMiddleware``. The Bedrock - # buffer-empty defense is folded into ``SurfSenseCompactionMiddleware``. + # ``SurfSenseCompactionMiddleware``. Compaction is the canonical + # token-budget defense; the Bedrock buffer-empty defense is folded + # into ``SurfSenseCompactionMiddleware``. summarization_mw = create_surfsense_compaction_middleware(llm, StateBackend) _ = flags.enable_compaction_v2 # historical flag; retained for telemetry parity - # Tier 1.1: ContextEditing prune. Trigger at 55% of model_max_input, + # ContextEditing prune. Trigger at 55% of ``max_input_tokens``, # earlier than summarization (~85%). When disabled, no edit runs. context_edit_mw = None if ( @@ -664,7 +660,10 @@ def _build_compiled_agent_blocking( backend_resolver=backend_resolver, ) - # Tier 1.4 / 1.8 / 1.9 / 1.10: built-in retry/fallback/limits. + # Resilience knobs: header-aware retry, model fallback, and + # per-thread / per-run call-count limits. The fallback / limit + # middlewares are vanilla LangChain primitives; ``RetryAfter`` is + # SurfSense's header-aware variant (see its module docstring). retry_mw = ( RetryAfterMiddleware(max_retries=3) if flags.enable_retry_after and not flags.disable_new_agent_stack @@ -700,14 +699,16 @@ def _build_compiled_agent_blocking( else None ) - # Tier 1.5: provider-compat _noop injection. + # Provider-compat ``_noop`` injection (mirrors OpenCode's + # ``llm.ts`` workaround for providers that reject empty assistant + # turns or alternating-role constraints). noop_mw = ( NoopInjectionMiddleware() if flags.enable_compaction_v2 and not flags.disable_new_agent_stack else None ) - # Tier 1.7: tool-call name repair (lowercase + invalid fallback). + # Tool-call name repair (lowercase + ``invalid`` fallback). # # ``registered_tool_names`` MUST cover every tool the model can legitimately # call. That includes the bound ``tools`` list AND every tool provided by @@ -737,18 +738,22 @@ def _build_compiled_agent_blocking( } repair_mw = ToolCallNameRepairMiddleware( registered_tool_names=registered_names, - fuzzy_match_threshold=None, # opencode parity: no fuzzy step + # Disable fuzzy matching to avoid silent rewrites; the + # lowercase + ``invalid`` fallback alone covers >95% of + # observed model errors. + fuzzy_match_threshold=None, ) - # Tier 1.11: doom-loop detector. Off by default until UI handles. + # Doom-loop detector. Off by default until the frontend handles + # ``permission == "doom_loop"`` interrupts. doom_loop_mw = ( DoomLoopMiddleware(threshold=3) if flags.enable_doom_loop and not flags.disable_new_agent_stack else None ) - # Tier 2.1: PermissionMiddleware. Layers, earliest -> latest (last - # match wins per opencode): + # PermissionMiddleware. Layers, earliest -> latest (last match wins, + # same evaluation order as OpenCode's ``permission/index.ts``): # # 1. ``surfsense_defaults`` — single ``allow */*`` rule. SurfSense # already runs per-tool HITL (see ``tools/hitl.py``) for mutating @@ -778,11 +783,11 @@ def _build_compiled_agent_blocking( ], ) - # Tier 5.2: ActionLogMiddleware. Off by default until the - # ``agent_action_log`` table is migrated. When enabled, persists one - # row per tool call with optional reverse_descriptor for - # /api/threads/{thread_id}/revert/{action_id}. Sits inside permission - # so denied calls aren't logged as completions. + # ActionLogMiddleware. Off by default until the ``agent_action_log`` + # table is migrated. When enabled, persists one row per tool call + # with optional reverse_descriptor for + # ``POST /api/threads/{thread_id}/revert/{action_id}``. Sits inside + # ``permission`` so denied calls aren't logged as completions. action_log_mw: ActionLogMiddleware | None = None if ( flags.enable_action_log @@ -804,23 +809,24 @@ def _build_compiled_agent_blocking( ) action_log_mw = None - # Tier 2.2: per-thread busy mutex. + # Per-thread busy mutex (refuse a second concurrent turn on the same + # thread; see :class:`BusyMutexMiddleware` docstring). busy_mutex_mw: BusyMutexMiddleware | None = ( BusyMutexMiddleware() if flags.enable_busy_mutex and not flags.disable_new_agent_stack else None ) - # Tier 3b: OpenTelemetry spans (model.call + tool.call). Lives just - # inside BusyMutex so it spans every retry/fallback attempt of the - # current turn but never wraps a queued/blocked turn. + # OpenTelemetry spans (model.call + tool.call). Lives just inside + # BusyMutex so it spans every retry/fallback attempt of the current + # turn but never wraps a queued/blocked turn. otel_mw: OtelSpanMiddleware | None = ( OtelSpanMiddleware() if flags.enable_otel and not flags.disable_new_agent_stack else None ) - # Tier 6: plugin entry-point loader. Off by default; opt-in via the + # Plugin entry-point loader. Off by default; opt-in via the # ``SURFSENSE_ENABLE_PLUGIN_LOADER`` flag. The allowlist is read from # the ``SURFSENSE_ALLOWED_PLUGINS`` env var (comma-separated). A future # PR can wire it through ``global_llm_config.yaml``. @@ -845,10 +851,10 @@ def _build_compiled_agent_blocking( ) plugin_middlewares = [] - # Tier 4.1: SkillsMiddleware. Loads built-in + space-authored skills - # via a CompositeBackend. Sources are layered: built-in first, space - # last, so a search-space-authored skill of the same name overrides - # the bundled one. + # SkillsMiddleware (deepagents) loads built-in + space-authored + # skills via a CompositeBackend. Sources are layered: built-in first, + # space last, so a search-space-authored skill of the same name + # overrides the bundled one. skills_mw: SkillsMiddleware | None = None if flags.enable_skills and not flags.disable_new_agent_stack: try: @@ -865,7 +871,8 @@ def _build_compiled_agent_blocking( logging.warning("SkillsMiddleware init failed; skipping: %s", exc) skills_mw = None - # Tier 2.5: LLM-driven tool selection for >30 tools. + # LangChain's LLM-driven tool selection — only enabled for stacks + # large enough to need narrowing (>30 tools). selector_mw: LLMToolSelectorMiddleware | None = None if ( flags.enable_llm_tool_selector @@ -934,12 +941,12 @@ def _build_compiled_agent_blocking( ) if filesystem_mode == FilesystemMode.CLOUD else None, - # Tier 4.1: skill loader. Placed before SubAgentMiddleware so - # subagents inherit the same skill metadata (subagent specs reference - # the same source paths via `default_skills_sources()`). + # Skill loader. Placed before SubAgentMiddleware so subagents + # inherit the same skill metadata (subagent specs reference the + # same source paths via ``default_skills_sources()``). skills_mw, SubAgentMiddleware(backend=StateBackend, subagents=subagent_specs), - # Tier 2.5: tool selection (only when >30 tools and flag on). + # Tool selection (only when >30 tools and flag on). selector_mw, # Defensive caps, then prune, then summarize. model_call_limit_mw, @@ -954,19 +961,19 @@ def _build_compiled_agent_blocking( # Tool-call repair must run after model emits but before # permission / dedup / doom-loop interpret the calls. repair_mw, - # Tier 2.1: deny/ask BEFORE the calls are forwarded to tool nodes. + # Permission deny/ask BEFORE the calls are forwarded to tool nodes. permission_mw, doom_loop_mw, - # Tier 5.2: action log sits inside permission so denied calls - # don't appear as completions, and outside dedup so each unique - # tool invocation gets its own row. + # Action log sits inside permission so denied calls don't appear + # as completions, and outside dedup so each unique tool invocation + # gets its own row. action_log_mw, PatchToolCallsMiddleware(), DedupHITLToolCallsMiddleware(agent_tools=list(tools)), - # Tier 6: plugin slot — sits just before AnthropicCache so plugin-side - # transforms see the final tool result and run before any caching - # heuristics. Multiple plugins in declared order; loader filtered by - # the admin allowlist already. + # Plugin slot — sits just before AnthropicCache so plugin-side + # transforms see the final tool result and run before any + # caching heuristics. Multiple plugins in declared order; loader + # filtered by the admin allowlist already. *plugin_middlewares, AnthropicPromptCachingMiddleware(unsupported_model_behavior="ignore"), ] diff --git a/surfsense_backend/app/agents/new_chat/errors.py b/surfsense_backend/app/agents/new_chat/errors.py index b7bac4536..a17333acc 100644 --- a/surfsense_backend/app/agents/new_chat/errors.py +++ b/surfsense_backend/app/agents/new_chat/errors.py @@ -2,10 +2,10 @@ Typed error taxonomy for the SurfSense agent stack. Used by: -- :class:`RetryAfterMiddleware` (Tier 1.4) — its ``retry_on`` callable - consults the error code to decide whether a retry is appropriate. -- :class:`PermissionMiddleware` (Tier 2.1) — emits - ``code="permission_denied"`` errors when a deny rule trips. +- :class:`RetryAfterMiddleware` — its ``retry_on`` callable consults + the error code to decide whether a retry is appropriate. +- :class:`PermissionMiddleware` — emits ``code="permission_denied"`` + errors when a deny rule trips. - All tools — return :class:`StreamingError` payloads in ``ToolMessage.additional_kwargs["error"]`` so the model and the retry/permission layers share a contract. diff --git a/surfsense_backend/app/agents/new_chat/feature_flags.py b/surfsense_backend/app/agents/new_chat/feature_flags.py index 89c4fb14f..55525abc5 100644 --- a/surfsense_backend/app/agents/new_chat/feature_flags.py +++ b/surfsense_backend/app/agents/new_chat/feature_flags.py @@ -1,9 +1,10 @@ """ Feature flags for the SurfSense new_chat agent stack. -These flags control rollout of OpenCode-pattern middleware ported into -SurfSense. They follow a "default-OFF for risky things, default-ON for -safe upgrades, master kill-switch for everything new" model. +These flags gate the newer agent middleware (some ported from OpenCode, +some sourced from ``langchain.agents.middleware`` / ``deepagents``, some +SurfSense-native). They follow a "default-OFF for risky things, +default-ON for safe upgrades, master kill-switch for everything new" model. All new middleware checks its flag at agent build time. If the master kill-switch ``SURFSENSE_DISABLE_NEW_AGENT_STACK`` is set, every new @@ -57,7 +58,7 @@ class AgentFeatureFlags: # regardless of its env value. Used for rapid rollback. disable_new_agent_stack: bool = False - # Tier 1 — Agent quality + # Agent quality — context budget, retry/limits, name-repair, doom-loop enable_context_editing: bool = False enable_compaction_v2: bool = False enable_retry_after: bool = False @@ -69,26 +70,26 @@ class AgentFeatureFlags: False # Default OFF until UI handles permission='doom_loop' ) - # Tier 2 — Safety + # Safety — permissions, concurrency, tool-set narrowing enable_permission: bool = False # Default OFF for first deploy enable_busy_mutex: bool = False enable_llm_tool_selector: bool = False # Default OFF — adds per-turn LLM cost - # Tier 4 — Skills + subagents + # Skills + subagents enable_skills: bool = False enable_specialized_subagents: bool = False enable_kb_planner_runnable: bool = False - # Tier 5 — Snapshot / revert + # Snapshot / revert enable_action_log: bool = False enable_revert_route: bool = ( False # Backend ships before UI; route returns 503 until this flips ) - # Tier 6 — Plugins + # Plugins enable_plugin_loader: bool = False - # Tier 3b — OTel (orthogonal: also requires OTEL_EXPORTER_OTLP_ENDPOINT) + # Observability — OTel (orthogonal; also requires OTEL_EXPORTER_OTLP_ENDPOINT) enable_otel: bool = False @classmethod @@ -108,7 +109,7 @@ class AgentFeatureFlags: return cls( disable_new_agent_stack=False, - # Tier 1 + # Agent quality enable_context_editing=_env_bool("SURFSENSE_ENABLE_CONTEXT_EDITING", False), enable_compaction_v2=_env_bool("SURFSENSE_ENABLE_COMPACTION_V2", False), enable_retry_after=_env_bool("SURFSENSE_ENABLE_RETRY_AFTER", False), @@ -121,13 +122,13 @@ class AgentFeatureFlags: "SURFSENSE_ENABLE_TOOL_CALL_REPAIR", False ), enable_doom_loop=_env_bool("SURFSENSE_ENABLE_DOOM_LOOP", False), - # Tier 2 + # Safety enable_permission=_env_bool("SURFSENSE_ENABLE_PERMISSION", False), enable_busy_mutex=_env_bool("SURFSENSE_ENABLE_BUSY_MUTEX", False), enable_llm_tool_selector=_env_bool( "SURFSENSE_ENABLE_LLM_TOOL_SELECTOR", False ), - # Tier 4 + # Skills + subagents enable_skills=_env_bool("SURFSENSE_ENABLE_SKILLS", False), enable_specialized_subagents=_env_bool( "SURFSENSE_ENABLE_SPECIALIZED_SUBAGENTS", False @@ -135,12 +136,12 @@ class AgentFeatureFlags: enable_kb_planner_runnable=_env_bool( "SURFSENSE_ENABLE_KB_PLANNER_RUNNABLE", False ), - # Tier 5 + # Snapshot / revert enable_action_log=_env_bool("SURFSENSE_ENABLE_ACTION_LOG", False), enable_revert_route=_env_bool("SURFSENSE_ENABLE_REVERT_ROUTE", False), - # Tier 6 + # Plugins enable_plugin_loader=_env_bool("SURFSENSE_ENABLE_PLUGIN_LOADER", False), - # Tier 3b + # Observability enable_otel=_env_bool("SURFSENSE_ENABLE_OTEL", False), ) diff --git a/surfsense_backend/app/agents/new_chat/middleware/busy_mutex.py b/surfsense_backend/app/agents/new_chat/middleware/busy_mutex.py index 1d95638d0..c57d85004 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/busy_mutex.py +++ b/surfsense_backend/app/agents/new_chat/middleware/busy_mutex.py @@ -1,11 +1,16 @@ """ BusyMutexMiddleware — per-thread asyncio lock + cancel token. -Tier 2.2 in the OpenCode-port plan. Mirrors opencode's -``Stream.scoped(AbortController)`` pattern (single-process, in-memory -lock + cooperative cancellation). For multi-worker deployments a -distributed lock backend (Redis or PostgreSQL advisory locks) is a -phase-2 follow-up. +LangChain has no built-in concept of "this thread is already running a +turn — refuse the second concurrent request". Without it, a user +double-clicking "send" or refreshing the page mid-stream can spawn two +turns racing on the same checkpoint, producing duplicated tool calls +and mangled state. + +Ported from OpenCode's ``Stream.scoped(AbortController)`` pattern: a +single-process, in-memory lock + cooperative cancellation token keyed by +``thread_id``. For multi-worker deployments a distributed lock backend +(Redis or PostgreSQL advisory locks) is a phase-2 follow-up. What this provides: - A ``WeakValueDictionary[str, asyncio.Lock]`` keyed by ``thread_id``; diff --git a/surfsense_backend/app/agents/new_chat/middleware/compaction.py b/surfsense_backend/app/agents/new_chat/middleware/compaction.py index b0a1a7ec5..16361e16b 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/compaction.py +++ b/surfsense_backend/app/agents/new_chat/middleware/compaction.py @@ -5,21 +5,22 @@ Subclasses :class:`deepagents.middleware.summarization.SummarizationMiddleware` to add SurfSense-specific behavior: 1. **Structured summary template** (OpenCode-style ``## Goal / Constraints / - Progress / Key Decisions / Next Steps / Critical Context / Relevant Files``). + Progress / Key Decisions / Next Steps / Critical Context / Relevant Files``) + — see :data:`SURFSENSE_SUMMARY_PROMPT` below. The base + ``SummarizationMiddleware`` only ships a freeform "summarize this" + prompt; the structured template is ported from OpenCode's + ``compaction.ts``. 2. **Protect SurfSense-specific SystemMessages** so injected hints (````, ````, ````, ````, ````, ````, ````) are *not* summarized away and are kept verbatim in the post-summary - message list. + message list. Mirrors OpenCode's ``PRUNE_PROTECTED_TOOLS`` philosophy + (some message types are part of the agent's contract and must survive + compaction unchanged). 3. **Sanitize ``content=None``** when feeding messages into ``get_buffer_string`` (Azure OpenAI / LiteLLM defense — when a provider streams an AIMessage containing only tool_calls and no text, ``content`` can be ``None`` and - ``get_buffer_string`` crashes iterating over ``None``). This used to live in - ``safe_summarization.py``; folded in here. - -This replaces ``app.agents.new_chat.middleware.safe_summarization``. - -Tier 1.3 in the OpenCode-port plan. + ``get_buffer_string`` crashes iterating over ``None``). SurfSense-specific. """ from __future__ import annotations @@ -42,7 +43,7 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -# OpenCode-faithful structured summary template. Mirrors +# Structured summary template ported from OpenCode's # ``opencode/packages/opencode/src/session/compaction.ts:40-75``. Kept as a # module-level constant so unit tests can assert on its sections. SURFSENSE_SUMMARY_PROMPT = """ diff --git a/surfsense_backend/app/agents/new_chat/middleware/context_editing.py b/surfsense_backend/app/agents/new_chat/middleware/context_editing.py index 360e3e28f..39bc57c8b 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/context_editing.py +++ b/surfsense_backend/app/agents/new_chat/middleware/context_editing.py @@ -1,15 +1,15 @@ """ SpillToBackendEdit + SpillingContextEditingMiddleware. -Mirrors OpenCode's spill-to-disk behavior in -``opencode/packages/opencode/src/tool/truncate.ts``. Before -``ClearToolUsesEdit`` rewrites old ``ToolMessage.content`` to a placeholder, -we capture the full original content and write it to the runtime backend -under ``/tool_outputs/{thread_id}/{message_id}.txt``. The placeholder is -upgraded to ``"[cleared — full output at /tool_outputs/.../{id}.txt; ask the -explore subagent to read it]"`` so the agent can recover it on demand. - -Tier 1.2 in the OpenCode-port plan. +LangChain's :class:`ClearToolUsesEdit` discards old ``ToolMessage.content`` +when the context-editing budget triggers, replacing the body with a fixed +placeholder. That's lossy: anything the agent might want to revisit is +gone. The spill-to-disk pattern (originally from OpenCode's +``opencode/packages/opencode/src/tool/truncate.ts``) keeps the prune +behavior but writes the full original payload to the runtime backend +under ``/tool_outputs/{thread_id}/{message_id}.txt`` first. The +placeholder is then upgraded to point at the spill path so the agent +(or a subagent) can read it back on demand. Why this is a middleware subclass instead of a plain ``ContextEdit``: ``ContextEdit.apply`` is sync, but writing to the backend is async. We diff --git a/surfsense_backend/app/agents/new_chat/middleware/dedup_tool_calls.py b/surfsense_backend/app/agents/new_chat/middleware/dedup_tool_calls.py index 3aff524fe..c55347284 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/dedup_tool_calls.py +++ b/surfsense_backend/app/agents/new_chat/middleware/dedup_tool_calls.py @@ -9,11 +9,10 @@ the duplicate call is stripped from the AIMessage that gets checkpointed. That means it is also safe across LangGraph ``interrupt()`` boundaries: the removed call will never appear on graph resume. -Dedup-key resolution order (Tier 2.3 / cleanup in the OpenCode-port plan): +Dedup-key resolution order: 1. :class:`ToolDefinition.dedup_key` — callable provided by the registry - entry. This is the canonical mechanism after the cleanup-tier removal - of the legacy ``PRIMARY_ARG`` map. + entry. This is the canonical mechanism. 2. ``tool.metadata["hitl_dedup_key"]`` — string with a primary arg name; used by MCP / Composio tools whose schemas the registry doesn't see. @@ -72,9 +71,8 @@ class DedupHITLToolCallsMiddleware(AgentMiddleware): # type: ignore[type-arg] The dedup-resolver map is built from two sources, in priority order: 1. ``tool.metadata["dedup_key"]`` — callable provided by the registry's - ``ToolDefinition.dedup_key`` (Tier 2.3). Receives the args dict - and returns a string signature. This is the canonical mechanism - after the cleanup-tier removal of the legacy ``PRIMARY_ARG`` map. + ``ToolDefinition.dedup_key``. Receives the args dict and returns + a string signature. This is the canonical mechanism. 2. ``tool.metadata["hitl_dedup_key"]`` — string with a primary arg name; primarily used by MCP / Composio tools. """ diff --git a/surfsense_backend/app/agents/new_chat/middleware/doom_loop.py b/surfsense_backend/app/agents/new_chat/middleware/doom_loop.py index 1dde87752..850ecd1d2 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/doom_loop.py +++ b/surfsense_backend/app/agents/new_chat/middleware/doom_loop.py @@ -1,17 +1,19 @@ """ DoomLoopMiddleware — pattern-based detector for repeated identical tool calls. -Mirrors ``opencode/packages/opencode/src/session/processor.ts`` doom-loop -behavior. When the same tool with the same arguments is called N times -in a row, the agent has likely entered an infinite loop. We surface this -to the user as an interrupt with ``permission="doom_loop"`` so the UI -can render an "Are you stuck? Continue / cancel?" affordance. +LangChain has :class:`ToolCallLimitMiddleware` which caps the *total* number +of tool calls per turn — but it can't tell apart "10 distinct, useful +calls" from "the same call 10 times in a row". This middleware fills that +gap with a sliding-window check on tool-call signatures, ported from +OpenCode's ``packages/opencode/src/session/processor.ts``. -Tier 1.11 in the OpenCode-port plan. +When the same tool with the same arguments is called N times in a row, +the agent has likely entered an infinite loop. We surface this to the +user as an interrupt with ``permission="doom_loop"`` so the UI can +render an "Are you stuck? Continue / cancel?" affordance. This ships **OFF by default** until the frontend explicitly handles -``context.permission == "doom_loop"`` interrupts (the plan flips -``SURFSENSE_ENABLE_DOOM_LOOP=true`` once the UI is ready). +``context.permission == "doom_loop"`` interrupts. Wire format: uses SurfSense's existing ``interrupt()`` payload shape (see ``app/agents/new_chat/tools/hitl.py``): @@ -69,7 +71,7 @@ class DoomLoopMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, Respon Args: threshold: How many consecutive identical signatures count as a - doom loop. Default 3 (opencode parity). + doom loop. Default 3 (matches OpenCode's processor.ts). """ def __init__(self, *, threshold: int = 3) -> None: @@ -182,7 +184,7 @@ class DoomLoopMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, Respon signatures[-1] if signatures else "", ) - # Tier 3b: interrupt.raised span with permission=doom_loop attribute + # Open an interrupt.raised span with permission=doom_loop attribute # so dashboards can break out doom-loop interrupts from regular # permission asks via the ``interrupt.permission`` attribute. with ot.interrupt_span( diff --git a/surfsense_backend/app/agents/new_chat/middleware/knowledge_search.py b/surfsense_backend/app/agents/new_chat/middleware/knowledge_search.py index 08ca8e18b..0820e8c3e 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/knowledge_search.py +++ b/surfsense_backend/app/agents/new_chat/middleware/knowledge_search.py @@ -592,10 +592,11 @@ class KnowledgePriorityMiddleware(AgentMiddleware): # type: ignore[type-arg] self.available_document_types = available_document_types self.top_k = top_k self.mentioned_document_ids = mentioned_document_ids or [] - # Tier 4.2: build the kb-planner private Runnable ONCE here so we - # don't pay the create_agent compile cost (50-200ms) on every turn. - # Disabled by default behind ``enable_kb_planner_runnable``; when off - # the planner falls back to the legacy ``self.llm.ainvoke`` path. + # Build the kb-planner private Runnable ONCE here so we don't pay + # the ``create_agent`` compile cost (50-200ms) on every turn. + # Disabled by default behind ``enable_kb_planner_runnable``; when + # off the planner falls back to the legacy ``self.llm.ainvoke`` + # path. self._planner: Runnable | None = None self._planner_compile_failed = False @@ -608,9 +609,9 @@ class KnowledgePriorityMiddleware(AgentMiddleware): # type: ignore[type-arg] lazily on first call, then memoized via ``self._planner``. The compiled agent is constructed without tools — the planner's - contract is "answer with structured JSON" — but with ``RetryAfter`` - + the OpenCode-port retry/limit middleware so it shares the parent - agent's resilience guarantees. + contract is "answer with structured JSON" — but it inherits the + :class:`RetryAfterMiddleware` so transient rate-limit errors + from the planner LLM call don't fail the whole turn. """ if self._planner is not None or self._planner_compile_failed: return self._planner @@ -658,9 +659,9 @@ class KnowledgePriorityMiddleware(AgentMiddleware): # type: ignore[type-arg] loop = asyncio.get_running_loop() t0 = loop.time() - # Tier 4.2: prefer the compiled-once planner Runnable when enabled; - # otherwise fall back to ``self.llm.ainvoke``. The ``surfsense:internal`` - # tag is preserved on both paths so ``_stream_agent_events`` still + # Prefer the compiled-once planner Runnable when enabled; otherwise + # fall back to ``self.llm.ainvoke``. The ``surfsense:internal`` tag + # is preserved on both paths so ``_stream_agent_events`` still # suppresses the planner's intermediate events from the UI. planner = self._build_kb_planner_runnable() try: diff --git a/surfsense_backend/app/agents/new_chat/middleware/noop_injection.py b/surfsense_backend/app/agents/new_chat/middleware/noop_injection.py index 8628479c7..503c73ccc 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/noop_injection.py +++ b/surfsense_backend/app/agents/new_chat/middleware/noop_injection.py @@ -1,18 +1,23 @@ """ ``_noop`` provider-compatibility tool + injection middleware. -OpenCode injects a ``_noop`` tool for LiteLLM/Bedrock/Copilot when the -model call has empty tools but message history includes prior -``tool_calls`` — some providers 400 in that shape (see -``opencode/packages/opencode/src/session/llm.ts:209-228``). SurfSense uses -LiteLLM, and the compaction summarize call (no tools, history full of -tool calls) hits this. Tier 1.5 in the OpenCode-port plan. +Some providers (LiteLLM, Bedrock, Copilot) 400 when a model call has +empty ``tools`` but the message history includes prior ``tool_calls`` — +they treat that shape as malformed even though it's perfectly valid +LangChain. SurfSense hits this on the compaction summarize call (no +tools, history full of tool calls). + +Ported from OpenCode's ``packages/opencode/src/session/llm.ts:209-228``, +which discovered and codified the workaround: inject a no-op tool *only* +on those provider shapes so the request validates without ever being +called. Operation: a :class:`NoopInjectionMiddleware` ``wrap_model_call`` checks if the request has zero tools but the last AI message in history includes -``tool_calls``. If yes, it injects the ``_noop`` tool only — never globally, -mirroring opencode's gating exactly. The :func:`noop_tool` returns empty -content when called (which it should never be in practice). +``tool_calls``. If yes, it injects the ``_noop`` tool only — never +globally — mirroring OpenCode's gating exactly. The :func:`noop_tool` +returns empty content when called (which it should never be in +practice). """ from __future__ import annotations @@ -45,8 +50,9 @@ def noop_tool() -> str: # Provider markers that benefit from ``_noop`` injection. These match -# opencode's gating list. We also accept any string containing one of -# these substrings (so e.g. ``litellm`` matches ``ChatLiteLLM``). +# OpenCode's gating list (``llm.ts:209-228``). We also accept any string +# containing one of these substrings so e.g. ``litellm`` matches +# ``ChatLiteLLM``. _NOOP_NEEDED_PROVIDERS: tuple[str, ...] = ( "litellm", "bedrock", diff --git a/surfsense_backend/app/agents/new_chat/middleware/otel_span.py b/surfsense_backend/app/agents/new_chat/middleware/otel_span.py index f51d2f7bb..cfe1edae4 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/otel_span.py +++ b/surfsense_backend/app/agents/new_chat/middleware/otel_span.py @@ -3,14 +3,14 @@ OpenTelemetry span middleware for the SurfSense ``new_chat`` agent. Wraps both ``model.call`` (LLM invocations) and ``tool.call`` (tool executions) with OTel spans, attaching low-cardinality span names and -high-cardinality identifiers as attributes (per the Tier 3b plan). +high-cardinality identifiers as attributes. This middleware is intentionally a thin adapter over :mod:`app.observability.otel`; when OTel is not configured all spans collapse to no-ops and the wrapper adds <1µs overhead per call. When OTel **is** configured (``OTEL_EXPORTER_OTLP_ENDPOINT`` set), every -model and tool call gets a span with the standard attributes the -plan's dashboards expect. +model and tool call gets a span with the standard attributes our +dashboards expect. """ from __future__ import annotations diff --git a/surfsense_backend/app/agents/new_chat/middleware/permission.py b/surfsense_backend/app/agents/new_chat/middleware/permission.py index 6e1f42baf..37719e96a 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/permission.py +++ b/surfsense_backend/app/agents/new_chat/middleware/permission.py @@ -1,10 +1,15 @@ """ PermissionMiddleware — pattern-based allow/deny/ask with HITL fallback. -Mirrors ``opencode/packages/opencode/src/permission/index.ts`` but uses -SurfSense's existing ``interrupt({type, action, context})`` payload shape -(see ``app/agents/new_chat/tools/hitl.py``) so the frontend keeps -working unchanged. Tier 2.1 in the OpenCode-port plan. +LangChain's :class:`HumanInTheLoopMiddleware` only supports a static +"this tool always asks" decision per tool. There's no rule-based +allow/deny/ask layered ruleset, no glob patterns, no per-search-space or +per-thread overrides, and no auto-deny synthesis. + +This middleware ports OpenCode's ``packages/opencode/src/permission/index.ts`` +ruleset model on top of SurfSense's existing ``interrupt({type, action, +context})`` payload shape (see ``app/agents/new_chat/tools/hitl.py``) so +the frontend keeps working unchanged. Operation: 1. ``aafter_model`` inspects the latest ``AIMessage.tool_calls``. @@ -24,9 +29,9 @@ Operation: The middleware also performs a *pre-model* tool-filter step (the ``before_model`` hook) so globally denied tools are stripped from the -exposed tool list before the model gets to see them. This is -opencode's ``Permission.disabled`` equivalent and dramatically reduces -the chance the model emits a deny-only call. +exposed tool list before the model gets to see them. This mirrors +OpenCode's ``Permission.disabled`` and dramatically reduces the chance +the model emits a deny-only call. """ from __future__ import annotations @@ -117,7 +122,7 @@ class PermissionMiddleware(AgentMiddleware): # type: ignore[type-arg] self._emit_interrupt = always_emit_interrupt_payload # ------------------------------------------------------------------ - # Tool-filter step (opencode `Permission.disabled` equivalent) + # Tool-filter step (mirrors OpenCode's ``Permission.disabled``) # ------------------------------------------------------------------ def _globally_denied(self, tool_name: str) -> bool: @@ -197,8 +202,8 @@ class PermissionMiddleware(AgentMiddleware): # type: ignore[type-arg] "always": patterns, }, } - # Tier 3b: permission.asked + interrupt.raised spans (no-op when - # OTel is disabled). Both fire here so dashboards can correlate + # Open ``permission.asked`` + ``interrupt.raised`` OTel spans + # (no-op when OTel is disabled) so dashboards can correlate # "we asked X" with "interrupt was actually delivered". with ( ot.permission_asked_span( diff --git a/surfsense_backend/app/agents/new_chat/middleware/retry_after.py b/surfsense_backend/app/agents/new_chat/middleware/retry_after.py index 394bb0371..0c3d3d017 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/retry_after.py +++ b/surfsense_backend/app/agents/new_chat/middleware/retry_after.py @@ -1,10 +1,16 @@ """ RetryAfterMiddleware — Header-aware retry with custom backoff and SSE eventing. -Why standalone instead of subclassing ``ModelRetryMiddleware``: the upstream -class calls module-level ``calculate_delay`` inline (no overridable -``_calculate_delay`` hook), so a subclass cannot inject Retry-After header -delays without rewriting the loop. Tier 1.4 in the OpenCode-port plan. +LangChain's :class:`ModelRetryMiddleware` retries on exceptions but ignores +the ``Retry-After`` HTTP header — it just runs its own exponential backoff. +That wastes time when a provider has explicitly told us how long to wait. +This middleware honors the header (mirroring OpenCode's +``packages/opencode/src/session/llm.ts`` retry pathway) and emits an SSE +event so the UI can show "rate-limited, retrying in Ns". + +We can't subclass ``ModelRetryMiddleware`` cleanly because its loop calls a +module-level ``calculate_delay`` inline (no overridable +``_calculate_delay`` hook), so this is a standalone implementation. Behaviour: - Extracts ``Retry-After`` / ``retry-after-ms`` from diff --git a/surfsense_backend/app/agents/new_chat/middleware/tool_call_repair.py b/surfsense_backend/app/agents/new_chat/middleware/tool_call_repair.py index 54df0cc60..9f81a168b 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/tool_call_repair.py +++ b/surfsense_backend/app/agents/new_chat/middleware/tool_call_repair.py @@ -1,10 +1,6 @@ """ ToolCallNameRepairMiddleware — two-stage tool-name repair. -Mirrors ``opencode/packages/opencode/src/session/llm.ts:339-358`` plus -``opencode/packages/opencode/src/tool/invalid.ts``. Tier 1.7 in the -OpenCode-port plan. - Operation: 1. **Stage 1 — lowercase repair:** if a tool call's ``name`` is not in the registry but ``name.lower()`` is, rewrite in place. Catches @@ -14,9 +10,13 @@ Operation: so the registered :func:`invalid_tool` returns the error to the model for self-correction. -Distinct from :class:`deepagents.middleware.PatchToolCallsMiddleware`, -which patches *dangling* tool calls (no matching ToolMessage) — that -class does not handle the wrong-name case at all. +Ported from OpenCode's ``packages/opencode/src/session/llm.ts:339-358`` ++ ``packages/opencode/src/tool/invalid.ts``. LangChain has no equivalent: +:class:`deepagents.middleware.PatchToolCallsMiddleware` patches +*dangling* tool calls (no matching ToolMessage) but does nothing about +wrong names, and the model framework's default behavior on an unknown +name is to crash the turn rather than route to a self-correction +fallback. """ from __future__ import annotations @@ -61,7 +61,8 @@ class ToolCallNameRepairMiddleware( ``invalid`` should be in this set so the fallback dispatches. fuzzy_match_threshold: Optional ``difflib`` ratio (0-1) for the fuzzy-match step that runs *between* lowercase and invalid. - Set to ``None`` to disable fuzzy matching (opencode parity). + Set to ``None`` to disable fuzzy matching (default in + OpenCode; we mirror that to avoid silent rewrites). """ def __init__( @@ -106,7 +107,7 @@ class ToolCallNameRepairMiddleware( call["response_metadata"] = metadata return call - # Optional fuzzy step (off by default for opencode parity) + # Optional fuzzy step (off by default — see class docstring) if self._fuzzy_threshold is not None: close = difflib.get_close_matches( name, registered, n=1, cutoff=self._fuzzy_threshold diff --git a/surfsense_backend/app/agents/new_chat/permissions.py b/surfsense_backend/app/agents/new_chat/permissions.py index 50a0cfbdc..523deb11f 100644 --- a/surfsense_backend/app/agents/new_chat/permissions.py +++ b/surfsense_backend/app/agents/new_chat/permissions.py @@ -1,21 +1,20 @@ """ Wildcard pattern matching + rule evaluation for the SurfSense permission system. -Mirrors ``opencode/packages/opencode/src/permission/evaluate.ts`` and -``opencode/packages/opencode/src/util/wildcard.ts`` precisely: +Ported from OpenCode's ``packages/opencode/src/permission/evaluate.ts`` and +``packages/opencode/src/util/wildcard.ts``. LangChain has no rule-based +permission evaluator, so we keep OpenCode's semantics intact: - ``Wildcard.match`` matches both the ``permission`` and the ``pattern`` fields of a rule against the requested ``(permission, pattern)`` pair. ``*`` matches any segment, ``**`` matches across separators. - The evaluator runs ``findLast`` over the **flattened** list of rules from all rulesets — last matching rule wins. -- The default fallback is ``ask`` (NOT deny), matching opencode. +- The default fallback is ``ask`` (NOT deny), matching OpenCode. - Multi-pattern requests AND together: if ANY pattern resolves to ``deny``, the whole request is denied; if ANY needs ``ask``, an interrupt is raised; only when all patterns ``allow`` does the request proceed. - -Tier 2.1 in the OpenCode-port plan. """ from __future__ import annotations diff --git a/surfsense_backend/app/agents/new_chat/plugin_loader.py b/surfsense_backend/app/agents/new_chat/plugin_loader.py index 426e28041..c52620d40 100644 --- a/surfsense_backend/app/agents/new_chat/plugin_loader.py +++ b/surfsense_backend/app/agents/new_chat/plugin_loader.py @@ -1,9 +1,10 @@ """Entry-point based plugin loader for SurfSense agent middleware. -The realization in the Tier 6 plan: LangChain's :class:`AgentMiddleware` ABC -already covers the practical surface most plugins need (``before_agent`` / -``before_model`` / ``wrap_tool_call`` / their async counterparts), so a -SurfSense-specific plugin protocol is unnecessary. +LangChain's :class:`AgentMiddleware` ABC already covers the practical +surface most plugins need (``before_agent`` / ``before_model`` / +``wrap_tool_call`` / their async counterparts), so a SurfSense-specific +plugin protocol would be redundant. We just need a way to discover and +admit third-party middleware safely. A plugin is therefore just an installable Python package that registers a factory callable under the ``surfsense.plugins`` entry-point group: diff --git a/surfsense_backend/app/agents/new_chat/plugins/year_substituter.py b/surfsense_backend/app/agents/new_chat/plugins/year_substituter.py index 3e2e631d2..2b7781b90 100644 --- a/surfsense_backend/app/agents/new_chat/plugins/year_substituter.py +++ b/surfsense_backend/app/agents/new_chat/plugins/year_substituter.py @@ -1,10 +1,10 @@ """Reference plugin: substitute ``{{year}}`` in tool descriptions. -Mirrors the OpenCode ``chat.system.transform`` example. Demonstrates the -:meth:`AgentMiddleware.awrap_tool_call` hook -- the plugin sees every tool -invocation and can rewrite the request *or* the result. This particular -plugin is read-only and only transforms the *description* the user might -see in error messages (no request mutation). +Demonstrates the :meth:`AgentMiddleware.awrap_tool_call` hook -- the +plugin sees every tool invocation and can rewrite the request *or* the +result. This particular plugin is read-only and only transforms the +*description* the user might see in error messages (no request +mutation). The plugin is built as a factory function so the entry-point loader can inject :class:`PluginContext` (containing the agent's LLM, search-space diff --git a/surfsense_backend/app/agents/new_chat/prompts/composer.py b/surfsense_backend/app/agents/new_chat/prompts/composer.py index 77b86aeef..42f8303e6 100644 --- a/surfsense_backend/app/agents/new_chat/prompts/composer.py +++ b/surfsense_backend/app/agents/new_chat/prompts/composer.py @@ -14,7 +14,13 @@ under :mod:`app.agents.new_chat.prompts`. It replaces the monolithic examples/ # one ``.md`` per tool with call examples routing/ # connector-specific routing notes (linear, slack, …) -Tier 3a in the OpenCode-port plan. +The model-family dispatch step (see :func:`detect_provider_variant`) +mirrors OpenCode's ``packages/opencode/src/session/system.ts`` — different +model families respond best to differently-styled prompts (Claude likes +XML/narrative, GPT-5 wants channel-aware pragmatic, Codex needs +terse/file:line, Gemini wants formal numbered steps, etc.). LangChain's +``dynamic_prompt`` helper supports per-call prompt swaps but ships no +out-of-the-box family classifier, so we keep our own. Backwards compatibility ======================= @@ -42,10 +48,11 @@ from app.db import ChatVisibility # When adding a new variant, also drop a matching ``providers/.md`` # file in this package and (if appropriate) extend the regex matchers below. # -# Stylistic clusters mirror OpenCode's prompt-per-family layout but adapted -# to SurfSense's "supplemental hints" architecture (each fragment is a -# focused style nudge, NOT a full system prompt — the main prompt is -# already assembled from base/ + tools/ + routing/). +# Stylistic clusters: each variant is a focused style nudge, NOT a full +# system prompt — the main prompt is already assembled from base/ + +# tools/ + routing/. The clustering itself (which models map to which +# style) follows OpenCode's ``system.ts`` family table; see the module +# docstring for credits. ProviderVariant = str # Known values: # "anthropic" — Claude family (XML-friendly, narrative todos) @@ -82,8 +89,8 @@ def detect_provider_variant(model_name: str | None) -> ProviderVariant: Order is significant: more-specific patterns are tried first so ``gpt-5-codex`` routes to ``"openai_codex"`` rather than - ``"openai_reasoning"`` (mirrors OpenCode's - ``packages/opencode/src/session/system.ts`` dispatch). + ``"openai_reasoning"`` — same dispatch order as OpenCode's + ``packages/opencode/src/session/system.ts``. """ if not model_name: return "default" diff --git a/surfsense_backend/app/agents/new_chat/subagents/__init__.py b/surfsense_backend/app/agents/new_chat/subagents/__init__.py index b9f21a0d2..7d678ec79 100644 --- a/surfsense_backend/app/agents/new_chat/subagents/__init__.py +++ b/surfsense_backend/app/agents/new_chat/subagents/__init__.py @@ -1,14 +1,17 @@ """Specialized user-facing subagents for the SurfSense agent. -Each subagent is a :class:`deepagents.SubAgent` typed-dict spec passed to -:class:`deepagents.SubAgentMiddleware`, which materializes them as ephemeral -runnables invoked via the ``task`` tool. +The :class:`deepagents.SubAgentMiddleware` already provides the +materialization machinery (each :class:`deepagents.SubAgent` typed-dict +spec is compiled into an ephemeral runnable invoked via the ``task`` +tool); what's specific to SurfSense is the *seeding* of those subagents +with declarative deny rules. Per-subagent permission rules are injected as a :class:`PermissionMiddleware` entry inside the subagent's ``middleware`` -field, mirroring opencode ``tool/task.ts`` which seeds child sessions with -deny rules for tools the parent does not want them touching (e.g. -``task``/``todowrite`` recursion, write tools for read-only research roles). +field. The auto-deny pattern (e.g. forbid ``task``/``todowrite`` +recursion, block write tools for read-only research roles) is borrowed +from OpenCode's ``packages/opencode/src/tool/task.ts``, which has +analogous logic for restricting child sessions. """ from .config import ( diff --git a/surfsense_backend/app/agents/new_chat/system_prompt.py b/surfsense_backend/app/agents/new_chat/system_prompt.py index 3919527d9..56f838d7e 100644 --- a/surfsense_backend/app/agents/new_chat/system_prompt.py +++ b/surfsense_backend/app/agents/new_chat/system_prompt.py @@ -1,13 +1,14 @@ """ Thin compatibility wrapper around :mod:`app.agents.new_chat.prompts.composer`. -Tier 3a of the OpenCode-port plan replaced the monolithic prompt strings -in this module with a fragment tree under ``prompts/`` and a composer -function. This module preserves the public function surface -(``build_surfsense_system_prompt`` / ``build_configurable_system_prompt`` / -``get_default_system_instructions`` / ``SURFSENSE_SYSTEM_PROMPT``) so that -existing call sites — `chat_deepagent.py`, anonymous chat routes, and the -configurable-prompt admin path — keep working without churn. +The composer split the previous monolithic prompt string into a fragment +tree under ``prompts/`` plus a model-family dispatch step (see the +composer module docstring for credits). This module preserves the public +function surface (``build_surfsense_system_prompt`` / +``build_configurable_system_prompt`` / +``get_default_system_instructions`` / ``SURFSENSE_SYSTEM_PROMPT``) so +that existing call sites — `chat_deepagent.py`, anonymous chat routes, +and the configurable-prompt admin path — keep working without churn. For new call sites prefer importing ``compose_system_prompt`` directly from :mod:`app.agents.new_chat.prompts.composer`. diff --git a/surfsense_backend/app/agents/new_chat/tools/invalid_tool.py b/surfsense_backend/app/agents/new_chat/tools/invalid_tool.py index df10fcbe3..ea4bc0bc1 100644 --- a/surfsense_backend/app/agents/new_chat/tools/invalid_tool.py +++ b/surfsense_backend/app/agents/new_chat/tools/invalid_tool.py @@ -6,8 +6,9 @@ tool, :class:`ToolCallNameRepairMiddleware` rewrites the call to ``invalid`` with the original name and a parser/validation error string. This tool's execution then returns that error to the model so it can self-correct. -Mirrors ``opencode/packages/opencode/src/tool/invalid.ts``. Tier 1.6 in -the OpenCode-port plan. +Ported from OpenCode's ``packages/opencode/src/tool/invalid.ts`` — +LangChain has no equivalent fallback path; the default behavior on an +unknown tool name is a hard ``ToolNotFoundError`` which kills the turn. Critically, the :class:`ToolDefinition` for this tool is **excluded** from the system-prompt tool list and from ``LLMToolSelectorMiddleware`` selection diff --git a/surfsense_backend/app/agents/new_chat/tools/registry.py b/surfsense_backend/app/agents/new_chat/tools/registry.py index fce1bf872..e8bab36fd 100644 --- a/surfsense_backend/app/agents/new_chat/tools/registry.py +++ b/surfsense_backend/app/agents/new_chat/tools/registry.py @@ -132,12 +132,10 @@ class ToolDefinition: that must be in ``available_connectors`` for the tool to be enabled. dedup_key: Optional callable that maps a tool's ``args`` dict to a string signature used by :class:`DedupHITLToolCallsMiddleware` - to drop duplicate calls. Replaces the legacy hardcoded - ``_NATIVE_HITL_TOOL_DEDUP_KEYS`` map (Tier 2.3 in the - OpenCode-port plan). + to drop duplicate calls within a single LLM response. reverse: Optional callable that, given the tool's ``(args, result)``, returns a ``ReverseDescriptor`` describing the inverse tool - invocation. Consumed by the snapshot/revert pipeline (Tier 5). + invocation. Consumed by the snapshot/revert pipeline. """ diff --git a/surfsense_backend/app/observability/otel.py b/surfsense_backend/app/observability/otel.py index 4f2257ab7..6791ab499 100644 --- a/surfsense_backend/app/observability/otel.py +++ b/surfsense_backend/app/observability/otel.py @@ -1,12 +1,10 @@ """ OpenTelemetry instrumentation helpers for the SurfSense agent stack. -Tier 3b in the OpenCode-port plan. - Goals ===== -- Provide one tiny, ergonomic API for the spans listed in the plan +- Provide one tiny, ergonomic API for the spans we care about (``tool.call``, ``model.call``, ``kb.search``, ``kb.persist``, ``compaction.run``, ``interrupt.raised``, ``permission.asked``). - Keep span **names** low-cardinality (``tool.call`` rather than diff --git a/surfsense_backend/app/routes/agent_revert_route.py b/surfsense_backend/app/routes/agent_revert_route.py index cbe4e7417..12484ff53 100644 --- a/surfsense_backend/app/routes/agent_revert_route.py +++ b/surfsense_backend/app/routes/agent_revert_route.py @@ -1,9 +1,9 @@ """POST ``/api/threads/{thread_id}/revert/{action_id}``: undo an agent action. -Per the Tier 5 plan, the route ships **before** the UI lights up the per-message -"Undo from here" affordance. To prevent accidental usage during the gap we -return ``503 Service Unavailable`` until the -``SURFSENSE_ENABLE_REVERT_ROUTE`` flag flips. Once enabled, the route runs: +The route ships **before** the UI lights up the per-message "Undo from +here" affordance. To prevent accidental usage during the gap we return +``503 Service Unavailable`` until the ``SURFSENSE_ENABLE_REVERT_ROUTE`` +flag flips. Once enabled, the route runs: 1. Authentication via :func:`current_active_user`. 2. Action lookup; 404 if the action does not belong to the thread. diff --git a/surfsense_backend/tests/unit/agents/new_chat/prompts/test_composer.py b/surfsense_backend/tests/unit/agents/new_chat/prompts/test_composer.py index aa0c215b9..397b1c787 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/prompts/test_composer.py +++ b/surfsense_backend/tests/unit/agents/new_chat/prompts/test_composer.py @@ -1,4 +1,4 @@ -"""Tests for the prompt fragment composer (Tier 3a).""" +"""Tests for the prompt fragment composer.""" from __future__ import annotations diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_otel_span.py b/surfsense_backend/tests/unit/agents/new_chat/test_otel_span.py index e5b171612..55434c04d 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/test_otel_span.py +++ b/surfsense_backend/tests/unit/agents/new_chat/test_otel_span.py @@ -1,4 +1,4 @@ -"""Tests for the OtelSpanMiddleware adapter (Tier 3b).""" +"""Tests for the OtelSpanMiddleware adapter.""" from __future__ import annotations diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_permissions.py b/surfsense_backend/tests/unit/agents/new_chat/test_permissions.py index 4924f2aee..8ec16617a 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/test_permissions.py +++ b/surfsense_backend/tests/unit/agents/new_chat/test_permissions.py @@ -1,4 +1,4 @@ -"""Tests for the wildcard matcher and rule evaluator (opencode evaluate.ts parity).""" +"""Tests for the wildcard matcher and rule evaluator (parity with OpenCode evaluate.ts).""" from __future__ import annotations diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_plugin_loader.py b/surfsense_backend/tests/unit/agents/new_chat/test_plugin_loader.py index c2118c697..5dbf765a7 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/test_plugin_loader.py +++ b/surfsense_backend/tests/unit/agents/new_chat/test_plugin_loader.py @@ -1,4 +1,4 @@ -"""Unit tests for the SurfSense plugin entry-point loader (Tier 6).""" +"""Unit tests for the SurfSense plugin entry-point loader.""" from __future__ import annotations diff --git a/surfsense_backend/tests/unit/observability/test_otel.py b/surfsense_backend/tests/unit/observability/test_otel.py index 583142098..fc5813973 100644 --- a/surfsense_backend/tests/unit/observability/test_otel.py +++ b/surfsense_backend/tests/unit/observability/test_otel.py @@ -1,4 +1,4 @@ -"""Tests for the SurfSense OpenTelemetry shim (Tier 3b).""" +"""Tests for the SurfSense OpenTelemetry shim.""" from __future__ import annotations diff --git a/surfsense_backend/tests/unit/services/test_revert_service.py b/surfsense_backend/tests/unit/services/test_revert_service.py index e2cbe383a..a81e52041 100644 --- a/surfsense_backend/tests/unit/services/test_revert_service.py +++ b/surfsense_backend/tests/unit/services/test_revert_service.py @@ -1,4 +1,4 @@ -"""Unit tests for the agent revert service (Tier 5.3).""" +"""Unit tests for the agent revert service.""" from __future__ import annotations