Merge remote-tracking branch 'upstream/dev' into feat/ui-revamp

surfsense_backend/app/agents/new_chat/filesystem_state.py

@@ -17,6 +17,7 @@ extra fields needed to implement Postgres-backed virtual filesystem semantics:
 * ``kb_matched_chunk_ids`` — internal hand-off for matched-chunk highlighting.
 * ``kb_anon_doc`` — Redis-loaded anonymous document (if any).
 * ``tree_version`` — bumped by persistence; invalidates the tree render cache.
+* ``workspace_tree_text`` — pre-rendered ``<workspace_tree>`` body for the turn.
 
 Tools mutate these fields ONLY via ``Command(update=...)`` returns; the
 reducers in :mod:`app.agents.new_chat.state_reducers` handle merging.
@@ -168,6 +169,9 @@ class SurfSenseFilesystemState(FilesystemState):
     tree_version: NotRequired[Annotated[int, _replace_reducer]]
     """Monotonically increasing counter; bumped when commits change the KB tree."""
 
+    workspace_tree_text: NotRequired[Annotated[str, _replace_reducer]]
+    """Pre-rendered ``<workspace_tree>`` body; shared with subagents to skip re-render."""
+
 
 __all__ = [
     "KbAnonDoc",

surfsense_backend/app/agents/new_chat/middleware/knowledge_search.py

@@ -585,6 +585,7 @@ class KnowledgePriorityMiddleware(AgentMiddleware):  # type: ignore[type-arg]
         available_document_types: list[str] | None = None,
         top_k: int = 10,
         mentioned_document_ids: list[int] | None = None,
+        inject_system_message: bool = True,  # For backwards compatibility
     ) -> None:
         self.llm = llm
         self.search_space_id = search_space_id
@@ -593,6 +594,7 @@ class KnowledgePriorityMiddleware(AgentMiddleware):  # type: ignore[type-arg]
         self.available_document_types = available_document_types
         self.top_k = top_k
         self.mentioned_document_ids = mentioned_document_ids or []
+        self.inject_system_message = inject_system_message
         # Build the kb-planner private Runnable ONCE here so we don't pay
         # the ``create_agent`` compile cost (50-200ms) on every turn.
         # Disabled by default behind ``enable_kb_planner_runnable``; when
@@ -773,14 +775,16 @@ class KnowledgePriorityMiddleware(AgentMiddleware):  # type: ignore[type-arg]
                 "mentioned": True,
             }
         ]
-        new_messages = list(state.get("messages") or [])
-        insert_at = max(len(new_messages) - 1, 0)
-        new_messages.insert(insert_at, _render_priority_message(priority))
-        return {
+        update: dict[str, Any] = {
             "kb_priority": priority,
             "kb_matched_chunk_ids": {},
-            "messages": new_messages,
         }
+        if self.inject_system_message:
+            new_messages = list(state.get("messages") or [])
+            insert_at = max(len(new_messages) - 1, 0)
+            new_messages.insert(insert_at, _render_priority_message(priority))
+            update["messages"] = new_messages
+        return update
 
     async def _authenticated_priority(
         self,
@@ -897,10 +901,6 @@ class KnowledgePriorityMiddleware(AgentMiddleware):  # type: ignore[type-arg]
             folder_entries = await self._materialize_folder_priority(folder_mention_ids)
             priority = folder_entries + priority
 
-        new_messages = list(messages)
-        insert_at = max(len(new_messages) - 1, 0)
-        new_messages.insert(insert_at, _render_priority_message(priority))
-
         _perf_log.info(
             "[kb_priority] completed in %.3fs query=%r priority=%d mentioned=%d folders=%d",
             asyncio.get_event_loop().time() - t0,
@@ -910,11 +910,16 @@ class KnowledgePriorityMiddleware(AgentMiddleware):  # type: ignore[type-arg]
             len(folder_mention_ids),
         )
 
-        return {
+        update: dict[str, Any] = {
             "kb_priority": priority,
             "kb_matched_chunk_ids": matched_chunk_ids,
-            "messages": new_messages,
         }
+        if self.inject_system_message:
+            new_messages = list(messages)
+            insert_at = max(len(new_messages) - 1, 0)
+            new_messages.insert(insert_at, _render_priority_message(priority))
+            update["messages"] = new_messages
+        return update
 
     async def _materialize_folder_priority(
         self, folder_ids: list[int]

surfsense_backend/app/agents/new_chat/middleware/knowledge_tree.py

@@ -105,12 +105,14 @@ class KnowledgeTreeMiddleware(AgentMiddleware):  # type: ignore[type-arg]
         llm: BaseChatModel | None = None,
         max_entries: int = MAX_TREE_ENTRIES,
         max_tokens: int = MAX_TREE_TOKENS,
+        inject_system_message: bool = True,  # For backwards compatibility
     ) -> None:
         self.search_space_id = search_space_id
         self.filesystem_mode = filesystem_mode
         self.llm = llm
         self.max_entries = max_entries
         self.max_tokens = max_tokens
+        self.inject_system_message = inject_system_message
         self._cache: dict[tuple[int, int, bool], str] = {}
 
     async def abefore_agent(  # type: ignore[override]
@@ -132,10 +134,13 @@ class KnowledgeTreeMiddleware(AgentMiddleware):  # type: ignore[type-arg]
         else:
             tree_msg = await self._render_kb_tree(state)
 
-        messages = list(state.get("messages") or [])
-        insert_at = max(len(messages) - 1, 0)
-        messages.insert(insert_at, SystemMessage(content=tree_msg))
-        update["messages"] = messages
+        update["workspace_tree_text"] = tree_msg
+
+        if self.inject_system_message:
+            messages = list(state.get("messages") or [])
+            insert_at = max(len(messages) - 1, 0)
+            messages.insert(insert_at, SystemMessage(content=tree_msg))
+            update["messages"] = messages
         return update
 
     def before_agent(  # type: ignore[override]

surfsense_backend/app/agents/new_chat/middleware/permission.py

@@ -23,7 +23,7 @@ Operation:
    SurfSense shape and LangChain HITL ``{"decisions": [{"type": ...}]}``
    replies are accepted via :func:`_normalize_permission_decision`.
    - ``once``: proceed.
-   - ``always``: also persist allow rules for ``request.always`` patterns.
+   - ``approve_always``: also persist allow rules for ``request.always`` patterns.
    - ``reject`` w/o feedback: raise :class:`RejectedError`.
    - ``reject`` w/  feedback: raise :class:`CorrectedError`.
 5. On ``allow``: proceed unchanged.
@@ -90,6 +90,7 @@ _LC_TYPE_TO_PERMISSION_DECISION: dict[str, str] = {
     "approve": "once",
     "reject": "reject",
     "edit": "once",
+    "approve_always": "approve_always",
 }
 
 
@@ -130,7 +131,7 @@ def _normalize_permission_decision(decision: Any) -> dict[str, Any]:
     mapped = _LC_TYPE_TO_PERMISSION_DECISION.get(raw_type)
     if mapped is None:
         # Tolerate legacy values arriving without ``decision_type`` wrapping.
-        if raw_type in {"once", "always", "reject"}:
+        if raw_type in {"once", "approve_always", "reject"}:
             mapped = raw_type
         else:
             logger.warning(
@@ -162,8 +163,8 @@ class PermissionMiddleware(AgentMiddleware):  # type: ignore[type-arg]
             of patterns to evaluate. When a tool isn't listed, the bare
             tool name is used as the only pattern.
         runtime_ruleset: Mutable :class:`Ruleset` that the middleware
-            extends in-place when the user replies ``"always"`` to an
-            ask interrupt. Reused across all calls in the same agent
+            extends in-place when the user replies ``"approve_always"`` to
+            an ask interrupt. Reused across all calls in the same agent
             instance so newly-allowed rules apply to subsequent calls.
         always_emit_interrupt_payload: If True, every ask uses the
             SurfSense interrupt wire format (default). Set False to
@@ -268,7 +269,7 @@ class PermissionMiddleware(AgentMiddleware):  # type: ignore[type-arg]
                     for r in rules
                 ],
                 # Rules of thumb for the frontend: surface the patterns
-                # the user can promote to "always" with a single reply.
+                # the user can promote to "approve_always" with a single reply.
                 "always": patterns,
             },
         }
@@ -287,12 +288,12 @@ class PermissionMiddleware(AgentMiddleware):  # type: ignore[type-arg]
         return _normalize_permission_decision(decision)
 
     def _persist_always(self, tool_name: str, patterns: list[str]) -> None:
-        """Promote ``always`` reply into runtime allow rules.
+        """Promote ``approve_always`` reply into runtime allow rules.
 
         Persistence to ``agent_permission_rules`` is done by the
         streaming layer (``stream_new_chat``) once it observes the
-        ``always`` reply — the middleware just keeps an in-memory
-        copy so subsequent calls in the same stream see the rule.
+        ``approve_always`` reply — the middleware just keeps an
+        in-memory copy so subsequent calls in the same stream see the rule.
         """
         for pattern in patterns:
             self._runtime_ruleset.rules.append(
@@ -377,7 +378,7 @@ class PermissionMiddleware(AgentMiddleware):  # type: ignore[type-arg]
                 kind = str(decision.get("decision_type") or "reject").lower()
                 if kind == "once":
                     kept_calls.append(call)
-                elif kind == "always":
+                elif kind == "approve_always":
                     self._persist_always(name, patterns)
                     kept_calls.append(call)
                 elif kind == "reject":

surfsense_backend/app/agents/new_chat/tools/mcp_tool.py

@@ -229,6 +229,7 @@ async def _create_mcp_tool_from_definition_stdio(
             "mcp_input_schema": input_schema,
             "mcp_transport": "stdio",
             "mcp_connector_name": connector_name or None,
+            "mcp_connector_id": connector_id,
             "mcp_is_generic": True,
             "hitl": True,
             # Full-args hash: shared identifiers (cloudId, workspaceId, …)