diff --git a/surfsense_backend/app/agents/multi_agent_chat/__init__.py b/surfsense_backend/app/agents/multi_agent_chat/__init__.py index ba4878d15..bdd54b4e0 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/__init__.py +++ b/surfsense_backend/app/agents/multi_agent_chat/__init__.py @@ -1,20 +1,14 @@ """ Multi-agent chat (LangChain Subagents pattern). -**Vertical slices** +**Layout (SRP)** -- :mod:`gmail` — connector tools, domain agent, ``domain_prompt.md`` -- :mod:`calendar` — connector tools, domain agent, ``domain_prompt.md`` - -**Shared** - -- :mod:`shared` — prompt loader, ``build_domain_agent``, connector deps, invoke result parsing - -**Cross-cutting** - -- :mod:`routing` — supervisor routing tools + invoke helpers -- :mod:`supervisor` — top graph + ``supervisor_prompt.md`` -- :mod:`integration` — ``create_multi_agent_chat`` +- :mod:`expert_agent.builtins` — general categories from the tool registry (research, memory, deliverables — not tied to one vendor). +- :mod:`expert_agent.connectors` — external integrations (one subgraph per product where split). +- :mod:`core` — prompts, compiled subgraph helper, delegation, registry subsets, tool-factory kwargs (:mod:`core.bindings`). +- :mod:`routing` — supervisor-facing ``@tool`` routers → domain invoke. +- :mod:`supervisor` — orchestrator graph + ``supervisor_prompt.md``. +- :mod:`integration` — async ``create_multi_agent_chat`` composer (partitions MCP tools into experts). Documentation: https://docs.langchain.com/oss/python/langchain/multi-agent @@ -23,38 +17,116 @@ https://docs.langchain.com/oss/python/langchain/multi-agent/subagents Display name: ``multi-agent-chat`` — Python package: ``multi_agent_chat``. """ -from app.agents.multi_agent_chat.calendar import ( - build_calendar_domain_agent, - build_google_calendar_connector_tools, +from app.agents.multi_agent_chat.expert_agent.builtins.deliverables import ( + build_deliverables_tools, + build_deliverables_domain_agent, ) -from app.agents.multi_agent_chat.gmail import ( - build_gmail_connector_tools, +from app.agents.multi_agent_chat.expert_agent.builtins.memory import ( + build_memory_tools, + build_memory_domain_agent, +) +from app.agents.multi_agent_chat.expert_agent.builtins.research import ( + build_research_tools, + build_research_domain_agent, +) +from app.agents.multi_agent_chat.expert_agent.connectors.calendar import ( + build_calendar_domain_agent, + build_calendar_tools, +) +from app.agents.multi_agent_chat.expert_agent.connectors.confluence import ( + build_confluence_tools, + build_confluence_domain_agent, +) +from app.agents.multi_agent_chat.expert_agent.connectors.discord import ( + build_discord_tools, + build_discord_domain_agent, +) +from app.agents.multi_agent_chat.expert_agent.connectors.dropbox import ( + build_dropbox_tools, + build_dropbox_domain_agent, +) +from app.agents.multi_agent_chat.expert_agent.connectors.gmail import ( + build_gmail_tools, build_gmail_domain_agent, ) -from app.agents.multi_agent_chat.integration import create_multi_agent_chat -from app.agents.multi_agent_chat.shared import ( +from app.agents.multi_agent_chat.expert_agent.connectors.google_drive import ( + build_google_drive_tools, + build_google_drive_domain_agent, +) +from app.agents.multi_agent_chat.expert_agent.connectors.luma import ( + build_luma_tools, + build_luma_domain_agent, +) +from app.agents.multi_agent_chat.expert_agent.connectors.notion import ( + build_notion_tools, + build_notion_domain_agent, +) +from app.agents.multi_agent_chat.expert_agent.connectors.onedrive import ( + build_onedrive_tools, + build_onedrive_domain_agent, +) +from app.agents.multi_agent_chat.expert_agent.connectors.teams import ( + build_teams_tools, + build_teams_domain_agent, +) +from app.agents.multi_agent_chat.core import ( + REGISTRY_ROUTING_CATEGORY_KEYS, + TOOL_NAMES_BY_CATEGORY, build_domain_agent, + build_registry_dependencies, + build_registry_tools_for_category, + compose_child_task, connector_binding, extract_last_assistant_text, read_prompt_md, ) +from app.agents.multi_agent_chat.integration import create_multi_agent_chat from app.agents.multi_agent_chat.routing import ( + DomainRoutingSpec, build_supervisor_routing_tools, - routing_tools_from_domain_agents, + routing_tools_from_specs, ) from app.agents.multi_agent_chat.supervisor import build_supervisor_agent __all__ = [ + "REGISTRY_ROUTING_CATEGORY_KEYS", + "TOOL_NAMES_BY_CATEGORY", + "DomainRoutingSpec", "build_calendar_domain_agent", + "build_confluence_tools", + "build_confluence_domain_agent", + "build_deliverables_tools", + "build_deliverables_domain_agent", + "build_discord_tools", + "build_discord_domain_agent", "build_domain_agent", - "build_gmail_connector_tools", + "build_dropbox_tools", + "build_dropbox_domain_agent", + "build_gmail_tools", "build_gmail_domain_agent", - "build_google_calendar_connector_tools", + "build_calendar_tools", + "build_google_drive_tools", + "build_google_drive_domain_agent", + "build_luma_tools", + "build_luma_domain_agent", + "build_memory_tools", + "build_memory_domain_agent", + "build_notion_tools", + "build_notion_domain_agent", + "build_onedrive_tools", + "build_onedrive_domain_agent", + "build_registry_dependencies", + "build_registry_tools_for_category", + "build_research_tools", + "build_research_domain_agent", "build_supervisor_agent", "build_supervisor_routing_tools", + "build_teams_tools", + "build_teams_domain_agent", "connector_binding", + "compose_child_task", "create_multi_agent_chat", "extract_last_assistant_text", "read_prompt_md", - "routing_tools_from_domain_agents", + "routing_tools_from_specs", ] diff --git a/surfsense_backend/app/agents/multi_agent_chat/calendar/__init__.py b/surfsense_backend/app/agents/multi_agent_chat/calendar/__init__.py deleted file mode 100644 index 7d207b01f..000000000 --- a/surfsense_backend/app/agents/multi_agent_chat/calendar/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Google Calendar vertical slice: connector tools, domain agent, ``domain_prompt.md``.""" - -from app.agents.multi_agent_chat.calendar.agent import build_calendar_domain_agent -from app.agents.multi_agent_chat.calendar.connector_tools import ( - build_google_calendar_connector_tools, -) - -__all__ = [ - "build_calendar_domain_agent", - "build_google_calendar_connector_tools", -] diff --git a/surfsense_backend/app/agents/multi_agent_chat/calendar/agent.py b/surfsense_backend/app/agents/multi_agent_chat/calendar/agent.py deleted file mode 100644 index 23110ea61..000000000 --- a/surfsense_backend/app/agents/multi_agent_chat/calendar/agent.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Google Calendar domain agent graph.""" - -from __future__ import annotations - -from collections.abc import Sequence - -import app.agents.multi_agent_chat.calendar as calendar_pkg -from langchain_core.language_models import BaseChatModel -from langchain_core.tools import BaseTool - -from app.agents.multi_agent_chat.shared.domain_agent_factory import build_domain_agent - - -def build_calendar_domain_agent(llm: BaseChatModel, tools: Sequence[BaseTool]): - """Compiled Calendar domain-agent graph (prompt + tools co-located under ``calendar``).""" - return build_domain_agent( - llm, - tools, - prompt_package=calendar_pkg.__name__, - prompt_stem="domain_prompt", - ) diff --git a/surfsense_backend/app/agents/multi_agent_chat/calendar/connector_tools.py b/surfsense_backend/app/agents/multi_agent_chat/calendar/connector_tools.py deleted file mode 100644 index 8fb7356ff..000000000 --- a/surfsense_backend/app/agents/multi_agent_chat/calendar/connector_tools.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Google Calendar connector LangChain tools (``new_chat`` factories).""" - -from __future__ import annotations - -from langchain_core.tools import BaseTool -from sqlalchemy.ext.asyncio import AsyncSession - -from app.agents.multi_agent_chat.shared.deps import connector_binding -from app.agents.new_chat.tools.google_calendar import ( - create_create_calendar_event_tool, - create_delete_calendar_event_tool, - create_search_calendar_events_tool, - create_update_calendar_event_tool, -) - - -def build_google_calendar_connector_tools( - *, - db_session: AsyncSession, - search_space_id: int, - user_id: str, -) -> list[BaseTool]: - d = connector_binding( - db_session=db_session, - search_space_id=search_space_id, - user_id=user_id, - ) - return [ - create_search_calendar_events_tool(**d), - create_create_calendar_event_tool(**d), - create_update_calendar_event_tool(**d), - create_delete_calendar_event_tool(**d), - ] diff --git a/surfsense_backend/app/agents/multi_agent_chat/calendar/domain_prompt.md b/surfsense_backend/app/agents/multi_agent_chat/calendar/domain_prompt.md deleted file mode 100644 index 6815e77db..000000000 --- a/surfsense_backend/app/agents/multi_agent_chat/calendar/domain_prompt.md +++ /dev/null @@ -1 +0,0 @@ -You are the Google Calendar domain agent. Use only the tools provided to complete calendar-related tasks. Stay focused on scheduling and calendar operations and respond concisely. diff --git a/surfsense_backend/app/agents/multi_agent_chat/gmail/__init__.py b/surfsense_backend/app/agents/multi_agent_chat/gmail/__init__.py deleted file mode 100644 index dbd4911e0..000000000 --- a/surfsense_backend/app/agents/multi_agent_chat/gmail/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Gmail vertical slice: connector tools, domain agent, ``domain_prompt.md``.""" - -from app.agents.multi_agent_chat.gmail.agent import build_gmail_domain_agent -from app.agents.multi_agent_chat.gmail.connector_tools import build_gmail_connector_tools - -__all__ = [ - "build_gmail_connector_tools", - "build_gmail_domain_agent", -] diff --git a/surfsense_backend/app/agents/multi_agent_chat/gmail/agent.py b/surfsense_backend/app/agents/multi_agent_chat/gmail/agent.py deleted file mode 100644 index 1e591986f..000000000 --- a/surfsense_backend/app/agents/multi_agent_chat/gmail/agent.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Gmail domain agent graph.""" - -from __future__ import annotations - -from collections.abc import Sequence - -import app.agents.multi_agent_chat.gmail as gmail_pkg -from langchain_core.language_models import BaseChatModel -from langchain_core.tools import BaseTool - -from app.agents.multi_agent_chat.shared.domain_agent_factory import build_domain_agent - - -def build_gmail_domain_agent(llm: BaseChatModel, tools: Sequence[BaseTool]): - """Compiled Gmail domain-agent graph (prompt + tools co-located under ``gmail``).""" - return build_domain_agent( - llm, - tools, - prompt_package=gmail_pkg.__name__, - prompt_stem="domain_prompt", - ) diff --git a/surfsense_backend/app/agents/multi_agent_chat/gmail/connector_tools.py b/surfsense_backend/app/agents/multi_agent_chat/gmail/connector_tools.py deleted file mode 100644 index 4042293ad..000000000 --- a/surfsense_backend/app/agents/multi_agent_chat/gmail/connector_tools.py +++ /dev/null @@ -1,37 +0,0 @@ -"""Gmail connector LangChain tools (``new_chat`` factories; order matches registry).""" - -from __future__ import annotations - -from langchain_core.tools import BaseTool -from sqlalchemy.ext.asyncio import AsyncSession - -from app.agents.multi_agent_chat.shared.deps import connector_binding -from app.agents.new_chat.tools.gmail import ( - create_create_gmail_draft_tool, - create_read_gmail_email_tool, - create_search_gmail_tool, - create_send_gmail_email_tool, - create_trash_gmail_email_tool, - create_update_gmail_draft_tool, -) - - -def build_gmail_connector_tools( - *, - db_session: AsyncSession, - search_space_id: int, - user_id: str, -) -> list[BaseTool]: - d = connector_binding( - db_session=db_session, - search_space_id=search_space_id, - user_id=user_id, - ) - return [ - create_search_gmail_tool(**d), - create_read_gmail_email_tool(**d), - create_create_gmail_draft_tool(**d), - create_send_gmail_email_tool(**d), - create_trash_gmail_email_tool(**d), - create_update_gmail_draft_tool(**d), - ] diff --git a/surfsense_backend/app/agents/multi_agent_chat/gmail/domain_prompt.md b/surfsense_backend/app/agents/multi_agent_chat/gmail/domain_prompt.md deleted file mode 100644 index 4f51f10f6..000000000 --- a/surfsense_backend/app/agents/multi_agent_chat/gmail/domain_prompt.md +++ /dev/null @@ -1 +0,0 @@ -You are the Gmail domain agent. Use only the tools provided to complete Gmail-related tasks. Stay focused on email operations and respond concisely. diff --git a/surfsense_backend/app/agents/multi_agent_chat/integration/create_multi_agent_chat.py b/surfsense_backend/app/agents/multi_agent_chat/integration/create_multi_agent_chat.py index 2d4046134..4bfb7f64d 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/integration/create_multi_agent_chat.py +++ b/surfsense_backend/app/agents/multi_agent_chat/integration/create_multi_agent_chat.py @@ -2,36 +2,74 @@ from __future__ import annotations +from typing import Any + from langchain_core.language_models import BaseChatModel +from langchain_core.tools import BaseTool from langgraph.types import Checkpointer from sqlalchemy.ext.asyncio import AsyncSession -from app.agents.multi_agent_chat.calendar import build_google_calendar_connector_tools -from app.agents.multi_agent_chat.gmail import build_gmail_connector_tools +from app.db import ChatVisibility + +from app.agents.new_chat.tools.mcp_tool import load_mcp_tools + +from app.agents.multi_agent_chat.core.mcp_partition import ( + fetch_mcp_connector_metadata_maps, + partition_mcp_tools_by_expert_route, +) +from app.agents.multi_agent_chat.core.registry import build_registry_dependencies from app.agents.multi_agent_chat.routing.supervisor_routing import build_supervisor_routing_tools from app.agents.multi_agent_chat.supervisor import build_supervisor_agent -def create_multi_agent_chat( +async def create_multi_agent_chat( llm: BaseChatModel, *, db_session: AsyncSession, search_space_id: int, user_id: str, checkpointer: Checkpointer | None = None, + thread_id: str | None = None, + firecrawl_api_key: str | None = None, + connector_service: Any | None = None, + available_connectors: list[str] | None = None, + available_document_types: list[str] | None = None, + thread_visibility: ChatVisibility = ChatVisibility.PRIVATE, + include_mcp_tools: bool = True, ): - """Build the full multi-agent chat graph (supervisor + Gmail + Calendar sub-agents via ``new_chat`` tools).""" + """Build the full multi-agent chat graph (supervisor + domain subgraphs via routing tools). + + **Builtins** (:mod:`expert_agent.builtins`): registry-grouped **categories** (research, memory, deliverables). + **Connectors** (:mod:`expert_agent.connectors`): **vendor integrations** — one subgraph each where split + (e.g. Gmail, Calendar, Discord, Teams, Notion, Confluence, Google Drive, Dropbox, OneDrive, Luma). + + MCP tools from ``new_chat`` (``load_mcp_tools``) are partitioned inside this package and attached only + to the matching expert subgraphs — not to the supervisor tool list as raw MCP calls. + + Deliverables (thread-scoped reports, podcasts, etc.) are registered only when ``thread_id`` is set. + """ + mcp_tools_by_route: dict[str, list[BaseTool]] | None = None + if include_mcp_tools: + mcp_flat = await load_mcp_tools(db_session, search_space_id) + id_map, name_map = await fetch_mcp_connector_metadata_maps(db_session, search_space_id) + mcp_tools_by_route = partition_mcp_tools_by_expert_route(mcp_flat, id_map, name_map) + + registry_dependencies = build_registry_dependencies( + db_session=db_session, + search_space_id=search_space_id, + user_id=user_id, + thread_id=thread_id or "", + llm=llm, + firecrawl_api_key=firecrawl_api_key, + connector_service=connector_service, + available_connectors=available_connectors, + available_document_types=available_document_types, + thread_visibility=thread_visibility, + ) routing_tools = build_supervisor_routing_tools( llm, - gmail_tools=build_gmail_connector_tools( - db_session=db_session, - search_space_id=search_space_id, - user_id=user_id, - ), - calendar_tools=build_google_calendar_connector_tools( - db_session=db_session, - search_space_id=search_space_id, - user_id=user_id, - ), + registry_dependencies=registry_dependencies, + include_deliverables=thread_id is not None, + mcp_tools_by_route=mcp_tools_by_route, ) return build_supervisor_agent(llm, tools=routing_tools, checkpointer=checkpointer) diff --git a/surfsense_backend/app/agents/multi_agent_chat/shared/__init__.py b/surfsense_backend/app/agents/multi_agent_chat/shared/__init__.py deleted file mode 100644 index 1ef1ad771..000000000 --- a/surfsense_backend/app/agents/multi_agent_chat/shared/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Cross-cutting helpers: prompt loading, domain agent factory, connector deps.""" - -from app.agents.multi_agent_chat.shared.deps import connector_binding -from app.agents.multi_agent_chat.shared.domain_agent_factory import build_domain_agent -from app.agents.multi_agent_chat.shared.invoke_output import extract_last_assistant_text -from app.agents.multi_agent_chat.shared.prompt_loader import read_prompt_md - -__all__ = [ - "build_domain_agent", - "connector_binding", - "extract_last_assistant_text", - "read_prompt_md", -] diff --git a/surfsense_backend/app/agents/multi_agent_chat/shared/deps.py b/surfsense_backend/app/agents/multi_agent_chat/shared/deps.py deleted file mode 100644 index c1e18e849..000000000 --- a/surfsense_backend/app/agents/multi_agent_chat/shared/deps.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Shared kwargs for ``new_chat`` connector tool factories.""" - -from __future__ import annotations - -from sqlalchemy.ext.asyncio import AsyncSession - - -def connector_binding( - *, - db_session: AsyncSession, - search_space_id: int, - user_id: str, -) -> dict[str, AsyncSession | int | str]: - return { - "db_session": db_session, - "search_space_id": search_space_id, - "user_id": user_id, - } diff --git a/surfsense_backend/app/agents/multi_agent_chat/shared/domain_agent_factory.py b/surfsense_backend/app/agents/multi_agent_chat/shared/domain_agent_factory.py deleted file mode 100644 index c6c5b061a..000000000 --- a/surfsense_backend/app/agents/multi_agent_chat/shared/domain_agent_factory.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Compile a domain agent graph from a co-located prompt + tool list.""" - -from __future__ import annotations - -from collections.abc import Sequence - -from langchain.agents import create_agent -from langchain_core.language_models import BaseChatModel -from langchain_core.tools import BaseTool - -from app.agents.multi_agent_chat.shared.prompt_loader import read_prompt_md - - -def build_domain_agent( - llm: BaseChatModel, - tools: Sequence[BaseTool], - *, - prompt_package: str, - prompt_stem: str = "domain_prompt", -): - """``create_agent`` + ``{prompt_stem}.md`` loaded from ``prompt_package``.""" - system_prompt = read_prompt_md(prompt_package, prompt_stem) - return create_agent( - llm, - system_prompt=system_prompt, - tools=list(tools), - ) diff --git a/surfsense_backend/app/agents/multi_agent_chat/shared/invoke_output.py b/surfsense_backend/app/agents/multi_agent_chat/shared/invoke_output.py deleted file mode 100644 index 2bbab6e57..000000000 --- a/surfsense_backend/app/agents/multi_agent_chat/shared/invoke_output.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Extract displayable text from a LangGraph agent ``invoke`` / ``ainvoke`` result.""" - -from __future__ import annotations - -from typing import Any - - -def extract_last_assistant_text(result: dict[str, Any]) -> str: - """Return the last message's string content, or ``\"\"`` if missing.""" - messages = result.get("messages") or [] - if not messages: - return "" - last = messages[-1] - content = getattr(last, "content", None) - if isinstance(content, str): - return content - return str(last) diff --git a/surfsense_backend/app/agents/multi_agent_chat/shared/prompt_loader.py b/surfsense_backend/app/agents/multi_agent_chat/shared/prompt_loader.py deleted file mode 100644 index 940647364..000000000 --- a/surfsense_backend/app/agents/multi_agent_chat/shared/prompt_loader.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Load ``*.md`` from any package (vertical slices use co-located prompts).""" - -from __future__ import annotations - -from importlib import resources - - -def read_prompt_md(package: str, stem: str) -> str: - """Read ``{stem}.md`` from the given import package (e.g. ``app.agents.multi_agent_chat.gmail``).""" - try: - ref = resources.files(package).joinpath(f"{stem}.md") - if not ref.is_file(): - return "" - text = ref.read_text(encoding="utf-8") - except (FileNotFoundError, ModuleNotFoundError, OSError, TypeError): - return "" - if text.endswith("\n"): - text = text[:-1] - return text diff --git a/surfsense_backend/app/agents/multi_agent_chat/supervisor/graph.py b/surfsense_backend/app/agents/multi_agent_chat/supervisor/graph.py index 5cee73c37..c157a719b 100644 --- a/surfsense_backend/app/agents/multi_agent_chat/supervisor/graph.py +++ b/surfsense_backend/app/agents/multi_agent_chat/supervisor/graph.py @@ -11,7 +11,7 @@ from langchain_core.language_models import BaseChatModel from langchain_core.tools import BaseTool from langgraph.types import Checkpointer -from app.agents.multi_agent_chat.shared.prompt_loader import read_prompt_md +from app.agents.multi_agent_chat.core.prompts import read_prompt_md def build_supervisor_agent( diff --git a/surfsense_backend/app/agents/new_chat_supervisor_baseline/__init__.py b/surfsense_backend/app/agents/new_chat_supervisor_baseline/__init__.py deleted file mode 100644 index e8939e4ca..000000000 --- a/surfsense_backend/app/agents/new_chat_supervisor_baseline/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Baseline deep-agent factory without SurfSense specialist subagents. - -Swap imports manually while building supervisor-style delegation:: - - # from app.agents.new_chat.chat_deepagent import create_surfsense_deep_agent - from app.agents.new_chat_supervisor_baseline.chat_deepagent import ( - create_surfsense_deep_agent, - ) - -""" - -from app.agents.new_chat_supervisor_baseline.chat_deepagent import ( - create_surfsense_deep_agent, -) - -__all__ = ["create_surfsense_deep_agent"] diff --git a/surfsense_backend/app/agents/new_chat_supervisor_baseline/chat_deepagent.py b/surfsense_backend/app/agents/new_chat_supervisor_baseline/chat_deepagent.py deleted file mode 100644 index 3626a4789..000000000 --- a/surfsense_backend/app/agents/new_chat_supervisor_baseline/chat_deepagent.py +++ /dev/null @@ -1,145 +0,0 @@ -""" -Supervisor baseline: **no registry tools** and **no tool-injecting middleware** -(no ``task`` / subagents, filesystem, todos, skills, permission, pruning, repair, …). - -Connector/document discovery still feeds :class:`KnowledgePriorityMiddleware` so turns -can include KB priority hints. - -System prompt: :func:`build_supervisor_system_prompt` — SurfSense ``agent_*`` identity -fragments plus supervisor-scoped KB/memory text and composer citation/provider blocks, -without tool lists or ``tool_routing`` (see module docstring there). - -See :mod:`app.agents.new_chat.chat_deepagent` for the full production agent. - -Implementation: :mod:`app.agents.new_chat_supervisor_baseline.deep_agent`. -""" - -import asyncio -import logging -import time -from collections.abc import Sequence - -from langchain_core.language_models import BaseChatModel -from langchain_core.tools import BaseTool -from langgraph.types import Checkpointer -from sqlalchemy.ext.asyncio import AsyncSession - -from app.agents.new_chat.feature_flags import AgentFeatureFlags, get_flags -from app.agents.new_chat.filesystem_selection import FilesystemSelection -from app.agents.new_chat.llm_config import AgentConfig -from app.db import ChatVisibility -from app.services.connector_service import ConnectorService -from app.utils.perf import get_perf_logger - -from app.agents.new_chat_supervisor_baseline.deep_agent.compiled_agent import ( - build_compiled_agent_blocking, -) -from app.agents.new_chat_supervisor_baseline.deep_agent.connector_searchable import ( - map_connectors_to_searchable_types, -) -from app.agents.new_chat_supervisor_baseline.supervisor_system_prompt import ( - build_supervisor_system_prompt, -) - -_perf_log = get_perf_logger() - - -async def create_surfsense_deep_agent( - llm: BaseChatModel, - search_space_id: int, - db_session: AsyncSession, - connector_service: ConnectorService, - checkpointer: Checkpointer, - user_id: str | None = None, - thread_id: int | None = None, - agent_config: AgentConfig | None = None, - enabled_tools: list[str] | None = None, - disabled_tools: list[str] | None = None, - additional_tools: Sequence[BaseTool] | None = None, - firecrawl_api_key: str | None = None, - thread_visibility: ChatVisibility | None = None, - mentioned_document_ids: list[int] | None = None, - anon_session_id: str | None = None, - filesystem_selection: FilesystemSelection | None = None, -): - """ - Build the supervisor baseline agent: registry tools are not loaded. - - Parameters such as ``enabled_tools``, ``additional_tools``, and ``firecrawl_api_key`` - are ignored for now; kept so call sites stay compatible. - """ - _ = (enabled_tools, disabled_tools, additional_tools, firecrawl_api_key, db_session) - - _t_agent_total = time.perf_counter() - - filesystem_selection = filesystem_selection or FilesystemSelection() - _fs_mode = filesystem_selection.mode - - available_connectors: list[str] | None = None - available_document_types: list[str] | None = None - - _t0 = time.perf_counter() - try: - connector_types = await connector_service.get_available_connectors( - search_space_id - ) - if connector_types: - available_connectors = map_connectors_to_searchable_types(connector_types) - - available_document_types = await connector_service.get_available_document_types( - search_space_id - ) - - except Exception as e: - logging.warning(f"Failed to discover available connectors/document types: {e}") - _perf_log.info( - "[create_agent] Connector/doc-type discovery in %.3fs", - time.perf_counter() - _t0, - ) - - visibility = thread_visibility or ChatVisibility.PRIVATE - - tools: list[BaseTool] = [] - - _flags: AgentFeatureFlags = get_flags() - _perf_log.info("[create_agent] supervisor baseline: 0 registry tools") - - _t0 = time.perf_counter() - - final_system_prompt = build_supervisor_system_prompt( - agent_config=agent_config, - thread_visibility=thread_visibility, - llm=llm, - ) - _perf_log.info( - "[create_agent] System prompt built in %.3fs", time.perf_counter() - _t0 - ) - - _t0 = time.perf_counter() - agent = await asyncio.to_thread( - build_compiled_agent_blocking, - llm=llm, - tools=tools, - final_system_prompt=final_system_prompt, - filesystem_mode=_fs_mode, - search_space_id=search_space_id, - user_id=user_id, - thread_id=thread_id, - visibility=visibility, - anon_session_id=anon_session_id, - available_connectors=available_connectors, - available_document_types=available_document_types, - mentioned_document_ids=mentioned_document_ids, - flags=_flags, - checkpointer=checkpointer, - ) - _perf_log.info( - "[create_agent] Middleware stack + graph compiled in %.3fs", - time.perf_counter() - _t0, - ) - - _perf_log.info( - "[create_agent] Total agent creation in %.3fs", - time.perf_counter() - _t_agent_total, - ) - return agent diff --git a/surfsense_backend/app/agents/new_chat_supervisor_baseline/deep_agent/__init__.py b/surfsense_backend/app/agents/new_chat_supervisor_baseline/deep_agent/__init__.py deleted file mode 100644 index df82f9377..000000000 --- a/surfsense_backend/app/agents/new_chat_supervisor_baseline/deep_agent/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Helpers used only by :mod:`app.agents.new_chat_supervisor_baseline.chat_deepagent`.""" diff --git a/surfsense_backend/app/agents/new_chat_supervisor_baseline/deep_agent/compiled_agent.py b/surfsense_backend/app/agents/new_chat_supervisor_baseline/deep_agent/compiled_agent.py deleted file mode 100644 index b43e0364d..000000000 --- a/surfsense_backend/app/agents/new_chat_supervisor_baseline/deep_agent/compiled_agent.py +++ /dev/null @@ -1,112 +0,0 @@ -"""Compile a minimal supervisor graph: no bound tools, no tool-injecting middleware.""" - -from __future__ import annotations - -from collections.abc import Sequence - -from deepagents import __version__ as deepagents_version -from deepagents.backends import StateBackend -from langchain.agents import create_agent -from langchain_anthropic.middleware import AnthropicPromptCachingMiddleware -from langchain_core.language_models import BaseChatModel -from langchain_core.tools import BaseTool -from langgraph.types import Checkpointer - -from app.agents.new_chat.context import SurfSenseContextSchema -from app.agents.new_chat.feature_flags import AgentFeatureFlags -from app.agents.new_chat.filesystem_selection import FilesystemMode -from app.agents.new_chat.middleware import ( - AnonymousDocumentMiddleware, - FileIntentMiddleware, - KnowledgeBasePersistenceMiddleware, - KnowledgePriorityMiddleware, - KnowledgeTreeMiddleware, - MemoryInjectionMiddleware, - create_surfsense_compaction_middleware, -) -from app.db import ChatVisibility - - -def build_compiled_agent_blocking( - *, - llm: BaseChatModel, - tools: Sequence[BaseTool], - final_system_prompt: str, - filesystem_mode: FilesystemMode, - search_space_id: int, - user_id: str | None, - thread_id: int | None, - visibility: ChatVisibility, - anon_session_id: str | None, - available_connectors: list[str] | None, - available_document_types: list[str] | None, - mentioned_document_ids: list[int] | None, - flags: AgentFeatureFlags, - checkpointer: Checkpointer, -): - """Build middleware + compile graph synchronously (typically ``asyncio.to_thread``). - - Intentionally excludes registry tools (``tools`` should be ``[]``), SubAgent/task, - filesystem/todo/skills middleware, and tool-centric hygiene (repair, dedup, permission). - """ - _ = flags # retained for API parity with callers; stack is fixed minimal for now - - _memory_middleware = MemoryInjectionMiddleware( - user_id=user_id, - search_space_id=search_space_id, - thread_visibility=visibility, - ) - - summarization_mw = create_surfsense_compaction_middleware(llm, StateBackend) - - deepagent_middleware = [ - _memory_middleware, - AnonymousDocumentMiddleware(anon_session_id=anon_session_id) - if filesystem_mode == FilesystemMode.CLOUD - else None, - KnowledgeTreeMiddleware( - search_space_id=search_space_id, - filesystem_mode=filesystem_mode, - llm=llm, - ) - if filesystem_mode == FilesystemMode.CLOUD - else None, - KnowledgePriorityMiddleware( - llm=llm, - search_space_id=search_space_id, - filesystem_mode=filesystem_mode, - available_connectors=available_connectors, - available_document_types=available_document_types, - mentioned_document_ids=mentioned_document_ids, - ), - FileIntentMiddleware(llm=llm), - KnowledgeBasePersistenceMiddleware( - search_space_id=search_space_id, - created_by_id=user_id, - filesystem_mode=filesystem_mode, - thread_id=thread_id, - ) - if filesystem_mode == FilesystemMode.CLOUD - else None, - summarization_mw, - AnthropicPromptCachingMiddleware(unsupported_model_behavior="ignore"), - ] - deepagent_middleware = [m for m in deepagent_middleware if m is not None] - - agent = create_agent( - llm, - system_prompt=final_system_prompt, - tools=list(tools), - middleware=deepagent_middleware, - context_schema=SurfSenseContextSchema, - checkpointer=checkpointer, - ) - return agent.with_config( - { - "recursion_limit": 10_000, - "metadata": { - "ls_integration": "deepagents", - "versions": {"deepagents": deepagents_version}, - }, - } - ) diff --git a/surfsense_backend/app/agents/new_chat_supervisor_baseline/deep_agent/connector_searchable.py b/surfsense_backend/app/agents/new_chat_supervisor_baseline/deep_agent/connector_searchable.py deleted file mode 100644 index 974416dfb..000000000 --- a/surfsense_backend/app/agents/new_chat_supervisor_baseline/deep_agent/connector_searchable.py +++ /dev/null @@ -1,62 +0,0 @@ -"""Map connector enum values to searchable document/connector type strings.""" - -from __future__ import annotations - -from typing import Any - -_CONNECTOR_TYPE_TO_SEARCHABLE: dict[str, str] = { - "TAVILY_API": "TAVILY_API", - "LINKUP_API": "LINKUP_API", - "BAIDU_SEARCH_API": "BAIDU_SEARCH_API", - "SLACK_CONNECTOR": "SLACK_CONNECTOR", - "TEAMS_CONNECTOR": "TEAMS_CONNECTOR", - "NOTION_CONNECTOR": "NOTION_CONNECTOR", - "GITHUB_CONNECTOR": "GITHUB_CONNECTOR", - "LINEAR_CONNECTOR": "LINEAR_CONNECTOR", - "DISCORD_CONNECTOR": "DISCORD_CONNECTOR", - "JIRA_CONNECTOR": "JIRA_CONNECTOR", - "CONFLUENCE_CONNECTOR": "CONFLUENCE_CONNECTOR", - "CLICKUP_CONNECTOR": "CLICKUP_CONNECTOR", - "GOOGLE_CALENDAR_CONNECTOR": "GOOGLE_CALENDAR_CONNECTOR", - "GOOGLE_GMAIL_CONNECTOR": "GOOGLE_GMAIL_CONNECTOR", - "GOOGLE_DRIVE_CONNECTOR": "GOOGLE_DRIVE_FILE", - "AIRTABLE_CONNECTOR": "AIRTABLE_CONNECTOR", - "LUMA_CONNECTOR": "LUMA_CONNECTOR", - "ELASTICSEARCH_CONNECTOR": "ELASTICSEARCH_CONNECTOR", - "WEBCRAWLER_CONNECTOR": "CRAWLED_URL", - "BOOKSTACK_CONNECTOR": "BOOKSTACK_CONNECTOR", - "CIRCLEBACK_CONNECTOR": "CIRCLEBACK", - "OBSIDIAN_CONNECTOR": "OBSIDIAN_CONNECTOR", - "DROPBOX_CONNECTOR": "DROPBOX_FILE", - "ONEDRIVE_CONNECTOR": "ONEDRIVE_FILE", - "COMPOSIO_GOOGLE_DRIVE_CONNECTOR": "GOOGLE_DRIVE_FILE", - "COMPOSIO_GMAIL_CONNECTOR": "GOOGLE_GMAIL_CONNECTOR", - "COMPOSIO_GOOGLE_CALENDAR_CONNECTOR": "GOOGLE_CALENDAR_CONNECTOR", -} - -_ALWAYS_AVAILABLE_DOC_TYPES: tuple[str, ...] = ( - "EXTENSION", - "FILE", - "NOTE", - "YOUTUBE_VIDEO", -) - - -def map_connectors_to_searchable_types(connector_types: list[Any]) -> list[str]: - """Map connector types to searchable strings; dedupe preserving order.""" - result_set: set[str] = set() - result_list: list[str] = [] - - for doc_type in _ALWAYS_AVAILABLE_DOC_TYPES: - if doc_type not in result_set: - result_set.add(doc_type) - result_list.append(doc_type) - - for ct in connector_types: - ct_str = ct.value if hasattr(ct, "value") else str(ct) - searchable = _CONNECTOR_TYPE_TO_SEARCHABLE.get(ct_str) - if searchable and searchable not in result_set: - result_set.add(searchable) - result_list.append(searchable) - - return result_list diff --git a/surfsense_backend/app/agents/new_chat_supervisor_baseline/prompts/__init__.py b/surfsense_backend/app/agents/new_chat_supervisor_baseline/prompts/__init__.py deleted file mode 100644 index 68441a70e..000000000 --- a/surfsense_backend/app/agents/new_chat_supervisor_baseline/prompts/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Supervisor-scoped prompt fragments (adaptations of ``new_chat/prompts/base``).""" diff --git a/surfsense_backend/app/agents/new_chat_supervisor_baseline/prompts/kb_policy_supervisor_private.md b/surfsense_backend/app/agents/new_chat_supervisor_baseline/prompts/kb_policy_supervisor_private.md deleted file mode 100644 index 45dc30869..000000000 --- a/surfsense_backend/app/agents/new_chat_supervisor_baseline/prompts/kb_policy_supervisor_private.md +++ /dev/null @@ -1,18 +0,0 @@ - -Adapted from ``prompts/base/kb_only_policy_private.md`` for supervisor-only runs (no web -search / scrape / connector tools on this node). - -CRITICAL RULE — KNOWLEDGE CONTEXT FIRST FOR FACTUAL QUESTIONS: -- For factual or informational questions, rely on information in this thread and on - knowledge SurfSense surfaces in your prompt (for example priority document excerpts - or injected memory text). Do not substitute unchecked general knowledge unless the - user explicitly opts in. -- If nothing in the conversation or injected context answers the question, you MUST: - 1. Say you could not find it in the available SurfSense context for this turn. - 2. Ask: "Would you like me to answer from my general knowledge instead?" - 3. ONLY provide a general-knowledge answer AFTER the user explicitly says yes. -- This policy does NOT apply to: - * Casual conversation, greetings, or meta-questions about SurfSense itself - * Formatting, summarization, or analysis of content already present in the conversation - * Following user instructions that are clearly task-oriented (e.g., "rewrite this in bullet points") - diff --git a/surfsense_backend/app/agents/new_chat_supervisor_baseline/prompts/kb_policy_supervisor_team.md b/surfsense_backend/app/agents/new_chat_supervisor_baseline/prompts/kb_policy_supervisor_team.md deleted file mode 100644 index c201d11c1..000000000 --- a/surfsense_backend/app/agents/new_chat_supervisor_baseline/prompts/kb_policy_supervisor_team.md +++ /dev/null @@ -1,18 +0,0 @@ - -Adapted from ``prompts/base/kb_only_policy_team.md`` for supervisor-only runs (no web -search / scrape / connector tools on this node). - -CRITICAL RULE — TEAM KNOWLEDGE CONTEXT FIRST FOR FACTUAL QUESTIONS: -- For factual or informational questions, rely on information in this thread and on - knowledge SurfSense surfaces in your prompt from the shared space (for example - priority document excerpts or injected memory text). Do not substitute unchecked - general knowledge unless a team member explicitly opts in. -- If nothing in the conversation or injected context answers the question, you MUST: - 1. Say you could not find it in the available SurfSense context for this turn. - 2. Ask: "Would you like me to answer from my general knowledge instead?" - 3. ONLY provide a general-knowledge answer AFTER a team member explicitly says yes. -- This policy does NOT apply to: - * Casual conversation, greetings, or meta-questions about SurfSense itself - * Formatting, summarization, or analysis of content already present in the conversation - * Following user instructions that are clearly task-oriented (e.g., "rewrite this in bullet points") - diff --git a/surfsense_backend/app/agents/new_chat_supervisor_baseline/prompts/memory_context_supervisor.md b/surfsense_backend/app/agents/new_chat_supervisor_baseline/prompts/memory_context_supervisor.md deleted file mode 100644 index 7d5a7c648..000000000 --- a/surfsense_backend/app/agents/new_chat_supervisor_baseline/prompts/memory_context_supervisor.md +++ /dev/null @@ -1,9 +0,0 @@ - -Derived from ``prompts/base/memory_protocol_*.md``, without requiring ``update_memory`` -calls (this supervisor node does not expose that tool). - -Personalized memory text may be injected into your prompt when configured. You cannot -persist new long-term memory from this supervisor node; if the user asks you to -remember something permanently, explain that doing so requires the full SurfSense -agent with memory tools enabled or another persistence path they configure. - diff --git a/surfsense_backend/app/agents/new_chat_supervisor_baseline/prompts/supervisor_graph_role.md b/surfsense_backend/app/agents/new_chat_supervisor_baseline/prompts/supervisor_graph_role.md deleted file mode 100644 index 875e09510..000000000 --- a/surfsense_backend/app/agents/new_chat_supervisor_baseline/prompts/supervisor_graph_role.md +++ /dev/null @@ -1,9 +0,0 @@ - -This node follows the LangGraph multi-agent **supervisor** pattern: the supervisor -language model responds from the current conversation and optional supervisor-scoped -system prompt (see LangChain Reference: ``langgraph_supervisor.create_supervisor``, -parameter ``prompt`` — typically a ``SystemMessage`` that scopes routing and handoff -behavior). In this SurfSense deployment the supervisor graph does **not** attach -registry tools or worker subgraphs—answer from messages and system-injected context, -and state plainly when the user expects tools or delegations that are not wired here. - diff --git a/surfsense_backend/app/agents/new_chat_supervisor_baseline/supervisor_system_prompt.py b/surfsense_backend/app/agents/new_chat_supervisor_baseline/supervisor_system_prompt.py deleted file mode 100644 index 82c0077e3..000000000 --- a/surfsense_backend/app/agents/new_chat_supervisor_baseline/supervisor_system_prompt.py +++ /dev/null @@ -1,121 +0,0 @@ -"""Supervisor-scoped system prompt for ``new_chat_supervisor_baseline``. - -Composition follows the same fragment discipline as -:func:`app.agents.new_chat.prompts.composer.compose_system_prompt`, but **omits** -sections that assume registry tools: ``base/tool_routing_*.md``, ``tools/_preamble.md``, -the tools/examples blocks, ``base/parameter_resolution.md`` (discovery lists concrete -tools), and ``base/memory_protocol_*.md`` (requires ``update_memory`` calls). - -**Authoritative supervisor semantics:** LangChain Reference documents -``langgraph_supervisor.create_supervisor`` — the supervisor graph accepts an optional -``prompt`` (typically a ``SystemMessage``) that scopes the supervisor LLM alongside -managed worker graphs. - -**SurfSense sources reused verbatim where applicable:** ``prompts/base/agent_private.md`` / -``agent_team.md`` from :mod:`app.agents.new_chat.prompts`. KB policy is adapted from -``base/kb_only_policy_*.md`` into supervisor-local fragments that reference injected -context instead of tool outputs. Provider and citation blocks reuse -``composer._build_provider_block`` / ``_build_citation_block`` and -``composer.detect_provider_variant`` unchanged. -""" - -from __future__ import annotations - -from datetime import UTC, datetime -from importlib import resources - -from langchain_core.language_models import BaseChatModel - -from app.agents.new_chat.llm_config import AgentConfig -from app.agents.new_chat.prompts import composer as pc -from app.db import ChatVisibility - -_SUP_PROMPTS_PKG = "app.agents.new_chat_supervisor_baseline.prompts" - - -def _read_supervisor_fragment(filename: str) -> str: - try: - ref = resources.files(_SUP_PROMPTS_PKG).joinpath(filename) - if not ref.is_file(): - return "" - text = ref.read_text(encoding="utf-8") - except (FileNotFoundError, ModuleNotFoundError, OSError): - return "" - if text.endswith("\n"): - text = text[:-1] - return text - - -def _build_supervisor_system_instruction_block( - *, - visibility: ChatVisibility, - resolved_today: str, -) -> str: - """```` body: LangGraph supervisor scope + SurfSense identity + adapted KB + memory limits.""" - variant = "team" if visibility == ChatVisibility.SEARCH_SPACE else "private" - sections = [ - _read_supervisor_fragment("supervisor_graph_role.md"), - pc._read_fragment(f"base/agent_{variant}.md"), - _read_supervisor_fragment(f"kb_policy_supervisor_{variant}.md"), - _read_supervisor_fragment("memory_context_supervisor.md"), - ] - body = "\n\n".join(s for s in sections if s) - block = f"\n\n{body}\n\n\n" - return block.format(resolved_today=resolved_today) - - -def resolve_llm_model_name(llm: BaseChatModel) -> str | None: - """Best-effort model id string for :func:`composer.detect_provider_variant`.""" - name = getattr(llm, "model_name", None) - if isinstance(name, str) and name.strip(): - return name.strip() - model = getattr(llm, "model", None) - if isinstance(model, str) and model.strip(): - return model.strip() - profile = getattr(llm, "profile", None) - if isinstance(profile, dict): - for key in ("model", "model_name"): - m = profile.get(key) - if isinstance(m, str) and m.strip(): - return m.strip() - return None - - -def build_supervisor_system_prompt( - *, - agent_config: AgentConfig | None, - thread_visibility: ChatVisibility | None, - llm: BaseChatModel, -) -> str: - """Assemble the supervisor system prompt (no tool-list or tool-routing fragments).""" - resolved_today = datetime.now(UTC).astimezone(UTC).date().isoformat() - visibility = thread_visibility or ChatVisibility.PRIVATE - model_name = resolve_llm_model_name(llm) - - if agent_config is not None: - custom = (agent_config.system_instructions or "").strip() - if custom: - sys_block = agent_config.system_instructions.format(resolved_today=resolved_today) - elif agent_config.use_default_system_instructions: - sys_block = _build_supervisor_system_instruction_block( - visibility=visibility, - resolved_today=resolved_today, - ) - else: - sys_block = "" - else: - sys_block = _build_supervisor_system_instruction_block( - visibility=visibility, - resolved_today=resolved_today, - ) - - provider_variant = pc.detect_provider_variant(model_name) - sys_block += pc._build_provider_block(provider_variant) - - if agent_config is None: - citations_enabled = True - else: - citations_enabled = agent_config.citations_enabled - - sys_block += pc._build_citation_block(citations_enabled) - return sys_block