diff --git a/surfsense_backend/app/agents/shared/middleware/__init__.py b/surfsense_backend/app/agents/shared/middleware/__init__.py index 9ca2f3960..737c51f25 100644 --- a/surfsense_backend/app/agents/shared/middleware/__init__.py +++ b/surfsense_backend/app/agents/shared/middleware/__init__.py @@ -21,9 +21,6 @@ from app.agents.shared.middleware.doom_loop import DoomLoopMiddleware from app.agents.shared.middleware.file_intent import ( FileIntentMiddleware, ) -from app.agents.shared.middleware.filesystem import ( - SurfSenseFilesystemMiddleware, -) from app.agents.shared.middleware.flatten_system import ( FlattenSystemMessageMiddleware, ) @@ -78,7 +75,6 @@ __all__ = [ "SpillToBackendEdit", "SpillingContextEditingMiddleware", "SurfSenseCompactionMiddleware", - "SurfSenseFilesystemMiddleware", "ToolCallNameRepairMiddleware", "build_skills_backend_factory", "commit_staged_filesystem_state", diff --git a/surfsense_backend/app/agents/shared/middleware/filesystem.py b/surfsense_backend/app/agents/shared/middleware/filesystem.py deleted file mode 100644 index 7968d76ba..000000000 --- a/surfsense_backend/app/agents/shared/middleware/filesystem.py +++ /dev/null @@ -1,1998 +0,0 @@ -"""Custom filesystem middleware for the SurfSense agent. - -This middleware fully overrides every deepagents filesystem tool so that the -``Command(update=...)`` payload can carry SurfSense-specific state fields -(``cwd``, ``staged_dirs``, ``pending_moves``, ``doc_id_by_path``, -``dirty_paths``) atomically alongside the standard ``files`` update. - -In CLOUD mode the backend is :class:`KBPostgresBackend` (lazy DB reads, no DB -writes). End-of-turn persistence is handled by -:class:`KnowledgeBasePersistenceMiddleware`. In DESKTOP_LOCAL_FOLDER mode the -backend is :class:`MultiRootLocalFolderBackend` and writes go straight to disk. - -New tools introduced here: - -* ``mkdir`` — cloud-only stages folder paths to ``state['staged_dirs']``; - desktop creates real directories. -* ``cd`` / ``pwd`` — manage ``state['cwd']`` (per-thread). -* ``move_file`` — staged commit in cloud, real disk move in desktop. -* ``list_tree`` — works in both modes (cloud uses - :func:`KBPostgresBackend.alist_tree_listing`). - -The middleware no longer ships ``save_document``; persistence is inferred -from ``write_file`` / ``edit_file`` against ``/documents/*`` paths. -""" - -from __future__ import annotations - -import asyncio -import json -import logging -import posixpath -import re -import secrets -from typing import Annotated, Any - -from daytona.common.errors import DaytonaError -from deepagents import FilesystemMiddleware -from deepagents.backends.protocol import EditResult, WriteResult -from deepagents.backends.utils import ( - create_file_data, - format_read_response, - validate_path, -) -from langchain.tools import ToolRuntime -from langchain_core.messages import ToolMessage -from langchain_core.tools import BaseTool, StructuredTool -from langgraph.types import Command - -from app.agents.shared.filesystem_selection import FilesystemMode -from app.agents.shared.filesystem_state import SurfSenseFilesystemState -from app.agents.shared.middleware.kb_postgres_backend import ( - KBPostgresBackend, - paginate_listing, -) -from app.agents.shared.middleware.multi_root_local_folder_backend import ( - MultiRootLocalFolderBackend, -) -from app.agents.shared.path_resolver import DOCUMENTS_ROOT -from app.agents.shared.sandbox import ( - _evict_sandbox_cache, - delete_sandbox, - get_or_create_sandbox, - is_sandbox_enabled, -) -from app.agents.shared.state_reducers import _CLEAR - -logger = logging.getLogger(__name__) - - -# ============================================================================= -# System Prompt (built per-session based on filesystem_mode) -# ============================================================================= -# -# Each chat session runs in exactly one filesystem mode. Including rules for -# the OTHER mode just wastes tokens and confuses the model, so we build the -# prompt + tool descriptions for the active mode only. - -_COMMON_PROMPT_HEADER = """## Following Conventions - -- Read files before editing — understand existing content before making changes. -- Mimic existing style, naming conventions, and patterns. -- Never claim a file was created/updated unless filesystem tool output confirms success. -- If a file write/edit fails, explicitly report the failure. -""" - -_CLOUD_SYSTEM_PROMPT = ( - _COMMON_PROMPT_HEADER - + """ -## Filesystem Tools - -All file paths must start with `/`. Relative paths resolve against the -current working directory (`cwd`, default `/documents`). - -- ls(path, offset=0, limit=200): list files and directories at the given path. -- read_file(path, offset, limit): read a file (paginated) from the filesystem. -- write_file(path, content): create a new text file in the workspace. -- edit_file(path, old, new): exact string-replacement edit (lazy-loads KB - documents on first edit). -- glob(pattern, path): find files matching a glob pattern. -- grep(pattern, path, glob): substring search across files. -- mkdir(path): create a folder under `/documents/` (committed at end of turn). -- cd(path): change the current working directory. -- pwd(): print the current working directory. -- move_file(source, dest): move/rename a file under `/documents/`. -- rm(path): delete a single file under `/documents/` (no `-r`). -- rmdir(path): delete an empty directory under `/documents/`. -- list_tree(path, max_depth, page_size): recursively list files/folders. - -## Persistence Rules - -- Files written under `/documents/<...>` are **persisted** at end of turn as - Documents in the user's knowledge base. -- Files whose **basename** starts with `temp_` (e.g. `temp_plan.md` or - `/documents/temp_scratch.md`) are **discarded** at end of turn — use this - prefix for any scratch/working content you do NOT want saved. -- All other paths (outside `/documents/` and not `temp_*`) are rejected. -- mkdir/move_file/rm/rmdir are staged this turn and committed at end of - turn alongside any new/edited documents. Snapshot/revert is enabled - for every destructive operation when action logging is on. - -## Reading Documents Efficiently - -Documents are formatted as XML. Each document contains: -- `` — title, type, URL, etc. -- `` — a table of every chunk with its **line range** and a - `matched="true"` flag for chunks that matched the search query. -- `` — the actual chunks in original document order. - -**Workflow**: when reading a large document, read the first ~20 lines to see -the ``, identify chunks marked `matched="true"`, then use -`read_file(path, offset=, limit=)` to jump directly to -those sections instead of reading the entire file sequentially. - -Use `` values as citation IDs in your answers. - -## Priority List - -You receive a `` system message each turn listing the -top-K paths most relevant to the user's query (by hybrid search). Read those -first — matched sections are flagged inside each document's ``. - -## Workspace Tree - -You receive a `` system message each turn with the current -folder/document layout. The tree may be truncated past a hard cap; in that -case, drill into specific folders with `ls(...)` or `list_tree(...)`. - -## grep Line Numbers - -`grep` searches across both your in-memory edits and the indexed chunks in -Postgres. State-cached files return real line numbers; database hits return -`line=0` because their position depends on per-document XML layout — call -`read_file(path)` to find the exact line. -""" -) - -_DESKTOP_SYSTEM_PROMPT = ( - _COMMON_PROMPT_HEADER - + """ -## Local Folder Mode - -This chat operates directly on the user's local folders. Writes and edits -hit disk immediately — there is no end-of-turn staging, no `/documents/` -namespace, and no `temp_` semantics. - -## Filesystem Tools - -All file paths must start with `/` and use mount-prefixed absolute paths -like `//file.ext`. Relative paths resolve against the current working -directory (`cwd`). - -- ls(path, offset=0, limit=200): list files and directories at the given path. -- read_file(path, offset, limit): read a file (paginated) from disk. -- write_file(path, content): write a file to disk. -- edit_file(path, old, new): exact string-replacement edit on disk. -- glob(pattern, path): find files matching a glob pattern. -- grep(pattern, path, glob): substring search across files. -- mkdir(path): create a directory on disk. -- cd(path): change the current working directory. -- pwd(): print the current working directory. -- move_file(source, dest): move/rename a file. -- rm(path): delete a single file from disk (no `-r`). NOT reversible. -- rmdir(path): delete an empty directory from disk. NOT reversible. -- list_tree(path, max_depth, page_size): recursively list files/folders. - -## Workflow Tips - -- If you are unsure which mounts are available, call `ls('/')` first. -- For large trees, prefer `list_tree` then `grep` then `read_file` over - brute-force directory traversal. -- Cross-mount moves are not supported. -- Desktop deletes hit disk immediately and cannot be undone via the - agent's revert flow — confirm before calling `rm`/`rmdir`. -""" -) - -_SANDBOX_PROMPT_ADDENDUM = ( - "\n- execute_code: run Python code in an isolated sandbox." - "\n\n## Code Execution" - "\n\nUse execute_code whenever a task benefits from running code." - " Never perform arithmetic manually." - "\n\nDocuments here are XML-wrapped markdown, not raw data files." - " To work with them programmatically, read the document first," - " extract the data, write it as a clean file (CSV, JSON, etc.)," - " and then run your code against it." -) - - -def _build_filesystem_system_prompt( - filesystem_mode: FilesystemMode, - *, - sandbox_available: bool, -) -> str: - """Build the filesystem system prompt for a given session mode. - - The prompt only describes rules and tools that actually apply in the - chosen mode — there is no cross-mode noise. - """ - base = ( - _CLOUD_SYSTEM_PROMPT - if filesystem_mode == FilesystemMode.CLOUD - else _DESKTOP_SYSTEM_PROMPT - ) - if sandbox_available: - base += _SANDBOX_PROMPT_ADDENDUM - return base - - -# Backwards-compatible alias retained for any external imports. -SURFSENSE_FILESYSTEM_SYSTEM_PROMPT = _CLOUD_SYSTEM_PROMPT - -# ============================================================================= -# Per-Tool Descriptions (shown to the LLM as the tool's docstring) -# ============================================================================= - -# ============================================================================= -# Per-Tool Descriptions (mode-specific; injected as the tool's docstring) -# ============================================================================= - -# --- mode-agnostic --------------------------------------------------------- - -SURFSENSE_READ_FILE_TOOL_DESCRIPTION = """Reads a file from the filesystem. - -Usage: -- By default, reads up to 100 lines from the beginning. -- Use `offset` and `limit` for pagination when files are large. -- Results include line numbers. -- Documents contain a `` near the top listing every chunk with - its line range and a `matched="true"` flag for search-relevant chunks. - Read the index first, then jump to matched chunks with - `read_file(path, offset=, limit=)`. -- Use chunk IDs (``) as citations in answers. -""" - -SURFSENSE_GLOB_TOOL_DESCRIPTION = """Find files matching a glob pattern. - -Supports standard glob patterns: `*`, `**`, `?`. -Returns absolute file paths. -""" - -SURFSENSE_CD_TOOL_DESCRIPTION = """Changes the current working directory (cwd). - -Args: -- path: absolute or relative directory path. Relative paths resolve against - the current cwd. - -The new cwd is used by other filesystem tools whenever a relative path is -given. Returns the resolved cwd. -""" - -SURFSENSE_PWD_TOOL_DESCRIPTION = """Prints the current working directory.""" - -SURFSENSE_EXECUTE_CODE_TOOL_DESCRIPTION = """Executes Python code in an isolated sandbox environment. - -Common data-science packages are pre-installed (pandas, numpy, matplotlib, -scipy, scikit-learn). - -Usage notes: -- No outbound network access. -- Returns combined stdout/stderr with exit code. -- Use print() to produce output. -- Use the optional timeout parameter to override the default timeout. -""" - -# --- cloud-only ------------------------------------------------------------ - -_CLOUD_LIST_FILES_TOOL_DESCRIPTION = """Lists files and directories at the given path. - -Usage: -- Provide an absolute path under `/documents` (relative paths resolve under - the current cwd, which defaults to `/documents`). -- For very large folders, use `offset` and `limit` to paginate the listing. -- Returns one entry per line; directories end with a trailing `/`. -""" - -_CLOUD_WRITE_FILE_TOOL_DESCRIPTION = """Writes a new text file to the workspace. - -Usage: -- Files written under `/documents/<...>` are persisted as Documents at end - of turn. -- Use a `temp_` filename prefix (e.g. `temp_plan.md` or `/documents/temp_x.md`) - for scratch/working files; they are automatically discarded at end of turn. -- Writes outside `/documents/` are rejected unless the basename starts with - `temp_`. -- Supported outputs include common LLM-friendly text formats like markdown, - json, yaml, csv, xml, html, css, sql, and code files. -- Avoid placeholders; produce concrete and useful text. -""" - -_CLOUD_EDIT_FILE_TOOL_DESCRIPTION = """Performs exact string replacements in files. - -IMPORTANT: -- Read the file before editing. -- Preserve exact indentation and formatting. -- Edits to documents under `/documents/` are persisted at end of turn. -- Edits to `temp_*` files are discarded at end of turn. -""" - -_CLOUD_MOVE_FILE_TOOL_DESCRIPTION = """Moves or renames a file or folder. - -Use absolute paths for both source and destination. - -Notes: -- `move_file` is staged this turn and committed at end of turn. -- The agent cannot overwrite an existing destination — pass a fresh dest - path or move the existing destination away first. -- The anonymous uploaded document is read-only and cannot be moved. -- Rename is a special case of move (same folder, different filename). -""" - -_CLOUD_LIST_TREE_TOOL_DESCRIPTION = """Lists files/folders recursively in a single bounded call. - -Args: -- path: absolute path to start from. Defaults to `/documents`. -- max_depth: recursion depth limit (default 8). -- page_size: maximum number of entries returned (max 1000). -- include_files / include_dirs: filter returned entry types. - -Returns JSON with: -- entries: [{path, is_dir, size, modified_at, depth}] -- truncated: true when additional entries were omitted due to page_size. -""" - -_CLOUD_GREP_TOOL_DESCRIPTION = """Search for a literal text pattern across files. - -Searches both your in-memory edits and the indexed chunks in Postgres. -State-cached file matches include real line numbers; database hits return -`line=0` because their position depends on per-document XML layout — call -`read_file(path)` afterwards to find the exact line. -""" - -_CLOUD_MKDIR_TOOL_DESCRIPTION = """Creates a directory under `/documents/`. - -Stages the folder for end-of-turn commit; the Folder row is inserted only -after the agent's turn finishes successfully. - -Args: -- path: absolute path of the new directory (must start with - `/documents/`). - -Notes: -- Parent folders are created as needed. -""" - -_CLOUD_RM_TOOL_DESCRIPTION = """Deletes a single file under `/documents/`. - -Mirrors POSIX `rm path` (no `-r`, no glob expansion). Stages the deletion -for end-of-turn commit; the row is removed only after the agent's turn -finishes successfully. - -Args: -- path: absolute or relative file path. Cannot point at a directory — use - `rmdir` for empty folders. Cannot target the root or `/documents`. - -Notes: -- The action is reversible via the per-action revert flow when action - logging is enabled. -- The anonymous uploaded document is read-only and cannot be deleted. -""" - -_CLOUD_RMDIR_TOOL_DESCRIPTION = """Deletes an empty directory under `/documents/`. - -Mirrors POSIX `rmdir path`: refuses non-empty directories. Recursive -deletion (`rm -r`) is intentionally NOT supported — clear contents with -`rm` first. - -Args: -- path: absolute or relative directory path. Cannot target the root, - `/documents`, the current cwd, or any ancestor of cwd (use `cd` to - move out first). - -Notes: -- Emptiness is evaluated against the post-staged view, so a same-turn - `rm /a/x.md` followed by `rmdir /a` is fine. -- If the directory was added in this same turn via `mkdir` and never - committed, the staged mkdir is dropped instead of issuing a delete. -- The action is reversible via the per-action revert flow when action - logging is enabled. -""" - -# --- desktop-only ---------------------------------------------------------- - -_DESKTOP_LIST_FILES_TOOL_DESCRIPTION = """Lists files and directories at the given path. - -Usage: -- Provide an absolute path using a mount prefix (e.g. `//sub/dir`). - Use `ls('/')` to discover available mounts. -- For very large folders, use `offset` and `limit` to paginate the listing. -- Returns one entry per line; directories end with a trailing `/`. -""" - -_DESKTOP_WRITE_FILE_TOOL_DESCRIPTION = """Writes a text file to disk. - -Usage: -- Use mount-prefixed absolute paths like `//sub/file.ext`. -- Writes hit disk immediately. There is no end-of-turn staging. -- Supported outputs include common LLM-friendly text formats like markdown, - json, yaml, csv, xml, html, css, sql, and code files. -- Avoid placeholders; produce concrete and useful text. -""" - -_DESKTOP_EDIT_FILE_TOOL_DESCRIPTION = """Performs exact string replacements in files on disk. - -IMPORTANT: -- Read the file before editing. -- Preserve exact indentation and formatting. -- Edits hit disk immediately. -""" - -_DESKTOP_MOVE_FILE_TOOL_DESCRIPTION = """Moves or renames a file or folder on disk. - -Use mount-prefixed absolute paths for both source and destination -(e.g. `//old.txt` -> `//new.txt`). - -Notes: -- Cross-mount moves are not supported. -- Rename is a special case of move (same folder, different filename). -""" - -_DESKTOP_LIST_TREE_TOOL_DESCRIPTION = """Lists files/folders recursively in a single bounded call. - -Args: -- path: absolute path to start from. Defaults to `/`. -- max_depth: recursion depth limit (default 8). -- page_size: maximum number of entries returned (max 1000). -- include_files / include_dirs: filter returned entry types. - -Returns JSON with: -- entries: [{path, is_dir, size, modified_at, depth}] -- truncated: true when additional entries were omitted due to page_size. -""" - -_DESKTOP_GREP_TOOL_DESCRIPTION = """Search for a literal text pattern across files. - -Searches files on disk and any in-memory edits. Returns real line numbers. -""" - -_DESKTOP_MKDIR_TOOL_DESCRIPTION = """Creates a directory on disk. - -Args: -- path: absolute mount-prefixed path of the new directory. - -Notes: -- Parent folders are created as needed. -""" - -_DESKTOP_RM_TOOL_DESCRIPTION = """Deletes a single file from disk. - -Mirrors POSIX `rm path` (no `-r`, no glob expansion). The deletion hits -disk immediately. Desktop deletes are NOT reversible via the agent's -revert flow. - -Args: -- path: absolute mount-prefixed file path. Cannot point at a directory — - use `rmdir` for empty folders. -""" - -_DESKTOP_RMDIR_TOOL_DESCRIPTION = """Deletes an empty directory from disk. - -Mirrors POSIX `rmdir path`: refuses non-empty directories. Recursive -deletion is NOT supported. The deletion hits disk immediately and is -NOT reversible via the agent's revert flow. - -Args: -- path: absolute mount-prefixed directory path. Cannot target the mount - root or any directory containing files/subfolders. -""" - - -def _build_tool_descriptions(filesystem_mode: FilesystemMode) -> dict[str, str]: - """Pick the active-mode description for every filesystem tool.""" - if filesystem_mode == FilesystemMode.CLOUD: - return { - "ls": _CLOUD_LIST_FILES_TOOL_DESCRIPTION, - "read_file": SURFSENSE_READ_FILE_TOOL_DESCRIPTION, - "write_file": _CLOUD_WRITE_FILE_TOOL_DESCRIPTION, - "edit_file": _CLOUD_EDIT_FILE_TOOL_DESCRIPTION, - "move_file": _CLOUD_MOVE_FILE_TOOL_DESCRIPTION, - "list_tree": _CLOUD_LIST_TREE_TOOL_DESCRIPTION, - "glob": SURFSENSE_GLOB_TOOL_DESCRIPTION, - "grep": _CLOUD_GREP_TOOL_DESCRIPTION, - "mkdir": _CLOUD_MKDIR_TOOL_DESCRIPTION, - "cd": SURFSENSE_CD_TOOL_DESCRIPTION, - "pwd": SURFSENSE_PWD_TOOL_DESCRIPTION, - "rm": _CLOUD_RM_TOOL_DESCRIPTION, - "rmdir": _CLOUD_RMDIR_TOOL_DESCRIPTION, - } - return { - "ls": _DESKTOP_LIST_FILES_TOOL_DESCRIPTION, - "read_file": SURFSENSE_READ_FILE_TOOL_DESCRIPTION, - "write_file": _DESKTOP_WRITE_FILE_TOOL_DESCRIPTION, - "edit_file": _DESKTOP_EDIT_FILE_TOOL_DESCRIPTION, - "move_file": _DESKTOP_MOVE_FILE_TOOL_DESCRIPTION, - "list_tree": _DESKTOP_LIST_TREE_TOOL_DESCRIPTION, - "glob": SURFSENSE_GLOB_TOOL_DESCRIPTION, - "grep": _DESKTOP_GREP_TOOL_DESCRIPTION, - "mkdir": _DESKTOP_MKDIR_TOOL_DESCRIPTION, - "cd": SURFSENSE_CD_TOOL_DESCRIPTION, - "pwd": SURFSENSE_PWD_TOOL_DESCRIPTION, - "rm": _DESKTOP_RM_TOOL_DESCRIPTION, - "rmdir": _DESKTOP_RMDIR_TOOL_DESCRIPTION, - } - - -# Backwards-compatible aliases retained for any external imports/tests that -# referenced the original CLOUD-flavoured constants. -SURFSENSE_LIST_FILES_TOOL_DESCRIPTION = _CLOUD_LIST_FILES_TOOL_DESCRIPTION -SURFSENSE_WRITE_FILE_TOOL_DESCRIPTION = _CLOUD_WRITE_FILE_TOOL_DESCRIPTION -SURFSENSE_EDIT_FILE_TOOL_DESCRIPTION = _CLOUD_EDIT_FILE_TOOL_DESCRIPTION -SURFSENSE_MOVE_FILE_TOOL_DESCRIPTION = _CLOUD_MOVE_FILE_TOOL_DESCRIPTION -SURFSENSE_LIST_TREE_TOOL_DESCRIPTION = _CLOUD_LIST_TREE_TOOL_DESCRIPTION -SURFSENSE_GREP_TOOL_DESCRIPTION = _CLOUD_GREP_TOOL_DESCRIPTION -SURFSENSE_MKDIR_TOOL_DESCRIPTION = _CLOUD_MKDIR_TOOL_DESCRIPTION - - -# ============================================================================= -# Helpers -# ============================================================================= - - -_TEMP_PREFIX = "temp_" - - -def _basename(path: str) -> str: - return path.rsplit("/", 1)[-1] - - -def _is_ancestor_of(candidate: str, target: str) -> bool: - """True iff ``candidate`` is a strict ancestor directory of ``target``. - - ``target`` itself is NOT considered an ancestor (use equality for that). - Both paths are assumed to be canonicalised, absolute, and free of - trailing slashes (except the root ``/``). - """ - if not candidate.startswith("/") or not target.startswith("/"): - return False - if candidate == target: - return False - prefix = candidate.rstrip("/") + "/" - return target.startswith(prefix) - - -class SurfSenseFilesystemMiddleware(FilesystemMiddleware): - """SurfSense-specific filesystem middleware (cloud + desktop).""" - - state_schema = SurfSenseFilesystemState - - _MAX_EXECUTE_TIMEOUT = 300 - - def __init__( - self, - *, - backend: Any = None, - filesystem_mode: FilesystemMode = FilesystemMode.CLOUD, - search_space_id: int | None = None, - created_by_id: str | None = None, - thread_id: int | str | None = None, - tool_token_limit_before_evict: int | None = 20000, - ) -> None: - self._filesystem_mode = filesystem_mode - self._search_space_id = search_space_id - self._created_by_id = created_by_id - self._thread_id = thread_id - self._sandbox_available = is_sandbox_enabled() and thread_id is not None - - # Build the prompt + tool descriptions for the active mode only — - # mixing both modes wastes tokens and confuses the model with rules - # it can't actually use this session. - system_prompt = _build_filesystem_system_prompt( - filesystem_mode, - sandbox_available=self._sandbox_available, - ) - - super().__init__( - backend=backend, - system_prompt=system_prompt, - custom_tool_descriptions=_build_tool_descriptions(filesystem_mode), - tool_token_limit_before_evict=tool_token_limit_before_evict, - max_execute_timeout=self._MAX_EXECUTE_TIMEOUT, - ) - self.tools = [t for t in self.tools if t.name != "execute"] - self.tools.append(self._create_mkdir_tool()) - self.tools.append(self._create_cd_tool()) - self.tools.append(self._create_pwd_tool()) - self.tools.append(self._create_move_file_tool()) - self.tools.append(self._create_rm_tool()) - self.tools.append(self._create_rmdir_tool()) - self.tools.append(self._create_list_tree_tool()) - if self._sandbox_available: - self.tools.append(self._create_execute_code_tool()) - - # ------------------------------------------------------------------ helpers - - def _is_cloud(self) -> bool: - return self._filesystem_mode == FilesystemMode.CLOUD - - @staticmethod - def _run_async_blocking(coro: Any) -> Any: - try: - loop = asyncio.get_running_loop() - if loop.is_running(): - return "Error: sync filesystem operation not supported inside an active event loop." - except RuntimeError: - pass - return asyncio.run(coro) - - @staticmethod - def _normalize_absolute_path(candidate: str) -> str: - normalized = re.sub(r"/+", "/", candidate.strip().replace("\\", "/")) - if not normalized: - return "/" - if normalized.startswith("/"): - return normalized - return f"/{normalized.lstrip('/')}" - - @staticmethod - def _extract_mount_from_path(path: str, mounts: tuple[str, ...]) -> str | None: - rel = path.lstrip("/") - if not rel: - return None - mount, _, _ = rel.partition("/") - if mount in mounts: - return mount - return None - - @staticmethod - def _local_parent_path(path: str) -> str: - rel = path.lstrip("/") - if "/" not in rel: - return "/" - parent = rel.rsplit("/", 1)[0].strip("/") - if not parent: - return "/" - return f"/{parent}" - - @staticmethod - def _path_exists_under_mount( - backend: MultiRootLocalFolderBackend, - mount: str, - local_path: str, - ) -> bool: - result = backend.list_tree( - f"/{mount}{local_path}", - max_depth=0, - page_size=1, - include_files=True, - include_dirs=True, - ) - return not bool(result.get("error")) - - def _normalize_local_mount_path( - self, - candidate: str, - runtime: ToolRuntime[None, SurfSenseFilesystemState], - ) -> str: - normalized = self._normalize_absolute_path(candidate) - backend = self._get_backend(runtime) - if not isinstance(backend, MultiRootLocalFolderBackend): - return normalized - - mounts = backend.list_mounts() - explicit_mount = self._extract_mount_from_path(normalized, mounts) - if explicit_mount: - return normalized - - if len(mounts) == 1: - return f"/{mounts[0]}{normalized}" - - suggested_mount: str | None = None - contract = runtime.state.get("file_operation_contract") or {} - suggested_path = contract.get("suggested_path") - if isinstance(suggested_path, str) and suggested_path.strip(): - normalized_suggested = self._normalize_absolute_path(suggested_path) - suggested_mount = self._extract_mount_from_path( - normalized_suggested, mounts - ) - - matching_mounts = [ - mount - for mount in mounts - if self._path_exists_under_mount(backend, mount, normalized) - ] - if len(matching_mounts) == 1: - return f"/{matching_mounts[0]}{normalized}" - - parent_path = self._local_parent_path(normalized) - if parent_path != "/": - parent_matching_mounts = [ - mount - for mount in mounts - if self._path_exists_under_mount(backend, mount, parent_path) - ] - if len(parent_matching_mounts) == 1: - return f"/{parent_matching_mounts[0]}{normalized}" - - if suggested_mount: - return f"/{suggested_mount}{normalized}" - - return f"/{backend.default_mount()}{normalized}" - - def _default_cwd(self) -> str: - return DOCUMENTS_ROOT if self._is_cloud() else "/" - - def _current_cwd(self, runtime: ToolRuntime[None, SurfSenseFilesystemState]) -> str: - cwd = runtime.state.get("cwd") if hasattr(runtime, "state") else None - if isinstance(cwd, str) and cwd.startswith("/"): - return cwd - return self._default_cwd() - - def _get_contract_suggested_path( - self, runtime: ToolRuntime[None, SurfSenseFilesystemState] - ) -> str: - contract = runtime.state.get("file_operation_contract") or {} - suggested = contract.get("suggested_path") - if isinstance(suggested, str) and suggested.strip(): - return self._normalize_absolute_path(suggested) - return self._default_cwd().rstrip("/") + "/notes.md" - - def _resolve_relative( - self, - path: str, - runtime: ToolRuntime[None, SurfSenseFilesystemState], - ) -> str: - candidate = path.strip() - if not candidate: - return self._current_cwd(runtime) - if candidate.startswith("/"): - return self._normalize_absolute_path(candidate) - cwd = self._current_cwd(runtime) - joined = posixpath.normpath(posixpath.join(cwd, candidate)) - return self._normalize_absolute_path(joined) - - def _resolve_write_target_path( - self, - file_path: str, - runtime: ToolRuntime[None, SurfSenseFilesystemState], - ) -> str: - candidate = file_path.strip() - if not candidate: - return self._get_contract_suggested_path(runtime) - if self._filesystem_mode == FilesystemMode.DESKTOP_LOCAL_FOLDER: - return self._normalize_local_mount_path(candidate, runtime) - return self._resolve_relative(candidate, runtime) - - def _resolve_move_target_path( - self, - file_path: str, - runtime: ToolRuntime[None, SurfSenseFilesystemState], - ) -> str: - candidate = file_path.strip() - if not candidate: - return "" - if self._filesystem_mode == FilesystemMode.DESKTOP_LOCAL_FOLDER: - return self._normalize_local_mount_path(candidate, runtime) - return self._resolve_relative(candidate, runtime) - - def _resolve_list_target_path( - self, - path: str, - runtime: ToolRuntime[None, SurfSenseFilesystemState], - ) -> str: - candidate = path.strip() or self._current_cwd(runtime) - if candidate == "/": - return "/" - if self._filesystem_mode == FilesystemMode.DESKTOP_LOCAL_FOLDER: - return self._normalize_local_mount_path(candidate, runtime) - return self._resolve_relative(candidate, runtime) - - # ------------------------------------------------------------------ namespace policy - - def _check_cloud_write_namespace( - self, - path: str, - runtime: ToolRuntime[None, SurfSenseFilesystemState], - ) -> str | None: - """Return an error string if cloud writes to ``path`` are not allowed. - - Order matters: - 1. Reject writes to the anonymous read-only doc. - 2. Allow ``/documents/*``. - 3. Allow ``temp_*`` basename anywhere. - 4. Reject everything else. - """ - if not self._is_cloud(): - return None - anon = runtime.state.get("kb_anon_doc") or {} - if isinstance(anon, dict): - anon_path = str(anon.get("path") or "") - if anon_path and anon_path == path: - return "Error: the anonymous uploaded document is read-only." - if path.startswith(DOCUMENTS_ROOT + "/") or path == DOCUMENTS_ROOT: - return None - if _basename(path).startswith(_TEMP_PREFIX): - return None - return ( - "Error: cloud writes must target /documents/<...> or use a 'temp_' " - f"basename for scratch (got '{path}')." - ) - - # ------------------------------------------------------------------ tool: ls - - def _create_ls_tool(self) -> BaseTool: - tool_description = ( - self._custom_tool_descriptions.get("ls") - or SURFSENSE_LIST_FILES_TOOL_DESCRIPTION - ) - - def sync_ls( - runtime: ToolRuntime[None, SurfSenseFilesystemState], - path: Annotated[ - str, - "Absolute path to the directory to list. Relative paths resolve against the current cwd.", - ] = "", - offset: Annotated[ - int, - "Number of entries to skip. Use for paginating large folders. Defaults to 0.", - ] = 0, - limit: Annotated[ - int, - "Maximum number of entries to return. Defaults to 200.", - ] = 200, - ) -> str: - return self._run_async_blocking( - async_ls(runtime, path=path, offset=offset, limit=limit) - ) - - async def async_ls( - runtime: ToolRuntime[None, SurfSenseFilesystemState], - path: Annotated[ - str, - "Absolute path to the directory to list. Relative paths resolve against the current cwd.", - ] = "", - offset: Annotated[ - int, - "Number of entries to skip. Use for paginating large folders. Defaults to 0.", - ] = 0, - limit: Annotated[ - int, - "Maximum number of entries to return. Defaults to 200.", - ] = 200, - ) -> str: - target = self._resolve_list_target_path(path, runtime) - try: - validated = validate_path(target) - except ValueError as exc: - return f"Error: {exc}" - if offset < 0: - offset = 0 - if limit < 1: - limit = 1 - backend = self._get_backend(runtime) - infos = await backend.als_info(validated) - page = paginate_listing(infos, offset=offset, limit=limit) - paths = [ - f"{fi.get('path', '')}/" if fi.get("is_dir") else fi.get("path", "") - for fi in page - ] - total = len(infos) - shown = len(page) - header = ( - f"{validated} ({shown} of {total} entries" - f"{f', offset={offset}' if offset else ''})" - ) - if not paths: - return f"{header}\n(empty)" - body = "\n".join(paths) - if total > offset + shown: - body += ( - f"\n... {total - offset - shown} more — call ls(" - f"'{validated}', offset={offset + shown}, limit={limit})" - ) - return f"{header}\n{body}" - - return StructuredTool.from_function( - name="ls", - description=tool_description, - func=sync_ls, - coroutine=async_ls, - ) - - # ------------------------------------------------------------------ tool: read_file - - def _create_read_file_tool(self) -> BaseTool: - tool_description = ( - self._custom_tool_descriptions.get("read_file") - or SURFSENSE_READ_FILE_TOOL_DESCRIPTION - ) - - async def async_read_file( - file_path: Annotated[ - str, - "Absolute path to the file to read. Relative paths resolve against the current cwd.", - ], - runtime: ToolRuntime[None, SurfSenseFilesystemState], - offset: Annotated[ - int, - "Line number to start reading from (0-indexed).", - ] = 0, - limit: Annotated[ - int, - "Maximum number of lines to read.", - ] = 100, - ) -> Command | str: - target = self._resolve_relative(file_path, runtime) - try: - validated = validate_path(target) - except ValueError as exc: - return f"Error: {exc}" - - files = runtime.state.get("files") or {} - if validated in files: - return format_read_response(files[validated], offset, limit) - - backend = self._get_backend(runtime) - if isinstance(backend, KBPostgresBackend): - loaded = await backend._load_file_data(validated) - if loaded is None: - return f"Error: File '{validated}' not found" - file_data, doc_id = loaded - rendered = format_read_response(file_data, offset, limit) - update: dict[str, Any] = { - "files": {validated: file_data}, - "messages": [ - ToolMessage( - content=rendered, - tool_call_id=runtime.tool_call_id, - ) - ], - } - if doc_id is not None: - update["doc_id_by_path"] = {validated: doc_id} - return Command(update=update) - - try: - rendered = await backend.aread(validated, offset=offset, limit=limit) - except Exception as exc: # pragma: no cover - defensive - return f"Error: {exc}" - return rendered - - def sync_read_file( - file_path: Annotated[ - str, - "Absolute path to the file to read. Relative paths resolve against the current cwd.", - ], - runtime: ToolRuntime[None, SurfSenseFilesystemState], - offset: Annotated[ - int, - "Line number to start reading from (0-indexed).", - ] = 0, - limit: Annotated[ - int, - "Maximum number of lines to read.", - ] = 100, - ) -> Command | str: - return self._run_async_blocking( - async_read_file(file_path, runtime, offset, limit) - ) - - return StructuredTool.from_function( - name="read_file", - description=tool_description, - func=sync_read_file, - coroutine=async_read_file, - ) - - # ------------------------------------------------------------------ tool: write_file - - def _create_write_file_tool(self) -> BaseTool: - tool_description = ( - self._custom_tool_descriptions.get("write_file") - or SURFSENSE_WRITE_FILE_TOOL_DESCRIPTION - ) - - async def async_write_file( - file_path: Annotated[ - str, - "Absolute path where the file should be created. Relative paths resolve against the current cwd.", - ], - content: Annotated[str, "Text content to write to the file."], - runtime: ToolRuntime[None, SurfSenseFilesystemState], - ) -> Command | str: - target = self._resolve_write_target_path(file_path, runtime) - try: - validated = validate_path(target) - except ValueError as exc: - return f"Error: {exc}" - - namespace_error = self._check_cloud_write_namespace(validated, runtime) - if namespace_error: - return namespace_error - - backend = self._get_backend(runtime) - res: WriteResult = await backend.awrite(validated, content) - if res.error: - return res.error - - path = res.path or validated - files_update = res.files_update or {path: create_file_data(content)} - update: dict[str, Any] = { - "files": files_update, - "messages": [ - ToolMessage( - content=f"Updated file {path}", - tool_call_id=runtime.tool_call_id, - ) - ], - } - if self._is_cloud(): - update["dirty_paths"] = [path] - update["dirty_path_tool_calls"] = {path: runtime.tool_call_id} - return Command(update=update) - - def sync_write_file( - file_path: Annotated[ - str, - "Absolute path where the file should be created. Relative paths resolve against the current cwd.", - ], - content: Annotated[str, "Text content to write to the file."], - runtime: ToolRuntime[None, SurfSenseFilesystemState], - ) -> Command | str: - return self._run_async_blocking( - async_write_file(file_path, content, runtime) - ) - - return StructuredTool.from_function( - name="write_file", - description=tool_description, - func=sync_write_file, - coroutine=async_write_file, - ) - - # ------------------------------------------------------------------ tool: edit_file - - def _create_edit_file_tool(self) -> BaseTool: - tool_description = ( - self._custom_tool_descriptions.get("edit_file") - or SURFSENSE_EDIT_FILE_TOOL_DESCRIPTION - ) - - async def async_edit_file( - file_path: Annotated[ - str, - "Absolute path to the file to edit. Relative paths resolve against the current cwd.", - ], - old_string: Annotated[ - str, - "Exact text to replace. Must be unique unless replace_all is True.", - ], - new_string: Annotated[ - str, - "Replacement text. Must differ from old_string.", - ], - runtime: ToolRuntime[None, SurfSenseFilesystemState], - *, - replace_all: Annotated[ - bool, - "If True, replace all occurrences of old_string. Defaults to False.", - ] = False, - ) -> Command | str: - target = self._resolve_relative(file_path, runtime) - try: - validated = validate_path(target) - except ValueError as exc: - return f"Error: {exc}" - - namespace_error = self._check_cloud_write_namespace(validated, runtime) - if namespace_error: - return namespace_error - - backend = self._get_backend(runtime) - files_state = runtime.state.get("files") or {} - doc_id_to_attach: int | None = None - - if ( - self._is_cloud() - and validated not in files_state - and isinstance(backend, KBPostgresBackend) - ): - loaded = await backend._load_file_data(validated) - if loaded is None: - return f"Error: File '{validated}' not found" - _, doc_id_to_attach = loaded - - res: EditResult = await backend.aedit( - validated, old_string, new_string, replace_all=replace_all - ) - if res.error: - return res.error - - path = res.path or validated - files_update = res.files_update or {} - update: dict[str, Any] = { - "files": files_update, - "messages": [ - ToolMessage( - content=( - f"Successfully replaced {res.occurrences} instance(s) " - f"of the string in '{path}'" - ), - tool_call_id=runtime.tool_call_id, - ) - ], - } - if self._is_cloud(): - update["dirty_paths"] = [path] - update["dirty_path_tool_calls"] = {path: runtime.tool_call_id} - if doc_id_to_attach is not None: - update["doc_id_by_path"] = {path: doc_id_to_attach} - return Command(update=update) - - def sync_edit_file( - file_path: Annotated[ - str, - "Absolute path to the file to edit. Relative paths resolve against the current cwd.", - ], - old_string: Annotated[ - str, - "Exact text to replace. Must be unique unless replace_all is True.", - ], - new_string: Annotated[ - str, - "Replacement text. Must differ from old_string.", - ], - runtime: ToolRuntime[None, SurfSenseFilesystemState], - *, - replace_all: Annotated[ - bool, - "If True, replace all occurrences of old_string. Defaults to False.", - ] = False, - ) -> Command | str: - return self._run_async_blocking( - async_edit_file( - file_path, old_string, new_string, runtime, replace_all=replace_all - ) - ) - - return StructuredTool.from_function( - name="edit_file", - description=tool_description, - func=sync_edit_file, - coroutine=async_edit_file, - ) - - # ------------------------------------------------------------------ tool: mkdir - - def _create_mkdir_tool(self) -> BaseTool: - tool_description = ( - self._custom_tool_descriptions.get("mkdir") - or SURFSENSE_MKDIR_TOOL_DESCRIPTION - ) - - async def async_mkdir( - path: Annotated[str, "Absolute or relative directory path to create."], - runtime: ToolRuntime[None, SurfSenseFilesystemState], - ) -> Command | str: - target = self._resolve_relative(path, runtime) - try: - validated = validate_path(target) - except ValueError as exc: - return f"Error: {exc}" - - if self._is_cloud(): - if not ( - validated.startswith(DOCUMENTS_ROOT + "/") - or validated == DOCUMENTS_ROOT - ): - return ( - "Error: cloud mkdir must target a path under /documents/ " - f"(got '{validated}')." - ) - return Command( - update={ - "staged_dirs": [validated], - "staged_dir_tool_calls": { - validated: runtime.tool_call_id, - }, - "messages": [ - ToolMessage( - content=( - f"Staged directory '{validated}' (will be created " - "at end of turn)." - ), - tool_call_id=runtime.tool_call_id, - ) - ], - } - ) - - backend = self._get_backend(runtime) - local_method = getattr(backend, "amkdir", None) or getattr( - backend, "mkdir", None - ) - if callable(local_method): - try: - res = local_method(validated, parents=True, exist_ok=True) - if asyncio.iscoroutine(res): - await res - except TypeError: - res = local_method(validated) - if asyncio.iscoroutine(res): - await res - except Exception as exc: # pragma: no cover - return f"Error: {exc}" - return f"Created directory {validated}" - - def sync_mkdir( - path: Annotated[str, "Absolute or relative directory path to create."], - runtime: ToolRuntime[None, SurfSenseFilesystemState], - ) -> Command | str: - return self._run_async_blocking(async_mkdir(path, runtime)) - - return StructuredTool.from_function( - name="mkdir", - description=tool_description, - func=sync_mkdir, - coroutine=async_mkdir, - ) - - # ------------------------------------------------------------------ tool: cd - - def _create_cd_tool(self) -> BaseTool: - tool_description = ( - self._custom_tool_descriptions.get("cd") or SURFSENSE_CD_TOOL_DESCRIPTION - ) - - async def async_cd( - path: Annotated[str, "Absolute or relative directory path to switch into."], - runtime: ToolRuntime[None, SurfSenseFilesystemState], - ) -> Command | str: - target = self._resolve_relative(path, runtime) - try: - validated = validate_path(target) - except ValueError as exc: - return f"Error: {exc}" - - backend = self._get_backend(runtime) - try: - infos = await backend.als_info(validated) - except Exception as exc: # pragma: no cover - defensive - return f"Error: {exc}" - staged_dirs = list(runtime.state.get("staged_dirs") or []) - files = runtime.state.get("files") or {} - cwd_exists = ( - bool(infos) - or validated in staged_dirs - or any(p == validated for p in files) - or any( - isinstance(p, str) and p.startswith(validated.rstrip("/") + "/") - for p in files - ) - or validated == "/" - or validated == DOCUMENTS_ROOT - ) - if not cwd_exists: - return f"Error: directory '{validated}' not found." - return Command( - update={ - "cwd": validated, - "messages": [ - ToolMessage( - content=f"cwd changed to {validated}", - tool_call_id=runtime.tool_call_id, - ) - ], - } - ) - - def sync_cd( - path: Annotated[str, "Absolute or relative directory path to switch into."], - runtime: ToolRuntime[None, SurfSenseFilesystemState], - ) -> Command | str: - return self._run_async_blocking(async_cd(path, runtime)) - - return StructuredTool.from_function( - name="cd", - description=tool_description, - func=sync_cd, - coroutine=async_cd, - ) - - # ------------------------------------------------------------------ tool: pwd - - def _create_pwd_tool(self) -> BaseTool: - tool_description = ( - self._custom_tool_descriptions.get("pwd") or SURFSENSE_PWD_TOOL_DESCRIPTION - ) - - def sync_pwd( - runtime: ToolRuntime[None, SurfSenseFilesystemState], - ) -> str: - return self._current_cwd(runtime) - - async def async_pwd( - runtime: ToolRuntime[None, SurfSenseFilesystemState], - ) -> str: - return self._current_cwd(runtime) - - return StructuredTool.from_function( - name="pwd", - description=tool_description, - func=sync_pwd, - coroutine=async_pwd, - ) - - # ------------------------------------------------------------------ tool: move_file - - def _create_move_file_tool(self) -> BaseTool: - tool_description = ( - self._custom_tool_descriptions.get("move_file") - or SURFSENSE_MOVE_FILE_TOOL_DESCRIPTION - ) - - async def async_move_file( - source_path: Annotated[str, "Absolute or relative source path."], - destination_path: Annotated[str, "Absolute or relative destination path."], - runtime: ToolRuntime[None, SurfSenseFilesystemState], - *, - overwrite: Annotated[ - bool, - "If True, replace existing destination. Cloud mode rejects True. Defaults to False.", - ] = False, - ) -> Command | str: - if not source_path.strip() or not destination_path.strip(): - return "Error: source_path and destination_path are required." - - source = self._resolve_move_target_path(source_path, runtime) - dest = self._resolve_move_target_path(destination_path, runtime) - try: - validated_source = validate_path(source) - validated_dest = validate_path(dest) - except ValueError as exc: - return f"Error: {exc}" - - if self._is_cloud(): - return await self._cloud_move_file( - runtime, - validated_source, - validated_dest, - overwrite=overwrite, - ) - - backend = self._get_backend(runtime) - res: WriteResult = await backend.amove( - validated_source, validated_dest, overwrite=overwrite - ) - if res.error: - return res.error - update: dict[str, Any] = { - "messages": [ - ToolMessage( - content=f"Moved '{validated_source}' to '{res.path or validated_dest}'", - tool_call_id=runtime.tool_call_id, - ) - ], - } - if res.files_update is not None: - update["files"] = res.files_update - return Command(update=update) - - def sync_move_file( - source_path: Annotated[str, "Absolute or relative source path."], - destination_path: Annotated[str, "Absolute or relative destination path."], - runtime: ToolRuntime[None, SurfSenseFilesystemState], - *, - overwrite: Annotated[ - bool, - "If True, replace existing destination. Cloud mode rejects True. Defaults to False.", - ] = False, - ) -> Command | str: - return self._run_async_blocking( - async_move_file( - source_path, destination_path, runtime, overwrite=overwrite - ) - ) - - return StructuredTool.from_function( - name="move_file", - description=tool_description, - func=sync_move_file, - coroutine=async_move_file, - ) - - async def _cloud_move_file( - self, - runtime: ToolRuntime[None, SurfSenseFilesystemState], - source: str, - dest: str, - *, - overwrite: bool, - ) -> Command | str: - backend = self._get_backend(runtime) - if not isinstance(backend, KBPostgresBackend): - return "Error: cloud move requires KBPostgresBackend." - - if source == dest: - return f"Moved '{source}' to '{dest}' (no-op)" - if overwrite: - return ( - "Error: overwrite=True is not supported in cloud mode. Move/edit " - "the destination doc explicitly first." - ) - if not source.startswith(DOCUMENTS_ROOT + "/"): - return ( - "Error: cloud move_file source must be under /documents/ (got " - f"'{source}')." - ) - if not dest.startswith(DOCUMENTS_ROOT + "/"): - return ( - "Error: cloud move_file destination must be under /documents/ (got " - f"'{dest}')." - ) - anon = runtime.state.get("kb_anon_doc") or {} - if isinstance(anon, dict): - anon_path = str(anon.get("path") or "") - if anon_path and (anon_path in (source, dest)): - return "Error: the anonymous uploaded document is read-only." - - files = runtime.state.get("files") or {} - doc_id_by_path = runtime.state.get("doc_id_by_path") or {} - pending_moves = list(runtime.state.get("pending_moves") or []) - - # Dest collision: occupied in state, in pending moves, or in DB. - if dest in files: - return f"Error: destination '{dest}' already exists." - if any(move.get("dest") == dest for move in pending_moves): - return f"Error: destination '{dest}' already exists." - if dest != source: - existing_dest = await backend._load_file_data(dest) - if existing_dest is not None: - return f"Error: destination '{dest}' already exists." - - # Source materialization: lazy load if not in state. - source_file_data = files.get(source) - source_doc_id = doc_id_by_path.get(source) - if source_file_data is None: - loaded = await backend._load_file_data(source) - if loaded is None: - return f"Error: source '{source}' not found." - source_file_data, loaded_doc_id = loaded - if source_doc_id is None: - source_doc_id = loaded_doc_id - - files_update: dict[str, Any] = {source: None, dest: source_file_data} - update: dict[str, Any] = { - "files": files_update, - "pending_moves": [ - { - "source": source, - "dest": dest, - "overwrite": False, - "tool_call_id": runtime.tool_call_id, - } - ], - "messages": [ - ToolMessage( - content=( - f"Moved '{source}' to '{dest}' (will commit at end of turn)." - ), - tool_call_id=runtime.tool_call_id, - ) - ], - } - - doc_id_update: dict[str, int | None] = {source: None} - if source_doc_id is not None: - doc_id_update[dest] = source_doc_id - update["doc_id_by_path"] = doc_id_update - - dirty_paths = list(runtime.state.get("dirty_paths") or []) - if source in dirty_paths: - new_dirty: list[Any] = [_CLEAR] - for entry in dirty_paths: - new_dirty.append(dest if entry == source else entry) - update["dirty_paths"] = new_dirty - return Command(update=update) - - # ------------------------------------------------------------------ tool: rm - - def _create_rm_tool(self) -> BaseTool: - tool_description = ( - self._custom_tool_descriptions.get("rm") or _CLOUD_RM_TOOL_DESCRIPTION - ) - - async def async_rm( - path: Annotated[ - str, - "Absolute or relative path to the file to delete.", - ], - runtime: ToolRuntime[None, SurfSenseFilesystemState], - ) -> Command | str: - if not path or not path.strip(): - return "Error: path is required." - - target = self._resolve_relative(path, runtime) - try: - validated = validate_path(target) - except ValueError as exc: - return f"Error: {exc}" - - if self._is_cloud(): - if validated in ("/", DOCUMENTS_ROOT): - return f"Error: refusing to rm '{validated}'." - if not validated.startswith(DOCUMENTS_ROOT + "/"): - return ( - "Error: cloud rm must target a path under /documents/ " - f"(got '{validated}')." - ) - - anon = runtime.state.get("kb_anon_doc") or {} - if isinstance(anon, dict) and str(anon.get("path") or "") == validated: - return "Error: the anonymous uploaded document is read-only." - - # Refuse if the path looks like a directory. - staged_dirs = list(runtime.state.get("staged_dirs") or []) - if validated in staged_dirs: - return ( - f"Error: '{validated}' is a directory. Use rmdir for " - "empty directories." - ) - pending_dir_deletes = list( - runtime.state.get("pending_dir_deletes") or [] - ) - if any( - isinstance(d, dict) and d.get("path") == validated - for d in pending_dir_deletes - ): - return f"Error: '{validated}' is already queued for rmdir." - - backend = self._get_backend(runtime) - if isinstance(backend, KBPostgresBackend): - # Detect "is a directory" via `ls`: if the path lists - # children we know it's a folder. Otherwise we still - # need to confirm it's a real file before staging. - children = await backend.als_info(validated) - if children: - return ( - f"Error: '{validated}' is a directory. Use rmdir for " - "empty directories." - ) - - # Already queued for delete this turn? - pending_deletes = list(runtime.state.get("pending_deletes") or []) - if any( - isinstance(d, dict) and d.get("path") == validated - for d in pending_deletes - ): - return f"'{validated}' is already queued for deletion." - - # Resolve doc_id (best-effort): file in state or DB. - files_state = runtime.state.get("files") or {} - doc_id_by_path = runtime.state.get("doc_id_by_path") or {} - resolved_doc_id: int | None = doc_id_by_path.get(validated) - if ( - validated not in files_state - and resolved_doc_id is None - and isinstance(backend, KBPostgresBackend) - ): - loaded = await backend._load_file_data(validated) - if loaded is None: - return f"Error: file '{validated}' not found." - _, resolved_doc_id = loaded - - files_update: dict[str, Any] = {validated: None} - update: dict[str, Any] = { - "pending_deletes": [ - { - "path": validated, - "tool_call_id": runtime.tool_call_id, - } - ], - "files": files_update, - "doc_id_by_path": {validated: None}, - "messages": [ - ToolMessage( - content=( - f"Staged delete of '{validated}' (will commit at " - "end of turn)." - ), - tool_call_id=runtime.tool_call_id, - ) - ], - } - - # Drop the path from dirty_paths so a same-turn write+rm - # doesn't recreate the doc at commit time. - dirty_paths = list(runtime.state.get("dirty_paths") or []) - if validated in dirty_paths: - new_dirty: list[Any] = [_CLEAR] - for entry in dirty_paths: - if entry != validated: - new_dirty.append(entry) - update["dirty_paths"] = new_dirty - update["dirty_path_tool_calls"] = {validated: None} - - return Command(update=update) - - # Desktop mode — hit disk immediately. - backend = self._get_backend(runtime) - adelete = getattr(backend, "adelete_file", None) - if not callable(adelete): - return "Error: rm is not supported by the active backend." - res: WriteResult = await adelete(validated) - if res.error: - return res.error - update_desktop: dict[str, Any] = { - "files": {validated: None}, - "messages": [ - ToolMessage( - content=f"Deleted file '{res.path or validated}'", - tool_call_id=runtime.tool_call_id, - ) - ], - } - return Command(update=update_desktop) - - def sync_rm( - path: Annotated[ - str, - "Absolute or relative path to the file to delete.", - ], - runtime: ToolRuntime[None, SurfSenseFilesystemState], - ) -> Command | str: - return self._run_async_blocking(async_rm(path, runtime)) - - return StructuredTool.from_function( - name="rm", - description=tool_description, - func=sync_rm, - coroutine=async_rm, - ) - - # ------------------------------------------------------------------ tool: rmdir - - def _create_rmdir_tool(self) -> BaseTool: - tool_description = ( - self._custom_tool_descriptions.get("rmdir") or _CLOUD_RMDIR_TOOL_DESCRIPTION - ) - - async def async_rmdir( - path: Annotated[ - str, - "Absolute or relative path of the empty directory to delete.", - ], - runtime: ToolRuntime[None, SurfSenseFilesystemState], - ) -> Command | str: - if not path or not path.strip(): - return "Error: path is required." - - target = self._resolve_relative(path, runtime) - try: - validated = validate_path(target) - except ValueError as exc: - return f"Error: {exc}" - - if self._is_cloud(): - if validated in ("/", DOCUMENTS_ROOT): - return f"Error: refusing to rmdir '{validated}'." - if not validated.startswith(DOCUMENTS_ROOT + "/"): - return ( - "Error: cloud rmdir must target a path under /documents/ " - f"(got '{validated}')." - ) - - cwd = self._current_cwd(runtime) - if validated == cwd or _is_ancestor_of(validated, cwd): - return ( - f"Error: cannot rmdir '{validated}' because the current " - "cwd is at or under it. cd out first." - ) - - staged_dirs = list(runtime.state.get("staged_dirs") or []) - pending_dir_deletes = list( - runtime.state.get("pending_dir_deletes") or [] - ) - if any( - isinstance(d, dict) and d.get("path") == validated - for d in pending_dir_deletes - ): - return f"'{validated}' is already queued for deletion." - - backend = self._get_backend(runtime) - - # The path must currently exist either in DB folder paths or - # in staged_dirs. We rely on KBPostgresBackend.als_info (which - # already accounts for pending deletes/moves) to evaluate - # both existence and emptiness against the post-staged view. - exists_in_staged = validated in staged_dirs - children: list[Any] = [] - if isinstance(backend, KBPostgresBackend): - children = list(await backend.als_info(validated)) - - # Detect "is a file" — if als_info returns no children but - # the path is actually a file, we should reject. We use - # _load_file_data to disambiguate file vs missing folder. - if ( - isinstance(backend, KBPostgresBackend) - and not children - and not exists_in_staged - ): - loaded = await backend._load_file_data(validated) - if loaded is not None: - return ( - f"Error: '{validated}' is a file. Use rm to delete files." - ) - # Confirm folder exists in DB by checking the parent listing. - parent = posixpath.dirname(validated) or "/" - parent_listing = await backend.als_info(parent) - parent_has_dir = any( - info.get("path") == validated and info.get("is_dir") - for info in parent_listing - ) - if not parent_has_dir: - return f"Error: directory '{validated}' not found." - - if children: - return ( - f"Error: directory '{validated}' is not empty. " - "Remove contents first." - ) - - # Same-turn mkdir un-stage: drop the staged_dirs entry - # entirely and skip queuing a DB delete (nothing was ever - # committed). - if exists_in_staged: - rest = [d for d in staged_dirs if d != validated] - return Command( - update={ - "staged_dirs": [_CLEAR, *rest], - "staged_dir_tool_calls": {validated: None}, - "messages": [ - ToolMessage( - content=(f"Un-staged directory '{validated}'."), - tool_call_id=runtime.tool_call_id, - ) - ], - } - ) - - return Command( - update={ - "pending_dir_deletes": [ - { - "path": validated, - "tool_call_id": runtime.tool_call_id, - } - ], - "messages": [ - ToolMessage( - content=( - f"Staged rmdir of '{validated}' (will commit " - "at end of turn)." - ), - tool_call_id=runtime.tool_call_id, - ) - ], - } - ) - - # Desktop mode — hit disk immediately. - backend = self._get_backend(runtime) - armdir = getattr(backend, "armdir", None) - if not callable(armdir): - return "Error: rmdir is not supported by the active backend." - res: WriteResult = await armdir(validated) - if res.error: - return res.error - return Command( - update={ - "messages": [ - ToolMessage( - content=f"Deleted directory '{res.path or validated}'", - tool_call_id=runtime.tool_call_id, - ) - ], - } - ) - - def sync_rmdir( - path: Annotated[ - str, - "Absolute or relative path of the empty directory to delete.", - ], - runtime: ToolRuntime[None, SurfSenseFilesystemState], - ) -> Command | str: - return self._run_async_blocking(async_rmdir(path, runtime)) - - return StructuredTool.from_function( - name="rmdir", - description=tool_description, - func=sync_rmdir, - coroutine=async_rmdir, - ) - - # ------------------------------------------------------------------ tool: list_tree - - def _create_list_tree_tool(self) -> BaseTool: - tool_description = ( - self._custom_tool_descriptions.get("list_tree") - or SURFSENSE_LIST_TREE_TOOL_DESCRIPTION - ) - - async def async_list_tree( - runtime: ToolRuntime[None, SurfSenseFilesystemState], - path: Annotated[ - str, - "Absolute path to start from. Defaults to /documents in cloud mode.", - ] = "", - max_depth: Annotated[int, "Recursion depth limit. Default 8."] = 8, - page_size: Annotated[int, "Maximum entries returned. Max 1000."] = 500, - include_files: Annotated[bool, "Include file entries."] = True, - include_dirs: Annotated[bool, "Include directory entries."] = True, - ) -> str: - if max_depth < 0: - return "Error: max_depth must be >= 0." - if page_size < 1: - return "Error: page_size must be >= 1." - if not include_files and not include_dirs: - return "Error: include_files and include_dirs cannot both be false." - - target = self._resolve_list_target_path(path, runtime) - try: - validated = validate_path(target) - except ValueError as exc: - return f"Error: {exc}" - - backend = self._get_backend(runtime) - if isinstance(backend, KBPostgresBackend): - result = await backend.alist_tree_listing( - validated, - max_depth=max_depth, - page_size=page_size, - include_files=include_files, - include_dirs=include_dirs, - ) - elif hasattr(backend, "alist_tree"): - result = await backend.alist_tree( - validated, - max_depth=max_depth, - page_size=page_size, - include_files=include_files, - include_dirs=include_dirs, - ) - else: - return "Error: list_tree is not supported by the active backend." - - if isinstance(result, dict) and isinstance(result.get("error"), str): - return result["error"] - return json.dumps(result, ensure_ascii=True) - - def sync_list_tree( - runtime: ToolRuntime[None, SurfSenseFilesystemState], - path: Annotated[ - str, - "Absolute path to start from. Defaults to /documents in cloud mode.", - ] = "", - max_depth: Annotated[int, "Recursion depth limit. Default 8."] = 8, - page_size: Annotated[int, "Maximum entries returned. Max 1000."] = 500, - include_files: Annotated[bool, "Include file entries."] = True, - include_dirs: Annotated[bool, "Include directory entries."] = True, - ) -> str: - return self._run_async_blocking( - async_list_tree( - runtime, - path=path, - max_depth=max_depth, - page_size=page_size, - include_files=include_files, - include_dirs=include_dirs, - ) - ) - - return StructuredTool.from_function( - name="list_tree", - description=tool_description, - func=sync_list_tree, - coroutine=async_list_tree, - ) - - # ------------------------------------------------------------------ tool: execute_code (sandbox) - - def _create_execute_code_tool(self) -> BaseTool: - def sync_execute_code( - command: Annotated[ - str, "Python code to execute. Use print() to see output." - ], - runtime: ToolRuntime[None, SurfSenseFilesystemState], - timeout: Annotated[ - int | None, - "Optional timeout in seconds.", - ] = None, - ) -> str: - if timeout is not None: - if timeout < 0: - return f"Error: timeout must be non-negative, got {timeout}." - if timeout > self._MAX_EXECUTE_TIMEOUT: - return f"Error: timeout {timeout}s exceeds maximum ({self._MAX_EXECUTE_TIMEOUT}s)." - return self._run_async_blocking( - self._execute_in_sandbox(command, runtime, timeout) - ) - - async def async_execute_code( - command: Annotated[ - str, "Python code to execute. Use print() to see output." - ], - runtime: ToolRuntime[None, SurfSenseFilesystemState], - timeout: Annotated[ - int | None, - "Optional timeout in seconds.", - ] = None, - ) -> str: - if timeout is not None: - if timeout < 0: - return f"Error: timeout must be non-negative, got {timeout}." - if timeout > self._MAX_EXECUTE_TIMEOUT: - return f"Error: timeout {timeout}s exceeds maximum ({self._MAX_EXECUTE_TIMEOUT}s)." - return await self._execute_in_sandbox(command, runtime, timeout) - - return StructuredTool.from_function( - name="execute_code", - description=SURFSENSE_EXECUTE_CODE_TOOL_DESCRIPTION, - func=sync_execute_code, - coroutine=async_execute_code, - ) - - @staticmethod - def _wrap_as_python(code: str) -> str: - sentinel = f"_PYEOF_{secrets.token_hex(8)}" - return f"python3 << '{sentinel}'\n{code}\n{sentinel}" - - async def _execute_in_sandbox( - self, - command: str, - runtime: ToolRuntime[None, SurfSenseFilesystemState], - timeout: int | None, - ) -> str: - assert self._thread_id is not None - command = self._wrap_as_python(command) - try: - return await self._try_sandbox_execute(command, runtime, timeout) - except (DaytonaError, Exception) as first_err: - logger.warning( - "Sandbox execute failed for thread %s, retrying: %s", - self._thread_id, - first_err, - ) - try: - await delete_sandbox(self._thread_id) - except Exception: - _evict_sandbox_cache(self._thread_id) - try: - return await self._try_sandbox_execute(command, runtime, timeout) - except Exception: - logger.exception( - "Sandbox retry also failed for thread %s", self._thread_id - ) - return "Error: Code execution is temporarily unavailable. Please try again." - - async def _try_sandbox_execute( - self, - command: str, - runtime: ToolRuntime[None, SurfSenseFilesystemState], - timeout: int | None, - ) -> str: - sandbox, _is_new = await get_or_create_sandbox(self._thread_id) - result = await sandbox.aexecute(command, timeout=timeout) - output = (result.output or "").strip() - if not output and result.exit_code == 0: - return ( - "[Code executed successfully but produced no output. " - "Use print() to display results, then try again.]" - ) - parts = [result.output] - if result.exit_code is not None: - status = "succeeded" if result.exit_code == 0 else "failed" - parts.append(f"\n[Command {status} with exit code {result.exit_code}]") - if result.truncated: - parts.append("\n[Output was truncated due to size limits]") - return "".join(parts) diff --git a/surfsense_backend/tests/integration/agents/multi_agent_chat/test_kb_filesystem_cloud.py b/surfsense_backend/tests/integration/agents/multi_agent_chat/test_kb_filesystem_cloud.py new file mode 100644 index 000000000..93b534504 --- /dev/null +++ b/surfsense_backend/tests/integration/agents/multi_agent_chat/test_kb_filesystem_cloud.py @@ -0,0 +1,198 @@ +"""Real-behavior tests for the LIVE knowledge-base filesystem middleware (B) in +cloud mode. + +Cloud mode is the default production filesystem for web chat. Unlike desktop, +cloud writes/edits/moves/deletes are *staged* into LangGraph state during the +turn and committed to Postgres at end-of-turn by the persistence middleware. +These tests drive the production ``build_filesystem_mw`` cloud tools through a +real ``create_agent`` graph and assert the staging contract (namespace policy, +read-from-stage, mkdir staging, duplicate rejection) — all deterministic and +DB-free because cloud ``awrite`` is pure in-state staging. + +The end-of-turn DB commit (``commit_staged_filesystem_state``) is covered +separately; here we lock the per-tool behavior that the reorg could break. +""" + +from __future__ import annotations + +import pytest +from langchain.agents import create_agent +from langchain_core.messages import HumanMessage, ToolMessage +from langgraph.checkpoint.memory import InMemorySaver + +from app.agents.multi_agent_chat.middleware.shared.filesystem import ( + build_filesystem_mw, +) +from app.agents.shared.filesystem_backends import build_backend_resolver +from app.agents.shared.filesystem_selection import FilesystemMode, FilesystemSelection +from tests.integration.harness import ScriptedTurn, build_scripted_harness + +pytestmark = [pytest.mark.integration, pytest.mark.asyncio] + +_SEARCH_SPACE_ID = 1 + + +def _build_cloud_fs_mw(): + """Build the production filesystem middleware in cloud mode. + + A non-None ``search_space_id`` makes the resolver hand out a + ``KBPostgresBackend``, exactly as production does. Staging operations never + touch the DB, so a dummy id is sufficient for these tests. + """ + selection = FilesystemSelection(mode=FilesystemMode.CLOUD) + resolver = build_backend_resolver(selection, search_space_id=_SEARCH_SPACE_ID) + return build_filesystem_mw( + backend_resolver=resolver, + filesystem_mode=FilesystemMode.CLOUD, + search_space_id=_SEARCH_SPACE_ID, + user_id="00000000-0000-0000-0000-000000000001", + thread_id=_SEARCH_SPACE_ID, + read_only=False, + ) + + +async def _run(turns: list[ScriptedTurn], thread: str): + harness = build_scripted_harness(turns=turns) + agent = create_agent( + harness.model, + tools=[], + middleware=[_build_cloud_fs_mw()], + checkpointer=InMemorySaver(), + ) + return await agent.ainvoke( + {"messages": [HumanMessage(content="do kb work")]}, + config={"configurable": {"thread_id": thread}}, + ) + + +def _tool_text(result, name: str) -> str: + for m in result["messages"]: + if isinstance(m, ToolMessage) and m.name == name: + return str(m.content) + raise AssertionError(f"no ToolMessage from {name!r}") + + +def _write(path: str, content: str, call_id: str) -> ScriptedTurn: + return ScriptedTurn( + tool_calls=[ + { + "name": "write_file", + "args": {"file_path": path, "content": content}, + "id": call_id, + } + ] + ) + + +async def test_cloud_write_then_read_returns_staged_content(): + """A cloud write stages into state and a later read returns that content.""" + result = await _run( + [ + _write("/documents/note.md", "cloud CANARY-CLD-1", "c1"), + ScriptedTurn( + tool_calls=[ + { + "name": "read_file", + "args": {"file_path": "/documents/note.md"}, + "id": "c2", + } + ] + ), + ScriptedTurn(text="done"), + ], + "fs-cloud-write-read", + ) + + assert "Updated file /documents/note.md" in _tool_text(result, "write_file") + assert "CANARY-CLD-1" in _tool_text(result, "read_file") + + +async def test_cloud_write_outside_documents_is_rejected(): + """Cloud namespace policy: writes must target /documents (non-temp paths).""" + result = await _run( + [ + _write("/scratch/note.md", "nope", "c1"), + ScriptedTurn(text="done"), + ], + "fs-cloud-namespace", + ) + + msg = _tool_text(result, "write_file") + assert "must target /documents" in msg + + +async def test_cloud_temp_prefixed_write_is_allowed_anywhere(): + """A ``temp_`` basename escapes the /documents namespace restriction.""" + result = await _run( + [ + _write("/temp_scratch.md", "ephemeral", "c1"), + ScriptedTurn(text="done"), + ], + "fs-cloud-temp", + ) + + msg = _tool_text(result, "write_file") + assert "must target /documents" not in msg + assert "Updated file" in msg + + +async def test_cloud_mkdir_stages_directory(): + """Cloud mkdir stages the directory for end-of-turn creation (no immediate IO).""" + result = await _run( + [ + ScriptedTurn( + tool_calls=[ + { + "name": "mkdir", + "args": {"path": "/documents/projects"}, + "id": "c1", + } + ] + ), + ScriptedTurn(text="done"), + ], + "fs-cloud-mkdir", + ) + + msg = _tool_text(result, "mkdir") + assert "Staged directory" in msg + assert "/documents/projects" in msg + + +async def test_cloud_mkdir_outside_documents_is_rejected(): + """Cloud mkdir is also restricted to the /documents namespace.""" + result = await _run( + [ + ScriptedTurn( + tool_calls=[ + {"name": "mkdir", "args": {"path": "/elsewhere"}, "id": "c1"} + ] + ), + ScriptedTurn(text="done"), + ], + "fs-cloud-mkdir-bad", + ) + + assert "must target a path under /documents" in _tool_text(result, "mkdir") + + +async def test_cloud_duplicate_write_is_rejected(): + """Writing to a path already staged this turn is rejected (use edit instead).""" + result = await _run( + [ + _write("/documents/dup.md", "first", "c1"), + _write("/documents/dup.md", "second", "c2"), + ScriptedTurn(text="done"), + ], + "fs-cloud-dup", + ) + + # Two write ToolMessages: first succeeds, second is rejected. + write_msgs = [ + str(m.content) + for m in result["messages"] + if isinstance(m, ToolMessage) and m.name == "write_file" + ] + assert len(write_msgs) == 2 + assert "Updated file" in write_msgs[0] + assert "already exists" in write_msgs[1] diff --git a/surfsense_backend/tests/integration/agents/multi_agent_chat/test_kb_filesystem_desktop.py b/surfsense_backend/tests/integration/agents/multi_agent_chat/test_kb_filesystem_desktop.py new file mode 100644 index 000000000..af94c0d89 --- /dev/null +++ b/surfsense_backend/tests/integration/agents/multi_agent_chat/test_kb_filesystem_desktop.py @@ -0,0 +1,349 @@ +"""Real-behavior tests for the LIVE knowledge-base filesystem middleware (B). + +These exercise ``app.agents.multi_agent_chat.middleware.shared.filesystem`` — +the decomposed middleware + tools that production actually mounts on the +knowledge_base subagent (via ``build_filesystem_mw``). The previous +``tests/unit/middleware/test_filesystem_*.py`` suite asserts a *dead twin* +(``app.agents.shared.middleware.filesystem``) that is never instantiated, so the +live tool path had no real coverage. + +Strategy: mount the production ``build_filesystem_mw`` on a minimal +``create_agent`` graph and drive its tools with the scripted harness. Desktop +mode binds a ``MultiRootLocalFolderBackend`` to a real ``tmp_path`` directory, +so every write/edit/move/rm is asserted against the real on-disk filesystem — +no mocks, only the LLM is scripted. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +from langchain.agents import create_agent +from langchain_core.messages import HumanMessage, ToolMessage +from langgraph.checkpoint.memory import InMemorySaver + +from app.agents.multi_agent_chat.middleware.shared.filesystem import ( + build_filesystem_mw, +) +from app.agents.shared.filesystem_backends import build_backend_resolver +from app.agents.shared.filesystem_selection import ( + FilesystemMode, + FilesystemSelection, + LocalFilesystemMount, +) +from tests.integration.harness import ScriptedTurn, build_scripted_harness + +pytestmark = [pytest.mark.integration, pytest.mark.asyncio] + +_MOUNT_ID = "workspace" + + +def _build_desktop_fs_mw(root: Path): + """Build the production filesystem middleware bound to a real local folder.""" + selection = FilesystemSelection( + mode=FilesystemMode.DESKTOP_LOCAL_FOLDER, + local_mounts=( + LocalFilesystemMount(mount_id=_MOUNT_ID, root_path=str(root)), + ), + ) + resolver = build_backend_resolver(selection) + return build_filesystem_mw( + backend_resolver=resolver, + filesystem_mode=FilesystemMode.DESKTOP_LOCAL_FOLDER, + search_space_id=1, + user_id="00000000-0000-0000-0000-000000000001", + thread_id=1, + read_only=False, + ) + + +async def _run(root: Path, turns: list[ScriptedTurn], thread: str): + """Assemble a 1-middleware agent and drive the scripted turns to completion.""" + harness = build_scripted_harness(turns=turns) + fs_mw = _build_desktop_fs_mw(root) + agent = create_agent( + harness.model, + tools=[], + middleware=[fs_mw], + checkpointer=InMemorySaver(), + ) + return await agent.ainvoke( + {"messages": [HumanMessage(content="do filesystem work")]}, + config={"configurable": {"thread_id": thread}}, + ) + + +def _tool_messages(result) -> list[ToolMessage]: + return [m for m in result["messages"] if isinstance(m, ToolMessage)] + + +def _tool_text(result, name: str) -> str: + for m in _tool_messages(result): + if m.name == name: + return str(m.content) + raise AssertionError(f"no ToolMessage from {name!r} in {_tool_messages(result)}") + + +async def test_write_then_read_round_trip(tmp_path: Path): + """write_file persists to the real folder and read_file returns the content.""" + result = await _run( + tmp_path, + [ + ScriptedTurn( + tool_calls=[ + { + "name": "write_file", + "args": { + "file_path": f"/{_MOUNT_ID}/notes.md", + "content": "hello FS-CANARY-001", + }, + "id": "c1", + } + ] + ), + ScriptedTurn( + tool_calls=[ + { + "name": "read_file", + "args": {"file_path": f"/{_MOUNT_ID}/notes.md"}, + "id": "c2", + } + ] + ), + ScriptedTurn(text="done"), + ], + "fs-desktop-write-read", + ) + + # Real on-disk effect, not a mock. + assert (tmp_path / "notes.md").read_text() == "hello FS-CANARY-001" + # The tool actually returned the file content. + assert "FS-CANARY-001" in _tool_text(result, "read_file") + + +async def test_write_then_ls_lists_file(tmp_path: Path): + """ls reflects a freshly written file in the real folder.""" + result = await _run( + tmp_path, + [ + ScriptedTurn( + tool_calls=[ + { + "name": "write_file", + "args": { + "file_path": f"/{_MOUNT_ID}/report.md", + "content": "x", + }, + "id": "c1", + } + ] + ), + ScriptedTurn( + tool_calls=[ + {"name": "ls", "args": {"path": f"/{_MOUNT_ID}"}, "id": "c2"} + ] + ), + ScriptedTurn(text="done"), + ], + "fs-desktop-ls", + ) + + assert (tmp_path / "report.md").exists() + assert "report.md" in _tool_text(result, "ls") + + +async def test_edit_file_rewrites_on_disk(tmp_path: Path): + """edit_file applies a real string replacement to the on-disk file.""" + result = await _run( + tmp_path, + [ + ScriptedTurn( + tool_calls=[ + { + "name": "write_file", + "args": { + "file_path": f"/{_MOUNT_ID}/doc.md", + "content": "the quick brown fox", + }, + "id": "c1", + } + ] + ), + ScriptedTurn( + tool_calls=[ + { + "name": "edit_file", + "args": { + "file_path": f"/{_MOUNT_ID}/doc.md", + "old_string": "brown", + "new_string": "red", + }, + "id": "c2", + } + ] + ), + ScriptedTurn(text="done"), + ], + "fs-desktop-edit", + ) + + assert (tmp_path / "doc.md").read_text() == "the quick red fox" + + +async def test_write_into_existing_subdir(tmp_path: Path): + """A write into an EXISTING subdirectory lands on disk under that folder.""" + (tmp_path / "sub").mkdir() + result = await _run( + tmp_path, + [ + ScriptedTurn( + tool_calls=[ + { + "name": "write_file", + "args": { + "file_path": f"/{_MOUNT_ID}/sub/inner.md", + "content": "nested", + }, + "id": "c1", + } + ] + ), + ScriptedTurn(text="done"), + ], + "fs-desktop-subdir", + ) + + assert "Error" not in _tool_text(result, "write_file") + assert (tmp_path / "sub" / "inner.md").read_text() == "nested" + + +async def test_write_to_missing_parent_dir_is_rejected(tmp_path: Path): + """Desktop write refuses to create a file under a non-existent directory. + + Real current behavior: the local-folder backend requires the parent to + exist (and ``mkdir`` is a no-op for this backend), so the agent cannot + fabricate new nested folders via ``write_file``. Locking this guards against + a silent behavior change during the agents-module reorg. + """ + result = await _run( + tmp_path, + [ + ScriptedTurn( + tool_calls=[ + { + "name": "write_file", + "args": { + "file_path": f"/{_MOUNT_ID}/missing/inner.md", + "content": "nested", + }, + "id": "c1", + } + ] + ), + ScriptedTurn(text="done"), + ], + "fs-desktop-missing-parent", + ) + + write_msg = _tool_text(result, "write_file") + assert "parent directory" in write_msg.lower() + assert not (tmp_path / "missing").exists() + + +async def test_move_file_relocates_on_disk(tmp_path: Path): + """move_file relocates the real file from source to destination.""" + await _run( + tmp_path, + [ + ScriptedTurn( + tool_calls=[ + { + "name": "write_file", + "args": { + "file_path": f"/{_MOUNT_ID}/src.md", + "content": "movable", + }, + "id": "c1", + } + ] + ), + ScriptedTurn( + tool_calls=[ + { + "name": "move_file", + "args": { + "source_path": f"/{_MOUNT_ID}/src.md", + "destination_path": f"/{_MOUNT_ID}/dst.md", + }, + "id": "c2", + } + ] + ), + ScriptedTurn(text="done"), + ], + "fs-desktop-move", + ) + + assert not (tmp_path / "src.md").exists() + assert (tmp_path / "dst.md").read_text() == "movable" + + +async def test_rm_deletes_file_on_disk(tmp_path: Path): + """rm removes the real file (desktop deletes are immediate).""" + await _run( + tmp_path, + [ + ScriptedTurn( + tool_calls=[ + { + "name": "write_file", + "args": { + "file_path": f"/{_MOUNT_ID}/trash.md", + "content": "bye", + }, + "id": "c1", + } + ] + ), + ScriptedTurn( + tool_calls=[ + { + "name": "rm", + "args": {"path": f"/{_MOUNT_ID}/trash.md"}, + "id": "c2", + } + ] + ), + ScriptedTurn(text="done"), + ], + "fs-desktop-rm", + ) + + assert not (tmp_path / "trash.md").exists() + + +async def test_rmdir_removes_empty_dir_on_disk(tmp_path: Path): + """rmdir removes a real empty directory.""" + (tmp_path / "gone").mkdir() + assert (tmp_path / "gone").is_dir() + + result = await _run( + tmp_path, + [ + ScriptedTurn( + tool_calls=[ + { + "name": "rmdir", + "args": {"path": f"/{_MOUNT_ID}/gone"}, + "id": "c1", + } + ] + ), + ScriptedTurn(text="done"), + ], + "fs-desktop-rmdir", + ) + + assert "Error" not in _tool_text(result, "rmdir") + assert not (tmp_path / "gone").exists() diff --git a/surfsense_backend/tests/unit/middleware/test_b_filesystem_path_resolution.py b/surfsense_backend/tests/unit/middleware/test_b_filesystem_path_resolution.py new file mode 100644 index 000000000..a5712bd7d --- /dev/null +++ b/surfsense_backend/tests/unit/middleware/test_b_filesystem_path_resolution.py @@ -0,0 +1,287 @@ +"""Path/cwd/namespace + multi-root mount-normalization tests for LIVE filesystem. + +Ported from the dead-twin suites: +* ``tests/unit/middleware/test_filesystem_middleware.py`` (cwd defaults, + relative resolution, cloud write-namespace policy) +* ``tests/unit/middleware/test_filesystem_verification.py`` (desktop + multi-root mount-prefix normalization) + +Both exercised ``app.agents.shared.middleware.filesystem`` (dead). This drives +the production free functions in +``app.agents.multi_agent_chat.middleware.shared.filesystem.middleware`` instead. +The functions only touch ``mw._filesystem_mode`` and ``mw._get_backend`` so we +pass a lightweight fake ``mw`` rather than constructing the full middleware. +""" + +from __future__ import annotations + +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from app.agents.multi_agent_chat.middleware.shared.filesystem.middleware.mode import ( + default_cwd, +) +from app.agents.multi_agent_chat.middleware.shared.filesystem.middleware.namespace_policy import ( + check_cloud_write_namespace, +) +from app.agents.multi_agent_chat.middleware.shared.filesystem.middleware.path_resolution import ( + current_cwd, + get_contract_suggested_path, + normalize_local_mount_path, + resolve_relative, +) +from app.agents.shared.filesystem_selection import FilesystemMode +from app.agents.shared.middleware.multi_root_local_folder_backend import ( + MultiRootLocalFolderBackend, +) + +pytestmark = pytest.mark.unit + + +def _mw(mode: FilesystemMode = FilesystemMode.CLOUD, backend=None): + return SimpleNamespace(_filesystem_mode=mode, _get_backend=lambda _rt: backend) + + +def _runtime(state: dict | None = None) -> SimpleNamespace: + return SimpleNamespace(state=state or {}) + + +# --------------------------------------------------------------------------- +# cwd defaults +# --------------------------------------------------------------------------- + + +class TestCwdDefaults: + def test_default_cwd_in_cloud_is_documents_root(self): + assert default_cwd(FilesystemMode.CLOUD) == "/documents" + + def test_default_cwd_in_desktop_is_root(self): + assert default_cwd(FilesystemMode.DESKTOP_LOCAL_FOLDER) == "/" + + def test_current_cwd_uses_state_when_set(self): + assert ( + current_cwd(_mw(), _runtime({"cwd": "/documents/notes"})) + == "/documents/notes" + ) + + def test_current_cwd_falls_back_to_default(self): + assert current_cwd(_mw(), _runtime({})) == "/documents" + + def test_current_cwd_ignores_invalid(self): + assert current_cwd(_mw(), _runtime({"cwd": "not-absolute"})) == "/documents" + + +# --------------------------------------------------------------------------- +# relative resolution +# --------------------------------------------------------------------------- + + +class TestRelativePathResolution: + def test_relative_path_resolves_against_cwd(self): + assert ( + resolve_relative(_mw(), "notes.md", _runtime({"cwd": "/documents/projects"})) + == "/documents/projects/notes.md" + ) + + def test_relative_path_with_dotdot(self): + assert ( + resolve_relative(_mw(), "../c.md", _runtime({"cwd": "/documents/a/b"})) + == "/documents/a/c.md" + ) + + def test_absolute_path_is_kept(self): + assert ( + resolve_relative(_mw(), "/other/x.md", _runtime({"cwd": "/documents"})) + == "/other/x.md" + ) + + def test_empty_path_returns_cwd(self): + assert ( + resolve_relative(_mw(), "", _runtime({"cwd": "/documents/projects"})) + == "/documents/projects" + ) + + +# --------------------------------------------------------------------------- +# contract suggested-path fallback +# --------------------------------------------------------------------------- + + +class TestContractSuggestedPath: + def test_falls_back_to_documents_notes_md_in_cloud(self): + suggested = get_contract_suggested_path( + _mw(FilesystemMode.CLOUD), + _runtime({"file_operation_contract": {}}), + ) + assert suggested == "/documents/notes.md" + + def test_falls_back_to_root_notes_md_in_desktop(self): + suggested = get_contract_suggested_path( + _mw(FilesystemMode.DESKTOP_LOCAL_FOLDER), + _runtime({"file_operation_contract": {}}), + ) + assert suggested == "/notes.md" + + +# --------------------------------------------------------------------------- +# cloud write-namespace policy +# --------------------------------------------------------------------------- + + +class TestCloudWriteNamespacePolicy: + def test_documents_path_allowed(self): + assert ( + check_cloud_write_namespace(_mw(), "/documents/foo.md", _runtime()) is None + ) + + def test_documents_root_allowed(self): + assert check_cloud_write_namespace(_mw(), "/documents", _runtime()) is None + + def test_temp_basename_anywhere_allowed(self): + assert ( + check_cloud_write_namespace(_mw(), "/temp_scratch.md", _runtime()) is None + ) + assert check_cloud_write_namespace(_mw(), "/foo/temp_x.md", _runtime()) is None + assert ( + check_cloud_write_namespace(_mw(), "/documents/temp_x.md", _runtime()) + is None + ) + + def test_other_paths_rejected(self): + err = check_cloud_write_namespace(_mw(), "/foo/bar.md", _runtime()) + assert err is not None + assert "must target /documents" in err + + def test_anon_doc_path_is_read_only(self): + runtime = _runtime( + { + "kb_anon_doc": { + "path": "/documents/uploaded.xml", + "title": "uploaded", + "content": "", + "chunks": [], + } + } + ) + err = check_cloud_write_namespace(_mw(), "/documents/uploaded.xml", runtime) + assert err is not None + assert "read-only" in err + + def test_desktop_mode_skips_namespace_policy(self): + assert ( + check_cloud_write_namespace( + _mw(FilesystemMode.DESKTOP_LOCAL_FOLDER), "/random/path.md", _runtime() + ) + is None + ) + + +# --------------------------------------------------------------------------- +# desktop multi-root mount normalization +# --------------------------------------------------------------------------- + + +def _desktop_mw(backend) -> SimpleNamespace: + return _mw(FilesystemMode.DESKTOP_LOCAL_FOLDER, backend) + + +class TestNormalizeLocalMountPath: + def test_prefixes_default_mount(self, tmp_path: Path): + root = tmp_path / "PC Backups" + root.mkdir() + backend = MultiRootLocalFolderBackend((("pc_backups", str(root)),)) + resolved = normalize_local_mount_path( + _desktop_mw(backend), + "/random-note.md", + _runtime({"file_operation_contract": {}}), + ) + assert resolved == "/pc_backups/random-note.md" + + def test_keeps_explicit_mount(self, tmp_path: Path): + root = tmp_path / "PC Backups" + root.mkdir() + backend = MultiRootLocalFolderBackend((("pc_backups", str(root)),)) + resolved = normalize_local_mount_path( + _desktop_mw(backend), + "/pc_backups/notes/random-note.md", + _runtime({"file_operation_contract": {}}), + ) + assert resolved == "/pc_backups/notes/random-note.md" + + def test_windows_backslashes(self, tmp_path: Path): + root = tmp_path / "PC Backups" + root.mkdir() + backend = MultiRootLocalFolderBackend((("pc_backups", str(root)),)) + resolved = normalize_local_mount_path( + _desktop_mw(backend), + r"\notes\random-note.md", + _runtime({"file_operation_contract": {}}), + ) + assert resolved == "/pc_backups/notes/random-note.md" + + def test_normalizes_mixed_separators(self, tmp_path: Path): + root = tmp_path / "PC Backups" + root.mkdir() + backend = MultiRootLocalFolderBackend((("pc_backups", str(root)),)) + resolved = normalize_local_mount_path( + _desktop_mw(backend), + r"\\notes//nested\\random-note.md", + _runtime({"file_operation_contract": {}}), + ) + assert resolved == "/pc_backups/notes/nested/random-note.md" + + def test_keeps_explicit_mount_with_backslashes(self, tmp_path: Path): + root = tmp_path / "PC Backups" + root.mkdir() + backend = MultiRootLocalFolderBackend((("pc_backups", str(root)),)) + resolved = normalize_local_mount_path( + _desktop_mw(backend), + r"\pc_backups\notes\random-note.md", + _runtime({"file_operation_contract": {}}), + ) + assert resolved == "/pc_backups/notes/random-note.md" + + def test_prefixes_posix_absolute_path(self, tmp_path: Path): + root = tmp_path / "PC Backups" + root.mkdir() + backend = MultiRootLocalFolderBackend((("pc_backups", str(root)),)) + resolved = normalize_local_mount_path( + _desktop_mw(backend), + "/var/log/app.log", + _runtime({"file_operation_contract": {}}), + ) + assert resolved == "/pc_backups/var/log/app.log" + + def test_prefers_unique_existing_parent_mount(self, tmp_path: Path): + root_a = tmp_path / "RootA" + root_b = tmp_path / "RootB" + (root_a / "other").mkdir(parents=True) + (root_b / "nested" / "deep").mkdir(parents=True) + backend = MultiRootLocalFolderBackend( + (("root_a", str(root_a)), ("root_b", str(root_b))) + ) + resolved = normalize_local_mount_path( + _desktop_mw(backend), + "/nested/deep/new-note.md", + _runtime({"file_operation_contract": {}}), + ) + assert resolved == "/root_b/nested/deep/new-note.md" + + def test_uses_suggested_mount_when_ambiguous(self, tmp_path: Path): + root_a = tmp_path / "RootA" + root_b = tmp_path / "RootB" + root_a.mkdir(parents=True) + root_b.mkdir(parents=True) + backend = MultiRootLocalFolderBackend( + (("root_a", str(root_a)), ("root_b", str(root_b))) + ) + resolved = normalize_local_mount_path( + _desktop_mw(backend), + "/brand-new-note.md", + _runtime( + {"file_operation_contract": {"suggested_path": "/root_b/notes/context.md"}} + ), + ) + assert resolved == "/root_b/brand-new-note.md" diff --git a/surfsense_backend/tests/unit/agents/new_chat/test_rm_rmdir_cloud.py b/surfsense_backend/tests/unit/middleware/test_b_filesystem_rm_rmdir_cloud.py similarity index 62% rename from surfsense_backend/tests/unit/agents/new_chat/test_rm_rmdir_cloud.py rename to surfsense_backend/tests/unit/middleware/test_b_filesystem_rm_rmdir_cloud.py index 4f0d4c48c..4a6fa3f95 100644 --- a/surfsense_backend/tests/unit/agents/new_chat/test_rm_rmdir_cloud.py +++ b/surfsense_backend/tests/unit/middleware/test_b_filesystem_rm_rmdir_cloud.py @@ -1,15 +1,14 @@ -"""Cloud-mode behavior tests for the new ``rm`` and ``rmdir`` filesystem tools. +"""Cloud-mode ``rm``/``rmdir`` staging tests for the LIVE filesystem middleware. -The tools build ``Command(update=...)`` payloads that the persistence -middleware applies at end of turn. These tests stub out the backend and -runtime to assert the staging payload shape: - -* ``rm`` queues into ``pending_deletes`` and tombstones state files. -* ``rm`` rejects directories, ``/documents``, root, and the anonymous doc. -* ``rmdir`` queues into ``pending_dir_deletes`` and rejects non-empty dirs. -* ``rmdir`` un-stages a same-turn ``mkdir`` rather than queuing a delete. -* ``rmdir`` refuses to drop the cwd or any of its ancestors. -* ``KBPostgresBackend`` view-helpers honor staged deletes. +Ported from the former ``tests/unit/agents/new_chat/test_rm_rmdir_cloud.py``, +which exercised the *dead twin* ``app.agents.shared.middleware.filesystem``. +This drives the production decomposed tools +(``app.agents.multi_agent_chat.middleware.shared.filesystem``) instead: it +builds the real middleware via ``build_filesystem_mw``, pulls the real ``rm`` / +``rmdir`` tools off it, and invokes their coroutines with a stubbed +``KBPostgresBackend`` + runtime so we can assert the end-of-turn staging +payloads (``pending_deletes`` / ``pending_dir_deletes``) and the destructive-op +guard rails (root, /documents, anon doc, non-empty, cwd/ancestor, file vs dir). """ from __future__ import annotations @@ -20,18 +19,31 @@ from unittest.mock import AsyncMock import pytest -from app.agents.shared.filesystem_selection import FilesystemMode -from app.agents.shared.middleware.filesystem import SurfSenseFilesystemMiddleware +from app.agents.multi_agent_chat.middleware.shared.filesystem import ( + build_filesystem_mw, +) +from app.agents.shared.filesystem_backends import build_backend_resolver +from app.agents.shared.filesystem_selection import FilesystemMode, FilesystemSelection from app.agents.shared.middleware.kb_postgres_backend import KBPostgresBackend +from app.agents.shared.state_reducers import _CLEAR pytestmark = pytest.mark.unit def _make_middleware(mode: FilesystemMode = FilesystemMode.CLOUD): - middleware = SurfSenseFilesystemMiddleware.__new__(SurfSenseFilesystemMiddleware) - middleware._filesystem_mode = mode - middleware._custom_tool_descriptions = {} - return middleware + selection = FilesystemSelection(mode=mode) + resolver = build_backend_resolver(selection, search_space_id=1) + return build_filesystem_mw( + backend_resolver=resolver, + filesystem_mode=mode, + search_space_id=1, + user_id="00000000-0000-0000-0000-000000000001", + thread_id=1, + ) + + +def _tool(mw, name: str): + return next(t for t in mw.tools if t.name == name) def _runtime(state: dict[str, Any] | None = None, *, tool_call_id: str = "tc-abc"): @@ -41,13 +53,12 @@ def _runtime(state: dict[str, Any] | None = None, *, tool_call_id: str = "tc-abc class _KBBackendStub(KBPostgresBackend): - """Construct-able subclass of :class:`KBPostgresBackend` for tests. + """Construct-able ``KBPostgresBackend`` subclass for tests. - We bypass the real ``__init__`` (which expects a runtime + DB session) - and inject just the methods the rm/rmdir tools touch. The class - inheritance keeps ``isinstance(backend, KBPostgresBackend)`` checks - inside the tools happy, which is what gates them from the desktop - code path. + Bypasses the real ``__init__`` (which expects a runtime + DB session) and + injects only the async methods the rm/rmdir tools touch. The class + inheritance keeps the ``isinstance(backend, KBPostgresBackend)`` checks in + the tools on the cloud path. """ def __init__(self, *, children=None, file_data=None) -> None: @@ -61,9 +72,8 @@ def _make_backend_stub(*, children=None, file_data=None) -> KBPostgresBackend: return _KBBackendStub(children=children, file_data=file_data) -def _bind_backend(middleware, backend): - """Inject a backend resolver onto the middleware test instance.""" - middleware._get_backend = lambda runtime: backend +def _bind_backend(mw, backend): + mw._get_backend = lambda runtime: backend return backend @@ -86,8 +96,7 @@ class TestRmStaging: tool_call_id="tc-1", ) - tool = m._create_rm_tool() - result = await tool.coroutine("/documents/notes.md", runtime=runtime) + result = await _tool(m, "rm").coroutine("/documents/notes.md", runtime=runtime) assert hasattr(result, "update"), f"expected Command, got {result!r}" update = result.update @@ -100,31 +109,22 @@ class TestRmStaging: @pytest.mark.asyncio async def test_rejects_documents_root(self): m = _make_middleware() - runtime = _runtime() - tool = m._create_rm_tool() - result = await tool.coroutine("/documents", runtime=runtime) + result = await _tool(m, "rm").coroutine("/documents", runtime=_runtime()) assert isinstance(result, str) assert "refusing to rm" in result @pytest.mark.asyncio async def test_rejects_root(self): m = _make_middleware() - runtime = _runtime() - tool = m._create_rm_tool() - result = await tool.coroutine("/", runtime=runtime) + result = await _tool(m, "rm").coroutine("/", runtime=_runtime()) assert isinstance(result, str) assert "refusing to rm" in result @pytest.mark.asyncio async def test_rejects_directory_via_staged_dirs(self): m = _make_middleware() - runtime = _runtime( - { - "staged_dirs": ["/documents/team-x"], - } - ) - tool = m._create_rm_tool() - result = await tool.coroutine("/documents/team-x", runtime=runtime) + runtime = _runtime({"staged_dirs": ["/documents/team-x"]}) + result = await _tool(m, "rm").coroutine("/documents/team-x", runtime=runtime) assert isinstance(result, str) assert "directory" in result.lower() assert "rmdir" in result @@ -138,9 +138,7 @@ class TestRmStaging: children=[{"path": "/documents/foo/x.md", "is_dir": False}] ), ) - runtime = _runtime() - tool = m._create_rm_tool() - result = await tool.coroutine("/documents/foo", runtime=runtime) + result = await _tool(m, "rm").coroutine("/documents/foo", runtime=_runtime()) assert isinstance(result, str) assert "directory" in result.lower() @@ -157,8 +155,9 @@ class TestRmStaging: } } ) - tool = m._create_rm_tool() - result = await tool.coroutine("/documents/uploaded.xml", runtime=runtime) + result = await _tool(m, "rm").coroutine( + "/documents/uploaded.xml", runtime=runtime + ) assert isinstance(result, str) assert "read-only" in result @@ -173,12 +172,9 @@ class TestRmStaging: "dirty_paths": ["/documents/notes.md"], } ) - tool = m._create_rm_tool() - result = await tool.coroutine("/documents/notes.md", runtime=runtime) - update = result.update - # First element is _CLEAR sentinel; the rest must NOT contain the - # rm'd path. - dirty = update.get("dirty_paths") or [] + result = await _tool(m, "rm").coroutine("/documents/notes.md", runtime=runtime) + dirty = result.update.get("dirty_paths") or [] + # First element is the _CLEAR sentinel; the rm'd path must not survive. assert "/documents/notes.md" not in dirty[1:] @@ -192,30 +188,19 @@ class TestRmdirStaging: async def test_stages_dir_delete_when_empty_and_db_backed(self): m = _make_middleware() backend = _bind_backend(m, _make_backend_stub(children=[])) - # Override _load_file_data to return None (folder, not a file) and - # parent listing to claim the folder exists. backend._load_file_data = AsyncMock(return_value=None) backend.als_info = AsyncMock( side_effect=[ [], # children of /documents/proj - [ - {"path": "/documents/proj", "is_dir": True}, - ], # parent listing + [{"path": "/documents/proj", "is_dir": True}], # parent listing ] ) - runtime = _runtime( - { - "cwd": "/documents", - }, - tool_call_id="tc-rd", - ) + runtime = _runtime({"cwd": "/documents"}, tool_call_id="tc-rd") - tool = m._create_rmdir_tool() - result = await tool.coroutine("/documents/proj", runtime=runtime) + result = await _tool(m, "rmdir").coroutine("/documents/proj", runtime=runtime) assert hasattr(result, "update") - update = result.update - assert update["pending_dir_deletes"] == [ + assert result.update["pending_dir_deletes"] == [ {"path": "/documents/proj", "tool_call_id": "tc-rd"} ] @@ -228,9 +213,9 @@ class TestRmdirStaging: children=[{"path": "/documents/proj/x.md", "is_dir": False}] ), ) - runtime = _runtime() - tool = m._create_rmdir_tool() - result = await tool.coroutine("/documents/proj", runtime=runtime) + result = await _tool(m, "rmdir").coroutine( + "/documents/proj", runtime=_runtime() + ) assert isinstance(result, str) assert "not empty" in result @@ -239,30 +224,25 @@ class TestRmdirStaging: m = _make_middleware() _bind_backend(m, _make_backend_stub(children=[])) runtime = _runtime( - { - "cwd": "/documents", - "staged_dirs": ["/documents/scratch"], - }, + {"cwd": "/documents", "staged_dirs": ["/documents/scratch"]}, tool_call_id="tc-rd", ) - tool = m._create_rmdir_tool() - result = await tool.coroutine("/documents/scratch", runtime=runtime) + result = await _tool(m, "rmdir").coroutine( + "/documents/scratch", runtime=runtime + ) assert hasattr(result, "update") update = result.update assert "pending_dir_deletes" not in update - # _CLEAR sentinel + remaining items (in this case, none). staged_after = update["staged_dirs"] - assert staged_after[0] == "\x00__SURFSENSE_FILESYSTEM_CLEAR__\x00" + assert staged_after[0] == _CLEAR assert "/documents/scratch" not in staged_after[1:] @pytest.mark.asyncio - async def test_rejects_root(self): + async def test_rejects_root_and_documents(self): m = _make_middleware() - runtime = _runtime() - tool = m._create_rmdir_tool() for victim in ("/", "/documents"): - result = await tool.coroutine(victim, runtime=runtime) + result = await _tool(m, "rmdir").coroutine(victim, runtime=_runtime()) assert isinstance(result, str) assert "refusing to rmdir" in result @@ -270,8 +250,7 @@ class TestRmdirStaging: async def test_rejects_cwd(self): m = _make_middleware() runtime = _runtime({"cwd": "/documents/proj"}) - tool = m._create_rmdir_tool() - result = await tool.coroutine("/documents/proj", runtime=runtime) + result = await _tool(m, "rmdir").coroutine("/documents/proj", runtime=runtime) assert isinstance(result, str) assert "cwd" in result.lower() @@ -279,8 +258,7 @@ class TestRmdirStaging: async def test_rejects_ancestor_of_cwd(self): m = _make_middleware() runtime = _runtime({"cwd": "/documents/proj/sub"}) - tool = m._create_rmdir_tool() - result = await tool.coroutine("/documents/proj", runtime=runtime) + result = await _tool(m, "rmdir").coroutine("/documents/proj", runtime=runtime) assert isinstance(result, str) assert "cwd" in result.lower() @@ -288,34 +266,31 @@ class TestRmdirStaging: async def test_rejects_files(self): m = _make_middleware() _bind_backend(m, _make_backend_stub(children=[], file_data={"content": ["x"]})) - runtime = _runtime() - tool = m._create_rmdir_tool() - result = await tool.coroutine("/documents/notes.md", runtime=runtime) + result = await _tool(m, "rmdir").coroutine( + "/documents/notes.md", runtime=_runtime() + ) assert isinstance(result, str) assert "is a file" in result # --------------------------------------------------------------------------- -# KBPostgresBackend view filter +# KBPostgresBackend staged-delete view filter (already the live backend) # --------------------------------------------------------------------------- class TestKBPostgresBackendDeleteFilter: - """als_info / glob / grep should suppress paths queued for delete.""" + """``als_info`` / glob / grep must suppress paths queued for delete.""" def _make_backend(self, state: dict[str, Any]) -> KBPostgresBackend: runtime = SimpleNamespace(state=state) - backend = KBPostgresBackend(search_space_id=1, runtime=runtime) - return backend + return KBPostgresBackend(search_space_id=1, runtime=runtime) def test_pending_filesystem_view_returns_deleted_paths(self): backend = self._make_backend( { - "pending_deletes": [ - {"path": "/documents/x.md", "tool_call_id": "t1"}, - ], + "pending_deletes": [{"path": "/documents/x.md", "tool_call_id": "t1"}], "pending_dir_deletes": [ - {"path": "/documents/d1", "tool_call_id": "t2"}, + {"path": "/documents/d1", "tool_call_id": "t2"} ], } ) diff --git a/surfsense_backend/tests/unit/middleware/test_b_filesystem_system_prompt.py b/surfsense_backend/tests/unit/middleware/test_b_filesystem_system_prompt.py new file mode 100644 index 000000000..eb8c3dc15 --- /dev/null +++ b/surfsense_backend/tests/unit/middleware/test_b_filesystem_system_prompt.py @@ -0,0 +1,54 @@ +"""Mode-specific system-prompt assembly tests for the LIVE filesystem middleware. + +Ported from ``TestModeSpecificPrompts`` in the former +``tests/unit/middleware/test_filesystem_middleware.py`` (which exercised the +dead twin ``app.agents.shared.middleware.filesystem._build_filesystem_system_prompt``). + +These drive the production ``build_system_prompt`` so the prompt the model +actually receives stays mode-scoped: cloud rules don't leak into desktop +sessions and vice-versa, and the sandbox section appears only when available. + +The per-tool *description* assertions from the old suite are intentionally NOT +ported: they assert exact prompt copy (tightly coupled to the old wording) and +guard prompt token hygiene rather than the code-movement refactor this suite +protects. +""" + +from __future__ import annotations + +import pytest + +from app.agents.multi_agent_chat.middleware.shared.filesystem.system_prompt import ( + build_system_prompt, +) +from app.agents.shared.filesystem_selection import FilesystemMode + +pytestmark = pytest.mark.unit + + +class TestModeSpecificPrompts: + def test_cloud_prompt_omits_desktop_section(self): + prompt = build_system_prompt(FilesystemMode.CLOUD, sandbox_available=False) + assert "Local Folder Mode" not in prompt + assert "mount-prefixed" not in prompt + assert "Persistence Rules" in prompt + assert "/documents" in prompt + assert "temp_" in prompt + + def test_desktop_prompt_omits_cloud_persistence_rules(self): + prompt = build_system_prompt( + FilesystemMode.DESKTOP_LOCAL_FOLDER, sandbox_available=False + ) + assert "Persistence Rules" not in prompt + assert "Workspace Tree" not in prompt + assert "Local Folder Mode" in prompt + assert "mount-prefixed" in prompt + + def test_sandbox_addendum_appended_when_available(self): + prompt = build_system_prompt(FilesystemMode.CLOUD, sandbox_available=True) + assert "execute_code" in prompt + assert "Code Execution" in prompt + + def test_sandbox_addendum_absent_when_unavailable(self): + prompt = build_system_prompt(FilesystemMode.CLOUD, sandbox_available=False) + assert "execute_code" not in prompt diff --git a/surfsense_backend/tests/unit/middleware/test_filesystem_middleware.py b/surfsense_backend/tests/unit/middleware/test_filesystem_middleware.py deleted file mode 100644 index 482d01b7a..000000000 --- a/surfsense_backend/tests/unit/middleware/test_filesystem_middleware.py +++ /dev/null @@ -1,220 +0,0 @@ -"""Unit tests for the SurfSense filesystem middleware new behaviors. - -Covers: -* cloud cwd defaults to ``/documents`` and relative paths resolve under it -* cloud writes outside ``/documents/`` are rejected unless basename starts - with ``temp_`` -* cloud writes/edits to the anonymous document are rejected (read-only) -* helper methods on the middleware (``_resolve_relative``, - ``_check_cloud_write_namespace``, ``_default_cwd``) - -These tests use ``__new__`` to bypass the heavy ``__init__`` and exercise -the helper methods directly so the test surface stays narrow and fast. -""" - -from __future__ import annotations - -from types import SimpleNamespace - -import pytest - -from app.agents.shared.filesystem_selection import FilesystemMode -from app.agents.shared.middleware.filesystem import ( - SurfSenseFilesystemMiddleware, - _build_filesystem_system_prompt, - _build_tool_descriptions, -) - -pytestmark = pytest.mark.unit - - -def _make_middleware(mode: FilesystemMode = FilesystemMode.CLOUD): - middleware = SurfSenseFilesystemMiddleware.__new__(SurfSenseFilesystemMiddleware) - middleware._filesystem_mode = mode - return middleware - - -def _runtime(state: dict | None = None) -> SimpleNamespace: - return SimpleNamespace(state=state or {}) - - -class TestCloudCwdDefaults: - def test_default_cwd_in_cloud_is_documents_root(self): - m = _make_middleware() - assert m._default_cwd() == "/documents" - - def test_default_cwd_in_desktop_is_root(self): - m = _make_middleware(FilesystemMode.DESKTOP_LOCAL_FOLDER) - assert m._default_cwd() == "/" - - def test_current_cwd_uses_state_when_set(self): - m = _make_middleware() - runtime = _runtime({"cwd": "/documents/notes"}) - assert m._current_cwd(runtime) == "/documents/notes" - - def test_current_cwd_falls_back_to_default(self): - m = _make_middleware() - runtime = _runtime({}) - assert m._current_cwd(runtime) == "/documents" - - def test_current_cwd_ignores_invalid(self): - m = _make_middleware() - runtime = _runtime({"cwd": "not-absolute"}) - assert m._current_cwd(runtime) == "/documents" - - -class TestRelativePathResolution: - def test_relative_path_resolves_against_cwd(self): - m = _make_middleware() - runtime = _runtime({"cwd": "/documents/projects"}) - assert ( - m._resolve_relative("notes.md", runtime) == "/documents/projects/notes.md" - ) - - def test_relative_path_with_dotdot(self): - m = _make_middleware() - runtime = _runtime({"cwd": "/documents/a/b"}) - assert m._resolve_relative("../c.md", runtime) == "/documents/a/c.md" - - def test_absolute_path_is_kept(self): - m = _make_middleware() - runtime = _runtime({"cwd": "/documents"}) - assert m._resolve_relative("/other/x.md", runtime) == "/other/x.md" - - def test_empty_path_returns_cwd(self): - m = _make_middleware() - runtime = _runtime({"cwd": "/documents/projects"}) - assert m._resolve_relative("", runtime) == "/documents/projects" - - -class TestCloudWriteNamespacePolicy: - def test_documents_path_allowed(self): - m = _make_middleware() - runtime = _runtime() - assert m._check_cloud_write_namespace("/documents/foo.md", runtime) is None - - def test_documents_root_allowed(self): - m = _make_middleware() - runtime = _runtime() - assert m._check_cloud_write_namespace("/documents", runtime) is None - - def test_temp_basename_anywhere_allowed(self): - m = _make_middleware() - runtime = _runtime() - assert m._check_cloud_write_namespace("/temp_scratch.md", runtime) is None - assert m._check_cloud_write_namespace("/foo/temp_x.md", runtime) is None - assert m._check_cloud_write_namespace("/documents/temp_x.md", runtime) is None - - def test_other_paths_rejected(self): - m = _make_middleware() - runtime = _runtime() - err = m._check_cloud_write_namespace("/foo/bar.md", runtime) - assert err is not None - assert "must target /documents" in err - - def test_anon_doc_path_is_read_only(self): - m = _make_middleware() - runtime = _runtime( - { - "kb_anon_doc": { - "path": "/documents/uploaded.xml", - "title": "uploaded", - "content": "", - "chunks": [], - } - } - ) - err = m._check_cloud_write_namespace("/documents/uploaded.xml", runtime) - assert err is not None - assert "read-only" in err - - def test_desktop_mode_skips_namespace_policy(self): - m = _make_middleware(FilesystemMode.DESKTOP_LOCAL_FOLDER) - runtime = _runtime() - assert m._check_cloud_write_namespace("/random/path.md", runtime) is None - - -class TestModeSpecificPrompts: - """The prompt and tool descriptions must only describe the active mode. - - Cross-mode noise wastes tokens and confuses the model with rules it - cannot use this session. - """ - - def test_cloud_prompt_omits_desktop_section(self): - prompt = _build_filesystem_system_prompt( - FilesystemMode.CLOUD, sandbox_available=False - ) - assert "Local Folder Mode" not in prompt - assert "mount-prefixed" not in prompt - assert "Persistence Rules" in prompt - assert "/documents" in prompt - assert "temp_" in prompt - - def test_desktop_prompt_omits_cloud_persistence_rules(self): - prompt = _build_filesystem_system_prompt( - FilesystemMode.DESKTOP_LOCAL_FOLDER, sandbox_available=False - ) - assert "Persistence Rules" not in prompt - assert "Workspace Tree" not in prompt - assert "" not in prompt - assert "Local Folder Mode" in prompt - assert "mount-prefixed" in prompt - - def test_cloud_tool_descs_omit_desktop_phrases(self): - descs = _build_tool_descriptions(FilesystemMode.CLOUD) - for name in ( - "write_file", - "edit_file", - "move_file", - "mkdir", - "rm", - "rmdir", - "list_tree", - "grep", - ): - text = descs[name] - assert "Desktop" not in text, f"{name} leaks desktop hints" - assert "Cloud mode:" not in text, f"{name} qualifies a cloud-only desc" - - def test_desktop_tool_descs_omit_cloud_phrases(self): - descs = _build_tool_descriptions(FilesystemMode.DESKTOP_LOCAL_FOLDER) - for name in ( - "write_file", - "edit_file", - "move_file", - "mkdir", - "rm", - "rmdir", - "list_tree", - "grep", - ): - text = descs[name] - assert "Cloud" not in text, f"{name} leaks cloud hints" - assert "/documents/" not in text, f"{name} mentions cloud namespace" - assert "temp_" not in text, f"{name} mentions cloud temp_ semantics" - - def test_cloud_descs_include_rm_and_rmdir(self): - descs = _build_tool_descriptions(FilesystemMode.CLOUD) - assert "rm" in descs and "rmdir" in descs - assert "Deletes a single file" in descs["rm"] - assert "Deletes an empty directory" in descs["rmdir"] - assert "rmdir" in descs["rmdir"] and "POSIX" in descs["rmdir"] - - def test_desktop_descs_warn_about_irreversibility(self): - descs = _build_tool_descriptions(FilesystemMode.DESKTOP_LOCAL_FOLDER) - assert "NOT reversible" in descs["rm"] - assert "NOT reversible" in descs["rmdir"] - - def test_sandbox_addendum_appended_when_available(self): - prompt = _build_filesystem_system_prompt( - FilesystemMode.CLOUD, sandbox_available=True - ) - assert "execute_code" in prompt - assert "Code Execution" in prompt - - def test_sandbox_addendum_absent_when_unavailable(self): - prompt = _build_filesystem_system_prompt( - FilesystemMode.CLOUD, sandbox_available=False - ) - assert "execute_code" not in prompt diff --git a/surfsense_backend/tests/unit/middleware/test_filesystem_verification.py b/surfsense_backend/tests/unit/middleware/test_filesystem_verification.py deleted file mode 100644 index 80306a801..000000000 --- a/surfsense_backend/tests/unit/middleware/test_filesystem_verification.py +++ /dev/null @@ -1,173 +0,0 @@ -from pathlib import Path - -import pytest - -from app.agents.shared.filesystem_selection import FilesystemMode -from app.agents.shared.middleware.filesystem import SurfSenseFilesystemMiddleware -from app.agents.shared.middleware.multi_root_local_folder_backend import ( - MultiRootLocalFolderBackend, -) - -pytestmark = pytest.mark.unit - - -class _RuntimeNoSuggestedPath: - state = {"file_operation_contract": {}} - - -class _RuntimeWithSuggestedPath: - def __init__(self, suggested_path: str) -> None: - self.state = {"file_operation_contract": {"suggested_path": suggested_path}} - - -def test_contract_suggested_path_falls_back_to_documents_notes_md() -> None: - middleware = SurfSenseFilesystemMiddleware.__new__(SurfSenseFilesystemMiddleware) - middleware._filesystem_mode = FilesystemMode.CLOUD - suggested = middleware._get_contract_suggested_path(_RuntimeNoSuggestedPath()) # type: ignore[arg-type] - # Cloud default cwd is /documents so the fallback lands in the KB. - assert suggested == "/documents/notes.md" - - -def test_contract_suggested_path_falls_back_to_root_notes_md_in_desktop() -> None: - middleware = SurfSenseFilesystemMiddleware.__new__(SurfSenseFilesystemMiddleware) - middleware._filesystem_mode = FilesystemMode.DESKTOP_LOCAL_FOLDER - suggested = middleware._get_contract_suggested_path(_RuntimeNoSuggestedPath()) # type: ignore[arg-type] - assert suggested == "/notes.md" - - -def test_normalize_local_mount_path_prefixes_default_mount(tmp_path: Path) -> None: - root = tmp_path / "PC Backups" - root.mkdir() - backend = MultiRootLocalFolderBackend((("pc_backups", str(root)),)) - runtime = _RuntimeNoSuggestedPath() - middleware = SurfSenseFilesystemMiddleware.__new__(SurfSenseFilesystemMiddleware) - middleware._get_backend = lambda _runtime: backend # type: ignore[method-assign] - - resolved = middleware._normalize_local_mount_path("/random-note.md", runtime) # type: ignore[arg-type] - - assert resolved == "/pc_backups/random-note.md" - - -def test_normalize_local_mount_path_keeps_explicit_mount(tmp_path: Path) -> None: - root = tmp_path / "PC Backups" - root.mkdir() - backend = MultiRootLocalFolderBackend((("pc_backups", str(root)),)) - runtime = _RuntimeNoSuggestedPath() - middleware = SurfSenseFilesystemMiddleware.__new__(SurfSenseFilesystemMiddleware) - middleware._get_backend = lambda _runtime: backend # type: ignore[method-assign] - - resolved = middleware._normalize_local_mount_path( # type: ignore[arg-type] - "/pc_backups/notes/random-note.md", - runtime, - ) - - assert resolved == "/pc_backups/notes/random-note.md" - - -def test_normalize_local_mount_path_windows_backslashes(tmp_path: Path) -> None: - root = tmp_path / "PC Backups" - root.mkdir() - backend = MultiRootLocalFolderBackend((("pc_backups", str(root)),)) - runtime = _RuntimeNoSuggestedPath() - middleware = SurfSenseFilesystemMiddleware.__new__(SurfSenseFilesystemMiddleware) - middleware._get_backend = lambda _runtime: backend # type: ignore[method-assign] - - resolved = middleware._normalize_local_mount_path( # type: ignore[arg-type] - r"\notes\random-note.md", - runtime, - ) - - assert resolved == "/pc_backups/notes/random-note.md" - - -def test_normalize_local_mount_path_normalizes_mixed_separators(tmp_path: Path) -> None: - root = tmp_path / "PC Backups" - root.mkdir() - backend = MultiRootLocalFolderBackend((("pc_backups", str(root)),)) - runtime = _RuntimeNoSuggestedPath() - middleware = SurfSenseFilesystemMiddleware.__new__(SurfSenseFilesystemMiddleware) - middleware._get_backend = lambda _runtime: backend # type: ignore[method-assign] - - resolved = middleware._normalize_local_mount_path( # type: ignore[arg-type] - r"\\notes//nested\\random-note.md", - runtime, - ) - - assert resolved == "/pc_backups/notes/nested/random-note.md" - - -def test_normalize_local_mount_path_keeps_explicit_mount_with_backslashes( - tmp_path: Path, -) -> None: - root = tmp_path / "PC Backups" - root.mkdir() - backend = MultiRootLocalFolderBackend((("pc_backups", str(root)),)) - runtime = _RuntimeNoSuggestedPath() - middleware = SurfSenseFilesystemMiddleware.__new__(SurfSenseFilesystemMiddleware) - middleware._get_backend = lambda _runtime: backend # type: ignore[method-assign] - - resolved = middleware._normalize_local_mount_path( # type: ignore[arg-type] - r"\pc_backups\notes\random-note.md", - runtime, - ) - - assert resolved == "/pc_backups/notes/random-note.md" - - -def test_normalize_local_mount_path_prefixes_posix_absolute_path_for_linux_and_macos( - tmp_path: Path, -) -> None: - root = tmp_path / "PC Backups" - root.mkdir() - backend = MultiRootLocalFolderBackend((("pc_backups", str(root)),)) - runtime = _RuntimeNoSuggestedPath() - middleware = SurfSenseFilesystemMiddleware.__new__(SurfSenseFilesystemMiddleware) - middleware._get_backend = lambda _runtime: backend # type: ignore[method-assign] - - resolved = middleware._normalize_local_mount_path("/var/log/app.log", runtime) # type: ignore[arg-type] - - assert resolved == "/pc_backups/var/log/app.log" - - -def test_normalize_local_mount_path_prefers_unique_existing_parent_mount( - tmp_path: Path, -) -> None: - root_a = tmp_path / "RootA" - root_b = tmp_path / "RootB" - (root_a / "other").mkdir(parents=True) - (root_b / "nested" / "deep").mkdir(parents=True) - backend = MultiRootLocalFolderBackend( - (("root_a", str(root_a)), ("root_b", str(root_b))) - ) - runtime = _RuntimeNoSuggestedPath() - middleware = SurfSenseFilesystemMiddleware.__new__(SurfSenseFilesystemMiddleware) - middleware._get_backend = lambda _runtime: backend # type: ignore[method-assign] - - resolved = middleware._normalize_local_mount_path( # type: ignore[arg-type] - "/nested/deep/new-note.md", - runtime, - ) - - assert resolved == "/root_b/nested/deep/new-note.md" - - -def test_normalize_local_mount_path_uses_suggested_mount_when_ambiguous( - tmp_path: Path, -) -> None: - root_a = tmp_path / "RootA" - root_b = tmp_path / "RootB" - root_a.mkdir(parents=True) - root_b.mkdir(parents=True) - backend = MultiRootLocalFolderBackend( - (("root_a", str(root_a)), ("root_b", str(root_b))) - ) - runtime = _RuntimeWithSuggestedPath("/root_b/notes/context.md") - middleware = SurfSenseFilesystemMiddleware.__new__(SurfSenseFilesystemMiddleware) - middleware._get_backend = lambda _runtime: backend # type: ignore[method-assign] - - resolved = middleware._normalize_local_mount_path( # type: ignore[arg-type] - "/brand-new-note.md", - runtime, - ) - - assert resolved == "/root_b/brand-new-note.md"