diff --git a/surfsense_backend/app/agents/new_chat/middleware/filesystem.py b/surfsense_backend/app/agents/new_chat/middleware/filesystem.py index d7bb339bd..3622bbcdf 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/filesystem.py +++ b/surfsense_backend/app/agents/new_chat/middleware/filesystem.py @@ -28,9 +28,6 @@ from langgraph.types import Command from sqlalchemy import delete, select from app.agents.new_chat.filesystem_selection import FilesystemMode -from app.agents.new_chat.middleware.multi_root_local_folder_backend import ( - MultiRootLocalFolderBackend, -) from app.agents.new_chat.sandbox import ( _evict_sandbox_cache, delete_sandbox, @@ -152,21 +149,19 @@ Notes: - Cross-mount moves are not supported. """ -SURFSENSE_LIST_TREE_TOOL_DESCRIPTION = """Lists files/folders recursively with cursor pagination. +SURFSENSE_LIST_TREE_TOOL_DESCRIPTION = """Lists files/folders recursively in a single bounded call. Use this in desktop local-folder mode to discover nested files at scale. Args: - path: absolute mount-prefixed path (e.g., //src) or "/" for mount roots. - max_depth: recursion depth limit (default 8). -- page_size: number of entries to return per page (max 1000). -- cursor: opaque continuation token from a previous call. +- page_size: maximum number of entries returned (max 1000). - include_files/include_dirs: filter returned entry types. Returns JSON with: - entries: [{path, is_dir, size, modified_at, depth}] -- next_cursor: continuation token or null -- has_more: whether additional pages exist +- truncated: true when additional entries were omitted due to page_size """ SURFSENSE_GLOB_TOOL_DESCRIPTION = """Find files matching a glob pattern. @@ -251,13 +246,13 @@ class SurfSenseFilesystemMiddleware(FilesystemMiddleware): if filesystem_mode == FilesystemMode.DESKTOP_LOCAL_FOLDER: system_prompt += ( "\n- move_file: move or rename files/folders in local-folder mode." - "\n- list_tree: recursively list nested local paths with cursor pagination." + "\n- list_tree: recursively list nested local paths in one bounded response." "\n\n## Local Folder Mode" "\n\nThis chat is running in desktop local-folder mode." " Keep all file operations local. Do not use save_document." " Always use mount-prefixed absolute paths like //file.ext." " If you are unsure which mounts are available, call ls('/') first." - " For big trees: use list_tree pages, then grep, then read_file." + " For big trees: use list_tree, then grep, then read_file." ) super().__init__( @@ -812,35 +807,14 @@ class SurfSenseFilesystemMiddleware(FilesystemMiddleware): """Only cloud mode persists file content to Document/Chunk tables.""" return self._filesystem_mode == FilesystemMode.CLOUD - def _default_mount_prefix(self, runtime: ToolRuntime[None, FilesystemState]) -> str: - backend = self._get_backend(runtime) - if isinstance(backend, MultiRootLocalFolderBackend): - return f"/{backend.default_mount()}" - return "" - - def _normalize_local_mount_path( - self, candidate: str, runtime: ToolRuntime[None, FilesystemState] - ) -> str: - backend = self._get_backend(runtime) - mount_prefix = self._default_mount_prefix(runtime) - normalized_candidate = re.sub(r"/+", "/", candidate.strip().replace("\\", "/")) - if not mount_prefix or not isinstance(backend, MultiRootLocalFolderBackend): - if normalized_candidate.startswith("/"): - return normalized_candidate - return f"/{normalized_candidate.lstrip('/')}" - - mount_names = set(backend.list_mounts()) - if normalized_candidate.startswith("/"): - first_segment = normalized_candidate.lstrip("/").split("/", 1)[0] - if first_segment in mount_names: - return normalized_candidate - return f"{mount_prefix}{normalized_candidate}" - - relative = normalized_candidate.lstrip("/") - first_segment = relative.split("/", 1)[0] - if first_segment in mount_names: - return f"/{relative}" - return f"{mount_prefix}/{relative}" + @staticmethod + def _normalize_absolute_path(candidate: str) -> str: + normalized = re.sub(r"/+", "/", candidate.strip().replace("\\", "/")) + if not normalized: + return "/" + if normalized.startswith("/"): + return normalized + return f"/{normalized.lstrip('/')}" def _get_contract_suggested_path( self, runtime: ToolRuntime[None, FilesystemState] @@ -848,14 +822,7 @@ class SurfSenseFilesystemMiddleware(FilesystemMiddleware): contract = runtime.state.get("file_operation_contract") or {} suggested = contract.get("suggested_path") if isinstance(suggested, str) and suggested.strip(): - cleaned = suggested.strip() - if self._filesystem_mode == FilesystemMode.DESKTOP_LOCAL_FOLDER: - return self._normalize_local_mount_path(cleaned, runtime) - return cleaned - if self._filesystem_mode == FilesystemMode.DESKTOP_LOCAL_FOLDER: - mount_prefix = self._default_mount_prefix(runtime) - if mount_prefix: - return f"{mount_prefix}/notes.md" + return self._normalize_absolute_path(suggested) return "/notes.md" def _resolve_write_target_path( @@ -867,7 +834,7 @@ class SurfSenseFilesystemMiddleware(FilesystemMiddleware): if not candidate: return self._get_contract_suggested_path(runtime) if self._filesystem_mode == FilesystemMode.DESKTOP_LOCAL_FOLDER: - return self._normalize_local_mount_path(candidate, runtime) + return self._normalize_absolute_path(candidate) if not candidate.startswith("/"): return f"/{candidate.lstrip('/')}" return candidate @@ -881,7 +848,7 @@ class SurfSenseFilesystemMiddleware(FilesystemMiddleware): if not candidate: return "" if self._filesystem_mode == FilesystemMode.DESKTOP_LOCAL_FOLDER: - return self._normalize_local_mount_path(candidate, runtime) + return self._normalize_absolute_path(candidate) if not candidate.startswith("/"): return f"/{candidate.lstrip('/')}" return candidate @@ -895,7 +862,7 @@ class SurfSenseFilesystemMiddleware(FilesystemMiddleware): if candidate == "/": return "/" if self._filesystem_mode == FilesystemMode.DESKTOP_LOCAL_FOLDER: - return self._normalize_local_mount_path(candidate, runtime) + return self._normalize_absolute_path(candidate) if not candidate.startswith("/"): return f"/{candidate.lstrip('/')}" return candidate @@ -1136,12 +1103,8 @@ class SurfSenseFilesystemMiddleware(FilesystemMiddleware): ] = 8, page_size: Annotated[ int, - "Number of entries to return per page. Defaults to 500 (max 1000).", + "Maximum number of entries to return. Defaults to 500 (max 1000).", ] = 500, - cursor: Annotated[ - str | None, - "Opaque cursor from a previous list_tree call.", - ] = None, include_files: Annotated[ bool, "Whether file entries should be included.", @@ -1171,7 +1134,6 @@ class SurfSenseFilesystemMiddleware(FilesystemMiddleware): validated_path, max_depth=max_depth, page_size=page_size, - cursor=cursor, include_files=include_files, include_dirs=include_dirs, ) @@ -1193,12 +1155,8 @@ class SurfSenseFilesystemMiddleware(FilesystemMiddleware): ] = 8, page_size: Annotated[ int, - "Number of entries to return per page. Defaults to 500 (max 1000).", + "Maximum number of entries to return. Defaults to 500 (max 1000).", ] = 500, - cursor: Annotated[ - str | None, - "Opaque cursor from a previous list_tree call.", - ] = None, include_files: Annotated[ bool, "Whether file entries should be included.", @@ -1228,7 +1186,6 @@ class SurfSenseFilesystemMiddleware(FilesystemMiddleware): validated_path, max_depth=max_depth, page_size=page_size, - cursor=cursor, include_files=include_files, include_dirs=include_dirs, ) diff --git a/surfsense_backend/app/agents/new_chat/middleware/local_folder_backend.py b/surfsense_backend/app/agents/new_chat/middleware/local_folder_backend.py index ef6a1657d..4f149a756 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/local_folder_backend.py +++ b/surfsense_backend/app/agents/new_chat/middleware/local_folder_backend.py @@ -9,9 +9,7 @@ import threading from collections import deque from contextlib import ExitStack from pathlib import Path -from time import time from typing import Any -from uuid import uuid4 from deepagents.backends.protocol import ( EditResult, @@ -43,8 +41,6 @@ class LocalFolderBackend: self._root = root self._locks: dict[str, threading.Lock] = {} self._locks_mu = threading.Lock() - self._tree_sessions: dict[str, dict[str, Any]] = {} - self._tree_sessions_ttl_s = 900 def _lock_for(self, path: str) -> threading.Lock: with self._locks_mu: @@ -89,16 +85,6 @@ class LocalFolderBackend: def _clamp_page_size(page_size: int) -> int: return max(1, min(page_size, 1000)) - def _prune_expired_tree_sessions(self) -> None: - now = time() - expired = [ - cursor - for cursor, session in self._tree_sessions.items() - if now - float(session.get("last_accessed_at", now)) > self._tree_sessions_ttl_s - ] - for cursor in expired: - self._tree_sessions.pop(cursor, None) - def _read_dir_entries(self, directory_path: str) -> list[dict[str, Any]]: directory = Path(directory_path) try: @@ -206,148 +192,82 @@ class LocalFolderBackend: *, max_depth: int | None = 8, page_size: int = 500, - cursor: str | None = None, include_files: bool = True, include_dirs: bool = True, ) -> dict[str, Any]: - self._prune_expired_tree_sessions() if not include_files and not include_dirs: return { "entries": [], - "next_cursor": None, - "has_more": False, "truncated": False, } normalized_depth = None if max_depth is None else max(0, int(max_depth)) page_limit = self._clamp_page_size(int(page_size)) - now = time() - - if cursor: - session = self._tree_sessions.get(cursor) - if not session: - return {"error": "Invalid or expired cursor"} - if ( - session.get("path") != path - or session.get("max_depth") != normalized_depth - or session.get("include_files") != include_files - or session.get("include_dirs") != include_dirs - ): - return {"error": "Cursor options do not match request options"} - state = session - else: - try: - start = self._resolve_virtual(path, allow_root=True) - except ValueError: - return {"error": f"Error: invalid path '{path}'"} - if not start.exists(): - return {"error": f"Error: path '{path}' not found"} - if start.is_file(): - stat_result = start.stat() - if include_files: - return { - "entries": [ - { - "path": self._to_virtual(start, self._root), - "is_dir": False, - "size": stat_result.st_size, - "modified_at": str(stat_result.st_mtime), - "depth": 0, - } - ], - "next_cursor": None, - "has_more": False, - "truncated": False, - } + try: + start = self._resolve_virtual(path, allow_root=True) + except ValueError: + return {"error": f"Error: invalid path '{path}'"} + if not start.exists(): + return {"error": f"Error: path '{path}' not found"} + if start.is_file(): + stat_result = start.stat() + if include_files: return { - "entries": [], - "next_cursor": None, - "has_more": False, + "entries": [ + { + "path": self._to_virtual(start, self._root), + "is_dir": False, + "size": stat_result.st_size, + "modified_at": str(stat_result.st_mtime), + "depth": 0, + } + ], "truncated": False, } - state = { - "path": path, - "max_depth": normalized_depth, - "include_files": include_files, - "include_dirs": include_dirs, - "pending_dirs": deque([(str(start), 0)]), - "active_dir": None, - "active_depth": 0, - "active_entries": [], - "active_index": 0, + return { + "entries": [], + "truncated": False, } + pending_dirs: deque[tuple[str, int]] = deque([(str(start), 0)]) entries: list[dict[str, Any]] = [] truncated = False - while len(entries) < page_limit: - active_entries = state.get("active_entries", []) - active_index = int(state.get("active_index", 0)) - if active_index >= len(active_entries): - pending_dirs = state.get("pending_dirs", []) - if not pending_dirs: - state["active_entries"] = [] - state["active_index"] = 0 - break - next_dir_path, next_depth = pending_dirs.popleft() - state["active_dir"] = next_dir_path - state["active_depth"] = next_depth - state["active_entries"] = self._read_dir_entries(next_dir_path) - state["active_index"] = 0 - active_entries = state["active_entries"] - active_index = 0 - - if active_index >= len(active_entries): - continue - - item = active_entries[active_index] - state["active_index"] = active_index + 1 - item_depth = int(state.get("active_depth", 0)) + 1 - if normalized_depth is not None and item_depth > normalized_depth: - continue - if item["is_dir"]: - if normalized_depth is None or item_depth <= normalized_depth: - state["pending_dirs"].append((item["absolute_path"], item_depth)) - if include_dirs: + while pending_dirs and not truncated: + next_dir_path, next_depth = pending_dirs.popleft() + active_entries = self._read_dir_entries(next_dir_path) + for item in active_entries: + item_depth = next_depth + 1 + if normalized_depth is not None and item_depth > normalized_depth: + continue + if item["is_dir"]: + if normalized_depth is None or item_depth <= normalized_depth: + pending_dirs.append((item["absolute_path"], item_depth)) + if include_dirs: + entries.append( + { + "path": item["path"], + "is_dir": True, + "size": 0, + "modified_at": item["modified_at"], + "depth": item_depth, + } + ) + elif include_files: entries.append( { "path": item["path"], - "is_dir": True, - "size": 0, + "is_dir": False, + "size": item["size"], "modified_at": item["modified_at"], "depth": item_depth, } ) - elif include_files: - entries.append( - { - "path": item["path"], - "is_dir": False, - "size": item["size"], - "modified_at": item["modified_at"], - "depth": item_depth, - } - ) - - if len(entries) >= page_limit: - truncated = True - break - - has_more = bool(state.get("pending_dirs")) or ( - int(state.get("active_index", 0)) < len(state.get("active_entries", [])) - ) - if has_more: - next_cursor = cursor or uuid4().hex - state["last_accessed_at"] = now - self._tree_sessions[next_cursor] = state - else: - next_cursor = None - if cursor: - self._tree_sessions.pop(cursor, None) + if len(entries) >= page_limit: + truncated = True + break return { "entries": entries, - "next_cursor": next_cursor, - "has_more": has_more, "truncated": truncated, } @@ -357,7 +277,6 @@ class LocalFolderBackend: *, max_depth: int | None = 8, page_size: int = 500, - cursor: str | None = None, include_files: bool = True, include_dirs: bool = True, ) -> dict[str, Any]: @@ -366,7 +285,6 @@ class LocalFolderBackend: path, max_depth=max_depth, page_size=page_size, - cursor=cursor, include_files=include_files, include_dirs=include_dirs, ) diff --git a/surfsense_backend/app/agents/new_chat/middleware/multi_root_local_folder_backend.py b/surfsense_backend/app/agents/new_chat/middleware/multi_root_local_folder_backend.py index 6760d76f0..82914f9ce 100644 --- a/surfsense_backend/app/agents/new_chat/middleware/multi_root_local_folder_backend.py +++ b/surfsense_backend/app/agents/new_chat/middleware/multi_root_local_folder_backend.py @@ -3,8 +3,6 @@ from __future__ import annotations import asyncio -import base64 -import json from pathlib import Path from typing import Any @@ -109,28 +107,6 @@ class MultiRootLocalFolderBackend: for mount in self._mount_order ] - @staticmethod - def _encode_tree_cursor(mount: str, local_cursor: str) -> str: - payload = json.dumps( - {"mount": mount, "cursor": local_cursor}, - separators=(",", ":"), - ).encode("utf-8") - return base64.urlsafe_b64encode(payload).decode("ascii") - - @staticmethod - def _decode_tree_cursor(cursor: str) -> tuple[str, str]: - try: - padded = cursor + "=" * ((4 - len(cursor) % 4) % 4) - data = base64.urlsafe_b64decode(padded.encode("ascii")) - parsed = json.loads(data.decode("utf-8")) - except Exception as exc: - raise ValueError("Invalid cursor") from exc - mount = parsed.get("mount") - local_cursor = parsed.get("cursor") - if not isinstance(mount, str) or not isinstance(local_cursor, str): - raise ValueError("Invalid cursor") - return mount, local_cursor - def _transform_infos(self, mount: str, infos: list[FileInfo]) -> list[FileInfo]: transformed: list[FileInfo] = [] for info in infos: @@ -162,11 +138,10 @@ class MultiRootLocalFolderBackend: *, max_depth: int | None = 8, page_size: int = 500, - cursor: str | None = None, include_files: bool = True, include_dirs: bool = True, ) -> dict[str, Any]: - if path == "/" and not cursor: + if path == "/": entries = [ { "path": f"/{mount}", @@ -179,20 +154,11 @@ class MultiRootLocalFolderBackend: ] return { "entries": entries if include_dirs else [], - "next_cursor": None, - "has_more": False, "truncated": False, } try: - if cursor: - mount, local_cursor = self._decode_tree_cursor(cursor) - if mount not in self._mount_to_backend: - return {"error": "Invalid or expired cursor"} - local_path = "/" - else: - mount, local_path = self._split_mount_path(path) - local_cursor = None + mount, local_path = self._split_mount_path(path) except ValueError as exc: return {"error": f"Error: {exc}"} @@ -200,7 +166,6 @@ class MultiRootLocalFolderBackend: local_path, max_depth=max_depth, page_size=page_size, - cursor=local_cursor, include_files=include_files, include_dirs=include_dirs, ) @@ -220,16 +185,8 @@ class MultiRootLocalFolderBackend: } ) - local_next_cursor = self._get_str(result, "next_cursor") - next_cursor = ( - self._encode_tree_cursor(mount, local_next_cursor) - if local_next_cursor - else None - ) return { "entries": entries, - "next_cursor": next_cursor, - "has_more": self._get_bool(result, "has_more"), "truncated": self._get_bool(result, "truncated"), } @@ -239,7 +196,6 @@ class MultiRootLocalFolderBackend: *, max_depth: int | None = 8, page_size: int = 500, - cursor: str | None = None, include_files: bool = True, include_dirs: bool = True, ) -> dict[str, Any]: @@ -248,7 +204,6 @@ class MultiRootLocalFolderBackend: path, max_depth=max_depth, page_size=page_size, - cursor=cursor, include_files=include_files, include_dirs=include_dirs, )