Merge Goal 4: route agent retrieval through browse

Merge ask/chat retrieval strategy updates into feat/pageindex-filesystem.

examples/pifs_demo.py

@@ -4,8 +4,8 @@ PageIndex FileSystem (PIFS) agent demo.
 This mirrors examples/agentic_vectorless_rag_demo.py, but exposes a corpus
 through the PageIndex FileSystem shell instead of direct PageIndex document
 tools. The agent receives one read-only bash-like PIFS tool and must retrieve
-evidence through commands such as ls, tree, find, grep, browse,
-cat <path> --structure, cat <path> --page, and cat <path> --node.
+evidence through commands such as ls, tree, browse, find, grep, cat <path>
+--structure, cat <path> --page, and cat <path> --node.
 
 The demo registers supported files under examples/documents. When a matching
 examples/documents/results/*_structure.json file exists, it is loaded into the
@@ -72,9 +72,15 @@ Retrieval strategy:
   or stable file_ref/document ids. Do not invent temporary ref_N aliases.
 - Folder paths such as /documents are positional command targets; do not put
   folder paths inside --where.
-- Use browse when available to find likely documents by semantic relevance.
-  Quote multi-word queries and include a path, for example:
+- After choosing a folder, use browse with a required quoted query to find
+  likely files, for example:
   browse /documents "Federal Reserve supervision regulation"
+- If the folder is uncertain, use recursive browse from a structural parent,
+  for example:
+  browse -R /documents "Federal Reserve supervision regulation"
+- browse returns file candidates only; it is not folder semantic recall.
+- After browse returns candidates, verify evidence with grep, cat <path>
+  --structure, cat <path> --node, or cat <path> --page before answering.
 - Use find --where only with JSON metadata DSL, for example:
   find /documents --where '{"file_format":"pdf"}'
 - Use grep -R only for lexical evidence; do not treat semantic candidates as
@@ -643,14 +649,14 @@ def run_smoke_commands(
     )
 
     command = 'browse /documents "Federal Reserve annual report supervision regulation section page range"'
-    summary = execute_json_command(json_executor, command)
-    summary_hits = ((summary.get("data") or {}).get("data") or [])
-    if summary_hits:
-        summary_result = f"{len(summary_hits)} browse candidates; top={summary_hits[0].get('external_id')}"
+    browse = execute_json_command(json_executor, command)
+    browse_hits = ((browse.get("data") or {}).get("data") or [])
+    if browse_hits:
+        summary_result = f"{len(browse_hits)} browse candidates; top={browse_hits[0].get('external_id')}"
     else:
         summary_result = "browse is available, but this tiny two-doc demo returned no candidates"
     show_capability(
-        label="Semantic browse",
+        label="Relevance browse",
         command=command,
         result=summary_result,
         raw=shell_executor.execute(command) if verbose else "",

pageindex/filesystem/agent.py

@@ -35,17 +35,19 @@ document contents in the workspace.
 
 If the user asks what tools or capabilities you have, describe only the PIFS
 virtual shell capabilities available inside this workspace: ls, tree, find,
-stat, grep, cat, and browse. Do not mention host runtime tools, SDK internals,
-or orchestration helpers that are not part of the PIFS shell.
+stat, grep, cat, and browse when they are available. Do not mention host
+runtime tools, SDK internals, or orchestration helpers that are not part of the
+PIFS shell.
 
 If the user asks a workspace-related topic question without naming a specific
-file, treat it as a retrieval task. Use available PIFS discovery commands to
-look for relevant files and inspect evidence before answering. Ask the user to
-clarify only after a reasonable search cannot identify relevant evidence.
+file, treat it as a retrieval task. Start with ls or tree to understand the
+folder structure, choose a folder, then use browse with the user's topic as the
+query to find candidate files. Inspect evidence before answering. Ask the user
+to clarify only after a reasonable search cannot identify relevant evidence.
 Do not conclude that no relevant document exists from one failed grep. If grep
-returns no matches for a workspace topic, verify with available semantic
-candidate discovery through browse, or inspect likely document structure,
-before saying that the workspace lacks evidence.
+returns no matches for a workspace topic, use browse on a relevant folder or
+inspect likely document structure before saying that the workspace lacks
+evidence.
 
 Follow the task prompt for command policy, retrieval strategy, and answer
 format. If the caller needs stricter behavior, pass an explicit system_prompt.
@@ -54,25 +56,24 @@ format. If the caller needs stricter behavior, pass an explicit system_prompt.
 BASH_TOOL_DESCRIPTION = """
 Run a command in the PageIndex FileSystem virtual shell. This is not a real
 operating-system shell. By default the tool is read-only: use ls, tree, find,
-grep, cat, stat, head, tail, sed, and browse as described in the workspace
-context. grep -R is lexical evidence search;
-grep does not support regex alternation such as "a|b"; run multiple grep
-commands or use browse for semantic candidate discovery instead. browse returns
-candidate documents ranked by relevance and does not guarantee literal text
-matches or final answer evidence. After choosing a likely browse candidate,
-verify the relevant claim with cat before answering. Use browse when the user
-asks for summary search, semantic search, or vector search and the command is
-listed as available. Quote multi-word semantic queries, for example:
-browse /documents "Federal Reserve". Do not write
-browse /documents Federal Reserve. Errors are returned as text prefixed with
-ERROR. Do not call
-commands that are not listed as available. When evidence is required, inspect it
-with cat or grep before answering. Prefer shell-like target-first cat syntax
-with stable targets: cat <path> --structure, cat <path> --page 31-59, and
-cat <path> --node 0009. You may also use file_ref or document_id when a path is
-ambiguous. Do not reconstruct paths from document titles; use exact targets
-returned by PIFS commands and quote paths containing spaces. After structure
-identifies a relevant section node, prefer
+grep, cat, stat, head, tail, sed, and browse when listed in the workspace
+context. grep -R is lexical evidence search; grep does not support regex
+alternation such as "a|b"; run multiple grep commands or use browse for
+relevance-ranked file discovery instead. Start broad workspace questions with
+ls or tree to understand folders. After choosing a folder, use positional
+browse syntax with a quoted query, for example:
+browse /documents "Federal Reserve". If the relevant folder is uncertain, use
+browse -R /documents "Federal Reserve" to retrieve file candidates across that
+folder tree. browse returns file candidates only; it does not perform folder
+semantic recall and does not guarantee final answer evidence. After choosing a
+likely browse candidate, verify the relevant claim with cat or grep before
+answering. Errors are returned as text prefixed with ERROR. Do not call commands
+that are not listed as available. When evidence is required, inspect it with cat
+or grep before answering. Prefer shell-like target-first cat syntax with stable
+targets: cat <path> --structure, cat <path> --page 31-59, and cat <path> --node
+0009. You may also use file_ref or document_id when a path is ambiguous. Do not reconstruct paths from document titles; use exact targets returned by PIFS
+commands and quote paths containing spaces. After structure identifies a
+relevant section node, prefer
 cat <path> --node <node_id>; use cat <path> --page <range> when the user asks
 for page-level evidence, no suitable node exists, or exact page text is needed.
 cat <path> --structure is paginated; request more with --offset if needed. Page
@@ -83,8 +84,8 @@ continue with another chunk before answering.
 For questions about metadata fields, available summaries, or whether metadata
 was provided, inspect stat --schema and stat <target> before making claims.
 Do not use stat as a general content/topic discovery step. For document Q&A,
-prefer ls/tree to choose a folder, browse/find/grep for candidates, then cat --structure and
-cat --node or cat --page for evidence.
+prefer ls/tree for folder selection, browse for file candidates, then cat
+--structure and cat --node or cat --page for evidence.
 """
 
 AGENT_TOOL_POLICY = """
@@ -94,12 +95,16 @@ Tool policy:
 - Use only commands listed in the workspace capabilities.
 - Folder paths such as /documents are positional command targets; never put folder paths in --where.
 - Use --where only with metadata fields shown by stat --schema.
+- Start with ls or tree to understand workspace and folder structure before semantic file retrieval.
+- After choosing a folder, use browse <folder> "<query>" for relevance-ranked file candidates; quote multi-word queries, for example browse /documents "Federal Reserve".
+- If the relevant folder is uncertain, use browse -R <folder> "<query>" to search recursively from a structural parent folder.
+- browse returns file candidates only; Do not use browse as folder semantic recall.
+- browse candidates are not final evidence. After selecting candidates, verify the relevant facts with cat or grep before making source-backed claims.
 - grep -R performs lexical evidence search.
-- grep does not support regex alternation such as "a|b"; run separate grep commands or use browse for semantic candidate discovery.
-- browse is the semantic candidate-discovery tool and does not guarantee literal text matches or final answer evidence. After selecting a likely browse candidate, verify the relevant facts with cat before answering.
+- grep does not support regex alternation such as "a|b"; run separate grep commands or use browse for relevance-ranked file discovery.
 - Do not use find | grep as an exhaustive search or as proof that no document exists; find output can be scoped or limited. Use metadata filters, browse, grep on a narrowed target, or cat on likely candidates instead.
-- A single failed grep is not enough evidence to say there is no relevant document. If grep returns no matches for a workspace-topic question, verify with browse or inspect likely document structure, before answering no-evidence.
-- If the user asks for summary search, semantic search, vector search, or "用 summary 搜", use browse <folder> "<query>"; quote multi-word queries, for example browse /documents "Federal Reserve"; use browse -R <folder> when the folder choice is uncertain; do not translate that request into find --where.
+- A single failed grep is not enough evidence to say there is no relevant document. If grep returns no matches for a workspace-topic question, verify with browse on a relevant folder or inspect likely document structure before answering no-evidence.
+- If the user asks for summary search, semantic search, vector search, or "用 summary 搜", use browse <folder> "<query>" with the default summary space; do not translate that request into find --where.
 - Tool errors are returned as ERROR text; recover by trying an available command.
 - Use cat or grep to gather evidence before making source-backed claims.
 - Do not reconstruct a file path from a title. Use exact paths returned by PIFS commands, or use file_ref/document_id when available; quote paths that contain spaces.

tests/test_pifs_agent_stream.py

@@ -1,7 +1,10 @@
+import ast
 import io
 import os
+import tempfile
 import threading
 import unittest
+from pathlib import Path
 from unittest.mock import patch
 from types import SimpleNamespace
 
@@ -15,6 +18,7 @@ from pageindex.filesystem.agent import (
     PIFSAgentSession,
     PIFSAgentStreamObserver,
     build_agent_model_settings,
+    build_pifs_agent_instructions,
     normalize_agent_stream_mode,
     normalize_reasoning_effort,
     normalize_reasoning_summary,
@@ -23,6 +27,22 @@ from pageindex.filesystem.agent import (
     should_disable_pifs_agent_tracing,
     should_use_openai_compatible_chat_model,
 )
+from pageindex.filesystem import PageIndexFileSystem
+
+
+def load_demo_agent_prompt() -> str:
+    demo_path = Path(__file__).resolve().parents[1] / "examples" / "pifs_demo.py"
+    module = ast.parse(demo_path.read_text(encoding="utf-8"))
+    for node in module.body:
+        if isinstance(node, ast.Assign):
+            names = [
+                target.id
+                for target in node.targets
+                if isinstance(target, ast.Name)
+            ]
+            if "PIFS_DEMO_AGENT_PROMPT" in names and isinstance(node.value, ast.Constant):
+                return str(node.value.value)
+    raise AssertionError("PIFS_DEMO_AGENT_PROMPT not found")
 
 
 class StructuredAnswer(BaseModel):
@@ -215,22 +235,43 @@ class PIFSAgentStreamTest(unittest.TestCase):
         self.assertIn("Do not run stat merely to understand what a document says", AGENT_TOOL_POLICY)
         self.assertIn("Do not use stat as a general content/topic discovery step", BASH_TOOL_DESCRIPTION)
 
-    def test_prompt_routes_semantic_search_to_browse(self):
+    def test_prompt_routes_topic_retrieval_through_browse_after_folder_exploration(self):
+        self.assertIn("Start with ls or tree", AGENT_TOOL_POLICY)
+        self.assertIn('browse <folder> "<query>"', AGENT_TOOL_POLICY)
+        self.assertIn('browse /documents "Federal Reserve"', BASH_TOOL_DESCRIPTION)
+        self.assertIn("If the relevant folder is uncertain", AGENT_TOOL_POLICY)
+        self.assertIn('browse -R <folder> "<query>"', AGENT_TOOL_POLICY)
+        self.assertIn("browse returns file candidates only", AGENT_TOOL_POLICY)
+        self.assertIn("verify the relevant facts with cat or grep", AGENT_TOOL_POLICY)
+        self.assertIn("cat <target> --structure", AGENT_TOOL_POLICY)
+        self.assertIn("cat <target> --node <node_id>", AGENT_TOOL_POLICY)
+        self.assertIn("cat <target> --page", AGENT_TOOL_POLICY)
+        self.assertIn("Do not use browse as folder semantic recall", AGENT_TOOL_POLICY)
+
+    def test_default_agent_prompts_do_not_suggest_legacy_semantic_commands(self):
+        prompt_surface = "\n".join(
+            [AGENT_SYSTEM_PROMPT, BASH_TOOL_DESCRIPTION, AGENT_TOOL_POLICY]
+        )
+
         for old_command in (
             "search-summary",
             "search-entity",
             "search-relation",
             "semantic-grep",
+            "find --name",
+            "find --relation",
         ):
-            self.assertNotIn(old_command, BASH_TOOL_DESCRIPTION)
-            self.assertNotIn(old_command, AGENT_TOOL_POLICY)
-        self.assertIn("Use browse when the user", BASH_TOOL_DESCRIPTION)
-        self.assertIn('use browse <folder> "<query>"', AGENT_TOOL_POLICY)
-        self.assertIn('browse /documents "Federal Reserve"', BASH_TOOL_DESCRIPTION)
-        self.assertIn("browse -R <folder>", AGENT_TOOL_POLICY)
-        self.assertIn("do not translate that request into find --where", AGENT_TOOL_POLICY)
-        self.assertIn("verify the relevant facts with cat", AGENT_TOOL_POLICY)
-        self.assertIn("verify the relevant claim with cat", BASH_TOOL_DESCRIPTION)
+            self.assertNotIn(old_command, prompt_surface)
+
+    def test_demo_prompt_uses_browse_strategy_and_not_legacy_semantic_search(self):
+        demo_prompt = load_demo_agent_prompt()
+
+        self.assertIn("Start with ls or tree", demo_prompt)
+        self.assertIn('browse /documents "Federal Reserve supervision regulation"', demo_prompt)
+        self.assertIn('browse -R /documents "Federal Reserve supervision regulation"', demo_prompt)
+        self.assertIn("verify", demo_prompt)
+        self.assertIn("cat <path> --structure", demo_prompt)
+        self.assertNotIn("search-summary", demo_prompt)
 
     def test_prompt_rejects_find_grep_as_exhaustive_search(self):
         self.assertIn("Do not use find | grep as an exhaustive search", AGENT_TOOL_POLICY)