fix: empty-list scope, get_tree bool casing, md fence tracking; dedupe base URL

- local get_agent_tools: `set(doc_ids) if doc_ids is not None else None` so
  doc_ids=[] is a scope of nothing (reject all), not open mode. The public
  query path already guarded []; this hardens direct callers.
- legacy get_tree: send summary=true/false (lowercase) instead of Python's
  capitalized True/False, matching the modern CloudBackend and the API.
- markdown parser: track the opening fence character so a ```-fence isn't
  closed by a ~~~ line (CommonMark), keeping '#'-lines inside it out of headings.
- dedupe the cloud base URL: single API_BASE in cloud_api, referenced by
  CloudBackend and PageIndexClient (was three independent copies).

Regression tests for each.
This commit is contained in:
mountain 2026-07-10 11:17:20 +08:00
parent e18ccdeaf8
commit d97231b480
8 changed files with 83 additions and 15 deletions

View file

@ -13,13 +13,12 @@ import urllib.parse
import requests
from typing import AsyncIterator
from ..cloud_api import API_BASE # single source of truth for the cloud base URL
from ..errors import CloudAPIError, DocumentNotFoundError, PageIndexError
from ..events import QueryEvent
logger = logging.getLogger(__name__)
API_BASE = "https://api.pageindex.ai"
_INTERNAL_TOOLS = frozenset({"ToolSearch", "Read", "Grep", "Glob", "Bash", "Edit", "Write"})

View file

@ -252,13 +252,17 @@ class LocalBackend:
- doc_ids=None (open mode): includes ``list_documents``; agent picks docs itself.
- doc_ids=[...] (scoped mode): no ``list_documents``; the other tools
hard-enforce the whitelist and reject out-of-scope doc_ids.
Note ``is not None``: an empty list is a scope of *nothing* (reject every
doc), NOT open mode. Using truthiness would let ``doc_ids=[]`` collapse to
``None`` and silently grant access to the whole collection.
"""
from agents import function_tool
import json
storage = self._storage
col_name = collection
backend = self
scope = set(doc_ids) if doc_ids else None
scope = set(doc_ids) if doc_ids is not None else None
def _reject(doc_id: str) -> str | None:
if scope is not None and doc_id not in scope:

View file

@ -5,6 +5,7 @@ from typing import Any, Iterator
from typing_extensions import deprecated
from .cloud_api import API_BASE
from .collection import Collection
from .config import IndexConfig
from .errors import PageIndexAPIError
@ -50,7 +51,7 @@ class PageIndexClient:
# Or use LocalClient / CloudClient for explicit mode selection
"""
BASE_URL = "https://api.pageindex.ai"
BASE_URL = API_BASE # single source of truth lives in cloud_api
def __init__(self, api_key: str | None = None, model: str = None,
retrieve_model: str = None, storage_path: str = None,

View file

@ -8,11 +8,16 @@ import requests
from .errors import PageIndexAPIError
# Single source of truth for the cloud API base URL — imported by the modern
# CloudBackend (as API_BASE) and PageIndexClient so a staging/migration change
# only has to happen here.
API_BASE = "https://api.pageindex.ai"
class LegacyCloudAPI:
"""Compatibility layer for the pageindex 0.2.x cloud SDK API."""
BASE_URL = "https://api.pageindex.ai"
BASE_URL = API_BASE
def __init__(self, api_key: str, base_url: str | None = None):
self.api_key = api_key
@ -85,7 +90,11 @@ class LegacyCloudAPI:
def get_tree(self, doc_id: str, node_summary: bool = False) -> dict[str, Any]:
response = self._request(
"GET",
f"/doc/{self._enc(doc_id)}/?type=tree&summary={node_summary}",
# Lowercase the bool: a Python f-string renders True/False with a
# capital letter, but the API expects summary=true/false (the modern
# CloudBackend sends lowercase). A case-sensitive server would
# otherwise silently drop node summaries.
f"/doc/{self._enc(doc_id)}/?type=tree&summary={'true' if node_summary else 'false'}",
"Failed to get tree result",
)
return response.json()

View file

@ -28,19 +28,27 @@ class MarkdownParser:
def _extract_headers(self, lines: list[str]) -> list[dict]:
header_pattern = r"^(#{1,6})\s+(.+)$"
# CommonMark allows both backtick and tilde fences; only recognizing
# backticks let a '#'-prefixed line inside a ~~~-fenced block (e.g. a
# shell comment in a code sample) be misparsed as a real heading.
code_block_pattern = r"^(?:```|~~~)"
# CommonMark allows both backtick and tilde fences, and a fence is
# closed only by one of the SAME character. Track which char opened the
# block so a ~~~ line inside a ```-fenced block (or vice versa) is
# treated as content, not a close — otherwise the block appears to end
# early and '#'-prefixed lines inside it get misparsed as headings.
fence_pattern = r"^(`{3,}|~{3,})"
headers = []
in_code_block = False
open_fence = None # the fence char ('`' or '~') of the open block, or None
for line_num, line in enumerate(lines, 1):
stripped = line.strip()
if re.match(code_block_pattern, stripped):
in_code_block = not in_code_block
fence = re.match(fence_pattern, stripped)
if fence:
marker = fence.group(1)[0]
if open_fence is None:
open_fence = marker # open a block
elif open_fence == marker:
open_fence = None # matching char closes it
# a non-matching fence char while a block is open is content
continue
if not in_code_block and stripped:
if open_fence is None and stripped:
match = re.match(header_pattern, stripped)
if match:
headers.append({

View file

@ -135,7 +135,7 @@ def test_get_ocr_and_tree_use_legacy_urls(monkeypatch):
assert get_calls[0]["method"] == "GET"
assert get_calls[0]["url"] == "https://api.pageindex.ai/doc/doc-1/?type=ocr&format=page"
assert get_calls[1]["url"] == "https://api.pageindex.ai/doc/doc-1/?type=tree&summary=True"
assert get_calls[1]["url"] == "https://api.pageindex.ai/doc/doc-1/?type=tree&summary=true"
def test_get_ocr_rejects_invalid_format():
@ -265,6 +265,22 @@ def test_chat_completions_stream_errors_are_pageindex_api_error(monkeypatch):
list(stream)
def test_get_tree_sends_lowercase_summary_bool(monkeypatch):
# A Python f-string renders True/False capitalized; the API expects
# summary=true/false. A case-sensitive server would silently drop summaries.
calls = []
def fake_request(method, url, **kwargs):
calls.append(url)
return FakeResponse(payload={"result": []})
monkeypatch.setattr("pageindex.cloud_api.requests.request", fake_request)
PageIndexClient("pi-test").get_tree("doc-1", node_summary=True)
assert "summary=true" in calls[0] and "summary=True" not in calls[0]
PageIndexClient("pi-test").get_tree("doc-1", node_summary=False)
assert "summary=false" in calls[1]
def test_delete_document_tolerates_empty_success_body(monkeypatch):
# A successful DELETE may return 200 with no body; delete_document must not
# raise JSONDecodeError parsing an empty response (the doc is already gone).

View file

@ -127,6 +127,18 @@ def test_scoped_mode_allows_in_scope_doc_id(populated_backend):
assert out.get("doc_name") == "alpha.pdf"
def test_empty_doc_ids_is_scoped_to_nothing_not_open_mode(populated_backend):
# doc_ids=[] means "scope to no documents", NOT open mode. It must exclude
# list_documents and reject every doc_id — otherwise an empty list would
# collapse to None (truthiness) and silently grant access to the whole
# collection.
tools = populated_backend.get_agent_tools("papers", doc_ids=[])
by_name = {t.name: t for t in tools.function_tools}
assert "list_documents" not in by_name
out = json.loads(_invoke_tool(by_name["get_document"], {"doc_id": "d1"}))
assert "error" in out and "not in scope" in out["error"]
@pytest.mark.parametrize("bad_pages", ["all", "5-", "abc", "3-1"])
def test_get_page_content_returns_actionable_error_for_bad_page_spec(populated_backend, bad_pages):
# A malformed page spec must come back as a correctable JSON error (like the

View file

@ -100,3 +100,22 @@ def test_tilde_fenced_code_blocks_are_recognized(tmp_path):
titles = [n.title for n in result.nodes]
assert titles == ["Real Header", "Real Sub"]
assert "not a real header" not in " ".join(n.title for n in result.nodes)
def test_backtick_fence_is_not_closed_by_a_tilde_line(tmp_path):
"""CommonMark: a ```-opened fence is closed only by ```. A ~~~ line inside
it is content, so a '#'-prefixed line stays inside the still-open block and
a real heading after the real close is still recognized."""
md = tmp_path / "mixed.md"
md.write_text(
"# Real Header\n"
"```\n"
"~~~\n" # tilde line INSIDE the backtick fence — NOT a close
"# not a heading\n" # stays inside the still-open code block
"```\n" # this (matching char) closes the fence
"## Real Sub\n"
)
result = MarkdownParser().parse(str(md))
titles = [n.title for n in result.nodes]
assert titles == ["Real Header", "Real Sub"]
assert "not a heading" not in " ".join(n.title for n in result.nodes)