diff --git a/pageindex/backend/cloud.py b/pageindex/backend/cloud.py index c57734c..ae6e8da 100644 --- a/pageindex/backend/cloud.py +++ b/pageindex/backend/cloud.py @@ -13,13 +13,12 @@ import urllib.parse import requests from typing import AsyncIterator +from ..cloud_api import API_BASE # single source of truth for the cloud base URL from ..errors import CloudAPIError, DocumentNotFoundError, PageIndexError from ..events import QueryEvent logger = logging.getLogger(__name__) -API_BASE = "https://api.pageindex.ai" - _INTERNAL_TOOLS = frozenset({"ToolSearch", "Read", "Grep", "Glob", "Bash", "Edit", "Write"}) diff --git a/pageindex/backend/local.py b/pageindex/backend/local.py index ab377f7..feb695c 100644 --- a/pageindex/backend/local.py +++ b/pageindex/backend/local.py @@ -252,13 +252,17 @@ class LocalBackend: - doc_ids=None (open mode): includes ``list_documents``; agent picks docs itself. - doc_ids=[...] (scoped mode): no ``list_documents``; the other tools hard-enforce the whitelist and reject out-of-scope doc_ids. + + Note ``is not None``: an empty list is a scope of *nothing* (reject every + doc), NOT open mode. Using truthiness would let ``doc_ids=[]`` collapse to + ``None`` and silently grant access to the whole collection. """ from agents import function_tool import json storage = self._storage col_name = collection backend = self - scope = set(doc_ids) if doc_ids else None + scope = set(doc_ids) if doc_ids is not None else None def _reject(doc_id: str) -> str | None: if scope is not None and doc_id not in scope: diff --git a/pageindex/client.py b/pageindex/client.py index 41d16da..de0dd83 100644 --- a/pageindex/client.py +++ b/pageindex/client.py @@ -5,6 +5,7 @@ from typing import Any, Iterator from typing_extensions import deprecated +from .cloud_api import API_BASE from .collection import Collection from .config import IndexConfig from .errors import PageIndexAPIError @@ -50,7 +51,7 @@ class PageIndexClient: # Or use LocalClient / CloudClient for explicit mode selection """ - BASE_URL = "https://api.pageindex.ai" + BASE_URL = API_BASE # single source of truth lives in cloud_api def __init__(self, api_key: str | None = None, model: str = None, retrieve_model: str = None, storage_path: str = None, diff --git a/pageindex/cloud_api.py b/pageindex/cloud_api.py index 1e583d7..29a28bc 100644 --- a/pageindex/cloud_api.py +++ b/pageindex/cloud_api.py @@ -8,11 +8,16 @@ import requests from .errors import PageIndexAPIError +# Single source of truth for the cloud API base URL — imported by the modern +# CloudBackend (as API_BASE) and PageIndexClient so a staging/migration change +# only has to happen here. +API_BASE = "https://api.pageindex.ai" + class LegacyCloudAPI: """Compatibility layer for the pageindex 0.2.x cloud SDK API.""" - BASE_URL = "https://api.pageindex.ai" + BASE_URL = API_BASE def __init__(self, api_key: str, base_url: str | None = None): self.api_key = api_key @@ -85,7 +90,11 @@ class LegacyCloudAPI: def get_tree(self, doc_id: str, node_summary: bool = False) -> dict[str, Any]: response = self._request( "GET", - f"/doc/{self._enc(doc_id)}/?type=tree&summary={node_summary}", + # Lowercase the bool: a Python f-string renders True/False with a + # capital letter, but the API expects summary=true/false (the modern + # CloudBackend sends lowercase). A case-sensitive server would + # otherwise silently drop node summaries. + f"/doc/{self._enc(doc_id)}/?type=tree&summary={'true' if node_summary else 'false'}", "Failed to get tree result", ) return response.json() diff --git a/pageindex/parser/markdown.py b/pageindex/parser/markdown.py index 04b7e22..e09bd05 100644 --- a/pageindex/parser/markdown.py +++ b/pageindex/parser/markdown.py @@ -28,19 +28,27 @@ class MarkdownParser: def _extract_headers(self, lines: list[str]) -> list[dict]: header_pattern = r"^(#{1,6})\s+(.+)$" - # CommonMark allows both backtick and tilde fences; only recognizing - # backticks let a '#'-prefixed line inside a ~~~-fenced block (e.g. a - # shell comment in a code sample) be misparsed as a real heading. - code_block_pattern = r"^(?:```|~~~)" + # CommonMark allows both backtick and tilde fences, and a fence is + # closed only by one of the SAME character. Track which char opened the + # block so a ~~~ line inside a ```-fenced block (or vice versa) is + # treated as content, not a close — otherwise the block appears to end + # early and '#'-prefixed lines inside it get misparsed as headings. + fence_pattern = r"^(`{3,}|~{3,})" headers = [] - in_code_block = False + open_fence = None # the fence char ('`' or '~') of the open block, or None for line_num, line in enumerate(lines, 1): stripped = line.strip() - if re.match(code_block_pattern, stripped): - in_code_block = not in_code_block + fence = re.match(fence_pattern, stripped) + if fence: + marker = fence.group(1)[0] + if open_fence is None: + open_fence = marker # open a block + elif open_fence == marker: + open_fence = None # matching char closes it + # a non-matching fence char while a block is open is content continue - if not in_code_block and stripped: + if open_fence is None and stripped: match = re.match(header_pattern, stripped) if match: headers.append({ diff --git a/tests/test_legacy_sdk_contract.py b/tests/test_legacy_sdk_contract.py index 27cff08..6abab5d 100644 --- a/tests/test_legacy_sdk_contract.py +++ b/tests/test_legacy_sdk_contract.py @@ -135,7 +135,7 @@ def test_get_ocr_and_tree_use_legacy_urls(monkeypatch): assert get_calls[0]["method"] == "GET" assert get_calls[0]["url"] == "https://api.pageindex.ai/doc/doc-1/?type=ocr&format=page" - assert get_calls[1]["url"] == "https://api.pageindex.ai/doc/doc-1/?type=tree&summary=True" + assert get_calls[1]["url"] == "https://api.pageindex.ai/doc/doc-1/?type=tree&summary=true" def test_get_ocr_rejects_invalid_format(): @@ -265,6 +265,22 @@ def test_chat_completions_stream_errors_are_pageindex_api_error(monkeypatch): list(stream) +def test_get_tree_sends_lowercase_summary_bool(monkeypatch): + # A Python f-string renders True/False capitalized; the API expects + # summary=true/false. A case-sensitive server would silently drop summaries. + calls = [] + + def fake_request(method, url, **kwargs): + calls.append(url) + return FakeResponse(payload={"result": []}) + + monkeypatch.setattr("pageindex.cloud_api.requests.request", fake_request) + PageIndexClient("pi-test").get_tree("doc-1", node_summary=True) + assert "summary=true" in calls[0] and "summary=True" not in calls[0] + PageIndexClient("pi-test").get_tree("doc-1", node_summary=False) + assert "summary=false" in calls[1] + + def test_delete_document_tolerates_empty_success_body(monkeypatch): # A successful DELETE may return 200 with no body; delete_document must not # raise JSONDecodeError parsing an empty response (the doc is already gone). diff --git a/tests/test_local_backend.py b/tests/test_local_backend.py index 1bb9662..6bbb346 100644 --- a/tests/test_local_backend.py +++ b/tests/test_local_backend.py @@ -127,6 +127,18 @@ def test_scoped_mode_allows_in_scope_doc_id(populated_backend): assert out.get("doc_name") == "alpha.pdf" +def test_empty_doc_ids_is_scoped_to_nothing_not_open_mode(populated_backend): + # doc_ids=[] means "scope to no documents", NOT open mode. It must exclude + # list_documents and reject every doc_id — otherwise an empty list would + # collapse to None (truthiness) and silently grant access to the whole + # collection. + tools = populated_backend.get_agent_tools("papers", doc_ids=[]) + by_name = {t.name: t for t in tools.function_tools} + assert "list_documents" not in by_name + out = json.loads(_invoke_tool(by_name["get_document"], {"doc_id": "d1"})) + assert "error" in out and "not in scope" in out["error"] + + @pytest.mark.parametrize("bad_pages", ["all", "5-", "abc", "3-1"]) def test_get_page_content_returns_actionable_error_for_bad_page_spec(populated_backend, bad_pages): # A malformed page spec must come back as a correctable JSON error (like the diff --git a/tests/test_markdown_parser.py b/tests/test_markdown_parser.py index bffda98..521c059 100644 --- a/tests/test_markdown_parser.py +++ b/tests/test_markdown_parser.py @@ -100,3 +100,22 @@ def test_tilde_fenced_code_blocks_are_recognized(tmp_path): titles = [n.title for n in result.nodes] assert titles == ["Real Header", "Real Sub"] assert "not a real header" not in " ".join(n.title for n in result.nodes) + + +def test_backtick_fence_is_not_closed_by_a_tilde_line(tmp_path): + """CommonMark: a ```-opened fence is closed only by ```. A ~~~ line inside + it is content, so a '#'-prefixed line stays inside the still-open block and + a real heading after the real close is still recognized.""" + md = tmp_path / "mixed.md" + md.write_text( + "# Real Header\n" + "```\n" + "~~~\n" # tilde line INSIDE the backtick fence — NOT a close + "# not a heading\n" # stays inside the still-open code block + "```\n" # this (matching char) closes the fence + "## Real Sub\n" + ) + result = MarkdownParser().parse(str(md)) + titles = [n.title for n in result.nodes] + assert titles == ["Real Header", "Real Sub"] + assert "not a heading" not in " ".join(n.title for n in result.nodes)