mirror of
https://github.com/VectifyAI/PageIndex.git
synced 2026-07-15 21:11:05 +02:00
Addresses items 4-8 from the max-effort review of PR #272 (VectifyAI/PageIndex#272). - agent.py: wrap_with_doc_context() strips '<'/'>' from doc_name/doc_description (untrusted: unsanitized filename / LLM-generated from document content) before inserting them into the <docs>...</docs> block, so embedded content can never form a literal </docs> that closes the delimiter early and escapes the untrusted-data boundary SCOPED_SYSTEM_PROMPT relies on. Deterministic per-field transform, doesn't touch the (cacheable) static system prompt. - ConfigLoader.load() (legacy 0.2.x compat) now routes merged overrides through IndexConfig before returning, so a legacy 'no' string gets pydantic's bool coercion instead of surviving as a truthy non-empty string — page_index_main's bare `if opt.if_add_node_summary:` checks (changed from `== 'yes'` elsewhere in this PR) were silently inverting caller intent and firing unwanted billed LLM calls. - verify_toc, process_large_node_recursively, tree_parser, generate_summaries_for_structure, generate_summaries_for_structure_md: added return_exceptions=True to their asyncio.gather calls (llm_completion/ llm_acompletion raise RuntimeError on retry exhaustion, added earlier in this PR), each with a degrade path matching the pattern already used by sibling hardened gathers in the same files. One transient LLM failure no longer aborts the whole document's indexing. - _llm_semaphore is now a true process-wide ceiling (threading.Semaphore, shared across every thread/event loop) instead of one asyncio.Semaphore per event loop -- concurrently indexing N documents on N threads no longer multiplies the effective cap by N. A max_concurrency_scope() override is layered as a second, nested, per-loop restriction that can only tighten the effective cap within the ceiling, never widen past it. - set_llm_params() mutated a bare process-wide dict with no per-call isolation, unlike max_concurrency which already had ContextVar scoping. Added llm_params_scope() (mirrors max_concurrency_scope) + IndexConfig.llm_params, wired into build_index() the same way max_concurrency already was, so concurrent indexing jobs with different llm kwargs don't leak into each other. Adds regression tests for all five. Full suite: 221 passed, 2 skipped (one pre-existing, unrelated flaky cloud-streaming test intermittently fails on rerun; confirmed independent of this change). Claude-Session: https://claude.ai/code/session_01Kx5DgKbhK1N8autqXH8SmS
118 lines
4.7 KiB
Python
118 lines
4.7 KiB
Python
"""The top-level pageindex.page_index / .page_index_md / .utils modules are
|
|
now deprecation shims over the canonical pageindex.index.* modules. These
|
|
tests pin the compatibility contract."""
|
|
import asyncio
|
|
import importlib
|
|
import warnings
|
|
|
|
import pytest
|
|
|
|
|
|
def test_plain_import_pageindex_does_not_warn():
|
|
# `import pageindex` must not route through the deprecation shims.
|
|
with warnings.catch_warnings():
|
|
warnings.simplefilter("error", PendingDeprecationWarning)
|
|
importlib.import_module("pageindex")
|
|
|
|
|
|
@pytest.mark.parametrize("mod", [
|
|
"pageindex.utils",
|
|
"pageindex.page_index",
|
|
"pageindex.page_index_md",
|
|
])
|
|
def test_legacy_submodule_import_warns(mod):
|
|
with warnings.catch_warnings(record=True) as caught:
|
|
warnings.simplefilter("always")
|
|
importlib.reload(importlib.import_module(mod))
|
|
assert any(issubclass(w.category, PendingDeprecationWarning) for w in caught)
|
|
|
|
|
|
def test_legacy_symbols_resolve_through_shims():
|
|
with warnings.catch_warnings():
|
|
warnings.simplefilter("ignore")
|
|
from pageindex.utils import ( # noqa: F401
|
|
get_page_tokens, ConfigLoader, convert_page_to_int,
|
|
get_leaf_nodes, remove_fields,
|
|
)
|
|
from pageindex.page_index import page_index, page_index_main # noqa: F401
|
|
from pageindex.page_index_md import md_to_tree # noqa: F401
|
|
|
|
|
|
def test_canonical_and_shim_share_one_implementation():
|
|
with warnings.catch_warnings():
|
|
warnings.simplefilter("ignore")
|
|
import pageindex.utils as shim
|
|
import pageindex.index.utils as canonical
|
|
# Same function object -> a single source of truth (no divergence possible).
|
|
assert shim.get_leaf_nodes is canonical.get_leaf_nodes
|
|
assert shim.get_page_tokens is canonical.get_page_tokens
|
|
|
|
|
|
def test_get_leaf_nodes_has_331_fix():
|
|
"""Canonical get_leaf_nodes must use .get('nodes'); clean_node deletes the
|
|
key on leaf nodes so [...]['nodes'] would KeyError (issue #330)."""
|
|
from pageindex.index.utils import get_leaf_nodes
|
|
# A leaf node with the 'nodes' key deleted (as clean_node leaves it).
|
|
leaves = get_leaf_nodes({"title": "Leaf", "start_index": 1, "end_index": 2})
|
|
assert leaves == [{"title": "Leaf", "start_index": 1, "end_index": 2}]
|
|
|
|
|
|
def test_configloader_no_longer_needs_config_yaml():
|
|
"""config.yaml was removed; ConfigLoader must build defaults from IndexConfig."""
|
|
from pageindex.index.utils import ConfigLoader
|
|
cfg = ConfigLoader().load({"model": "gpt-5.4"})
|
|
assert cfg.model == "gpt-5.4"
|
|
assert cfg.if_add_node_summary is True # IndexConfig default
|
|
with pytest.raises(ValueError, match="Unknown config keys"):
|
|
ConfigLoader().load({"nope": 1})
|
|
|
|
|
|
def test_configloader_coerces_legacy_yes_no_strings():
|
|
"""A legacy caller passing 'no' must get a real False, not a truthy
|
|
string — page_index_main's `if opt.if_add_node_summary:` checks (bare
|
|
truthy, not `== 'yes'`) would otherwise silently invert caller intent and
|
|
fire unwanted billed LLM calls."""
|
|
from pageindex.index.utils import ConfigLoader
|
|
cfg = ConfigLoader().load({"if_add_node_summary": "no", "if_add_doc_description": "no"})
|
|
assert cfg.if_add_node_summary is False
|
|
assert cfg.if_add_doc_description is False
|
|
assert bool(cfg.if_add_node_summary) is False
|
|
|
|
cfg2 = ConfigLoader().load({"if_add_node_id": "yes"})
|
|
assert cfg2.if_add_node_id is True
|
|
|
|
|
|
def test_md_to_tree_shim_is_the_canonical_function():
|
|
"""The shim no longer wraps md_to_tree with its own coercion — the
|
|
canonical implementation coerces internally, so the shim is a pure
|
|
re-export (single source of truth, can't diverge from the canonical
|
|
behavior)."""
|
|
with warnings.catch_warnings():
|
|
warnings.simplefilter("ignore")
|
|
import pageindex.page_index_md as shim
|
|
import pageindex.index.page_index_md as canonical
|
|
assert shim.md_to_tree is canonical.md_to_tree
|
|
|
|
|
|
def test_md_to_tree_coerces_legacy_yes_no_strings(tmp_path):
|
|
"""A bare 'no' must not read as truthy True — exercised end-to-end (no
|
|
LLM calls needed with summary/description disabled)."""
|
|
from pageindex.index.page_index_md import md_to_tree
|
|
|
|
md_path = tmp_path / "doc.md"
|
|
md_path.write_text("# Title\nbody\n\n## Sub\nmore body\n")
|
|
|
|
result = asyncio.run(md_to_tree(
|
|
md_path=str(md_path),
|
|
if_add_node_summary="no",
|
|
if_add_node_id="yes",
|
|
if_add_doc_description="no",
|
|
))
|
|
assert "doc_description" not in result
|
|
|
|
def _has_summary(nodes):
|
|
return any("summary" in n or (n.get("nodes") and _has_summary(n["nodes"]))
|
|
for n in nodes)
|
|
|
|
assert not _has_summary(result["structure"])
|
|
assert all("node_id" in n for n in result["structure"])
|