mirror of
https://github.com/VectifyAI/PageIndex.git
synced 2026-07-15 21:11:05 +02:00
04cb9cb02d
Verified 12 findings from an xhigh-effort review of the prior review-fix
batch; all confirmed real. Most trace back to one root cause: the
build_index() text-stripping fix (8f536cb) correctly stopped Markdown from
leaking full text by default, but broke every path that assumed text could
be re-read later.
Correctness:
- LocalBackend._fill_node_text (get_document(include_text=True)) only handled
PDF's start_index/end_index convention; Markdown nodes use line_num and got
silently empty text. Now handles both.
- get_page_content's Markdown fallback (triggered when a StorageEngine
legitimately returns None from get_pages()) read from the now-text-stripped
structure. It now re-derives from the source file, mirroring the PDF
fallback, so it no longer depends on structure text at all.
- add_document's PDF-only text-stripping branch (with the stale "markdown
needs text in structure for fallback retrieval" comment) is now dead/wrong
since build_index() already applies if_add_node_text uniformly — removed.
- _validate_llm_provider's keyless-provider allowlist was missing several
local LiteLLM providers (xinference, llamafile, triton, oobabooga,
openai_like, docker_model_runner, custom, custom_openai, petals) that need
no API key just like ollama/lm_studio; expanded.
- The three agent-tool closures (get_document, get_document_structure,
get_page_content) had three different not-found patterns; two bypassed the
backend's DocumentNotFoundError entirely. Extracted LocalBackend.
_require_document as the single existence check every method/tool now uses.
- examples/agentic_vectorless_rag_demo.py's hand-rolled Agent() didn't apply
the litellm/ prefix normalization the SDK does internally, so its own
documented "any LiteLLM provider" claim broke for non-openai models.
- cloud delete_collection's cache eviction removed the "folders unavailable"
None sentinel too, forcing a wasted re-fetch; now only pops on a real id.
Cleanup / altitude:
- build_index() skips the remove_structure_text walk entirely when text was
never added (content_based + if_add_node_summary=False + if_add_node_text=
False) instead of a guaranteed no-op tree walk.
- page_index()'s locals()-capture-as-kwargs (fragile by construction) replaced
with an explicit dict of the named parameters.
- run_pageindex.py's _cli_bool and the page_index_md.py legacy shim's
_coerce_bool were duplicate, diverging implementations; both now bind
directly to the canonical pageindex.index.page_index_md._coerce_bool.
- retrieve.py's _get_md_page_content delegated its own traversal instead of
calling the canonical get_md_page_content; now a one-line delegation.
- FileTypeError's docstring now calls out the except-ordering gotcha from
also subclassing ValueError.
17 new regression tests (tests/test_review_fixes_2.py) plus 2 updated in
tests/test_legacy_shims.py for the simplified md_to_tree shim. Full suite:
210 passed, 2 skipped.
Claude-Session: https://claude.ai/code/session_01Kx5DgKbhK1N8autqXH8SmS
103 lines
4 KiB
Python
103 lines
4 KiB
Python
"""The top-level pageindex.page_index / .page_index_md / .utils modules are
|
|
now deprecation shims over the canonical pageindex.index.* modules. These
|
|
tests pin the compatibility contract."""
|
|
import asyncio
|
|
import importlib
|
|
import warnings
|
|
|
|
import pytest
|
|
|
|
|
|
def test_plain_import_pageindex_does_not_warn():
|
|
# `import pageindex` must not route through the deprecation shims.
|
|
with warnings.catch_warnings():
|
|
warnings.simplefilter("error", PendingDeprecationWarning)
|
|
importlib.import_module("pageindex")
|
|
|
|
|
|
@pytest.mark.parametrize("mod", [
|
|
"pageindex.utils",
|
|
"pageindex.page_index",
|
|
"pageindex.page_index_md",
|
|
])
|
|
def test_legacy_submodule_import_warns(mod):
|
|
with warnings.catch_warnings(record=True) as caught:
|
|
warnings.simplefilter("always")
|
|
importlib.reload(importlib.import_module(mod))
|
|
assert any(issubclass(w.category, PendingDeprecationWarning) for w in caught)
|
|
|
|
|
|
def test_legacy_symbols_resolve_through_shims():
|
|
with warnings.catch_warnings():
|
|
warnings.simplefilter("ignore")
|
|
from pageindex.utils import ( # noqa: F401
|
|
get_page_tokens, ConfigLoader, convert_page_to_int,
|
|
get_leaf_nodes, remove_fields,
|
|
)
|
|
from pageindex.page_index import page_index, page_index_main # noqa: F401
|
|
from pageindex.page_index_md import md_to_tree # noqa: F401
|
|
|
|
|
|
def test_canonical_and_shim_share_one_implementation():
|
|
with warnings.catch_warnings():
|
|
warnings.simplefilter("ignore")
|
|
import pageindex.utils as shim
|
|
import pageindex.index.utils as canonical
|
|
# Same function object -> a single source of truth (no divergence possible).
|
|
assert shim.get_leaf_nodes is canonical.get_leaf_nodes
|
|
assert shim.get_page_tokens is canonical.get_page_tokens
|
|
|
|
|
|
def test_get_leaf_nodes_has_331_fix():
|
|
"""Canonical get_leaf_nodes must use .get('nodes'); clean_node deletes the
|
|
key on leaf nodes so [...]['nodes'] would KeyError (issue #330)."""
|
|
from pageindex.index.utils import get_leaf_nodes
|
|
# A leaf node with the 'nodes' key deleted (as clean_node leaves it).
|
|
leaves = get_leaf_nodes({"title": "Leaf", "start_index": 1, "end_index": 2})
|
|
assert leaves == [{"title": "Leaf", "start_index": 1, "end_index": 2}]
|
|
|
|
|
|
def test_configloader_no_longer_needs_config_yaml():
|
|
"""config.yaml was removed; ConfigLoader must build defaults from IndexConfig."""
|
|
from pageindex.index.utils import ConfigLoader
|
|
cfg = ConfigLoader().load({"model": "gpt-5.4"})
|
|
assert cfg.model == "gpt-5.4"
|
|
assert cfg.if_add_node_summary is True # IndexConfig default
|
|
with pytest.raises(ValueError, match="Unknown config keys"):
|
|
ConfigLoader().load({"nope": 1})
|
|
|
|
|
|
def test_md_to_tree_shim_is_the_canonical_function():
|
|
"""The shim no longer wraps md_to_tree with its own coercion — the
|
|
canonical implementation coerces internally, so the shim is a pure
|
|
re-export (single source of truth, can't diverge from the canonical
|
|
behavior)."""
|
|
with warnings.catch_warnings():
|
|
warnings.simplefilter("ignore")
|
|
import pageindex.page_index_md as shim
|
|
import pageindex.index.page_index_md as canonical
|
|
assert shim.md_to_tree is canonical.md_to_tree
|
|
|
|
|
|
def test_md_to_tree_coerces_legacy_yes_no_strings(tmp_path):
|
|
"""A bare 'no' must not read as truthy True — exercised end-to-end (no
|
|
LLM calls needed with summary/description disabled)."""
|
|
from pageindex.index.page_index_md import md_to_tree
|
|
|
|
md_path = tmp_path / "doc.md"
|
|
md_path.write_text("# Title\nbody\n\n## Sub\nmore body\n")
|
|
|
|
result = asyncio.run(md_to_tree(
|
|
md_path=str(md_path),
|
|
if_add_node_summary="no",
|
|
if_add_node_id="yes",
|
|
if_add_doc_description="no",
|
|
))
|
|
assert "doc_description" not in result
|
|
|
|
def _has_summary(nodes):
|
|
return any("summary" in n or (n.get("nodes") and _has_summary(n["nodes"]))
|
|
for n in nodes)
|
|
|
|
assert not _has_summary(result["structure"])
|
|
assert all("node_id" in n for n in result["structure"])
|