mirror of
https://github.com/VectifyAI/PageIndex.git
synced 2026-07-18 21:21:05 +02:00
fix(index): strip node text in the level_based path too (no default leak)
build_tree_from_levels seeds every node's 'text', but the removal was gated on `strategy != "level_based"`, so a default Markdown index (level_based, if_add_node_text=False) leaked each node's full text into get_document_structure / storage — inconsistent with if_add_node_text=False, the README, and the legacy md_to_tree. Move the strip to the end of build_index and apply it to BOTH strategies: summary/description generation runs first and still sees the text, and create_clean_structure_for_description doesn't depend on text. From Codex review of PR #272. Claude-Session: https://claude.ai/code/session_01Kx5DgKbhK1N8autqXH8SmS
This commit is contained in:
parent
89132cb94c
commit
8f536cb8d7
2 changed files with 44 additions and 3 deletions
|
|
@ -95,6 +95,41 @@ def test_null_logger_methods():
|
|||
logger.info({"key": "value"})
|
||||
|
||||
|
||||
def _structure_has_text(nodes) -> bool:
|
||||
for n in nodes:
|
||||
if "text" in n:
|
||||
return True
|
||||
if n.get("nodes") and _structure_has_text(n["nodes"]):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def test_level_based_strips_text_by_default():
|
||||
"""Markdown (level_based) must honor if_add_node_text=False — build_tree_from_
|
||||
levels seeds 'text', and it used to leak into the output/storage."""
|
||||
from pageindex.config import IndexConfig
|
||||
nodes = [
|
||||
ContentNode(content="# Intro\nbody one", tokens=5, title="Intro", index=1, level=1),
|
||||
ContentNode(content="## Sub\nbody two", tokens=5, title="Sub", index=2, level=2),
|
||||
]
|
||||
parsed = ParsedDocument(doc_name="d", nodes=nodes)
|
||||
# No summary/description -> no LLM calls.
|
||||
opt = IndexConfig(if_add_node_summary=False, if_add_doc_description=False,
|
||||
if_add_node_text=False)
|
||||
result = build_index(parsed, opt=opt)
|
||||
assert not _structure_has_text(result["structure"])
|
||||
|
||||
|
||||
def test_level_based_keeps_text_when_requested():
|
||||
from pageindex.config import IndexConfig
|
||||
nodes = [ContentNode(content="# Intro\nbody", tokens=5, title="Intro", index=1, level=1)]
|
||||
parsed = ParsedDocument(doc_name="d", nodes=nodes)
|
||||
opt = IndexConfig(if_add_node_summary=False, if_add_doc_description=False,
|
||||
if_add_node_text=True)
|
||||
result = build_index(parsed, opt=opt)
|
||||
assert _structure_has_text(result["structure"])
|
||||
|
||||
|
||||
def test_check_title_appearance_tolerates_out_of_range_physical_index():
|
||||
"""An LLM-emitted physical_index outside page_list must be marked 'no', not
|
||||
raise IndexError (which happens during task construction, outside the
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue