fix(index): strip node text in the level_based path too (no default leak)

build_tree_from_levels seeds every node's 'text', but the removal was gated on
`strategy != "level_based"`, so a default Markdown index (level_based,
if_add_node_text=False) leaked each node's full text into
get_document_structure / storage — inconsistent with if_add_node_text=False,
the README, and the legacy md_to_tree.

Move the strip to the end of build_index and apply it to BOTH strategies:
summary/description generation runs first and still sees the text, and
create_clean_structure_for_description doesn't depend on text. From Codex
review of PR #272.

Claude-Session: https://claude.ai/code/session_01Kx5DgKbhK1N8autqXH8SmS
This commit is contained in:
mountain 2026-07-08 20:02:04 +08:00
parent 89132cb94c
commit 8f536cb8d7
2 changed files with 44 additions and 3 deletions

View file

@ -95,6 +95,41 @@ def test_null_logger_methods():
logger.info({"key": "value"})
def _structure_has_text(nodes) -> bool:
for n in nodes:
if "text" in n:
return True
if n.get("nodes") and _structure_has_text(n["nodes"]):
return True
return False
def test_level_based_strips_text_by_default():
"""Markdown (level_based) must honor if_add_node_text=False — build_tree_from_
levels seeds 'text', and it used to leak into the output/storage."""
from pageindex.config import IndexConfig
nodes = [
ContentNode(content="# Intro\nbody one", tokens=5, title="Intro", index=1, level=1),
ContentNode(content="## Sub\nbody two", tokens=5, title="Sub", index=2, level=2),
]
parsed = ParsedDocument(doc_name="d", nodes=nodes)
# No summary/description -> no LLM calls.
opt = IndexConfig(if_add_node_summary=False, if_add_doc_description=False,
if_add_node_text=False)
result = build_index(parsed, opt=opt)
assert not _structure_has_text(result["structure"])
def test_level_based_keeps_text_when_requested():
from pageindex.config import IndexConfig
nodes = [ContentNode(content="# Intro\nbody", tokens=5, title="Intro", index=1, level=1)]
parsed = ParsedDocument(doc_name="d", nodes=nodes)
opt = IndexConfig(if_add_node_summary=False, if_add_doc_description=False,
if_add_node_text=True)
result = build_index(parsed, opt=opt)
assert _structure_has_text(result["structure"])
def test_check_title_appearance_tolerates_out_of_range_physical_index():
"""An LLM-emitted physical_index outside page_list must be marked 'no', not
raise IndexError (which happens during task construction, outside the