From 8f536cb8d7246229a1142fb1aac5b6e8c140a910 Mon Sep 17 00:00:00 2001
From: mountain <kose2livs@gmail.com>
Date: Wed, 8 Jul 2026 20:02:04 +0800
Subject: [PATCH] fix(index): strip node text in the level_based path too (no
 default leak)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

build_tree_from_levels seeds every node's 'text', but the removal was gated on
`strategy != "level_based"`, so a default Markdown index (level_based,
if_add_node_text=False) leaked each node's full text into
get_document_structure / storage — inconsistent with if_add_node_text=False,
the README, and the legacy md_to_tree.

Move the strip to the end of build_index and apply it to BOTH strategies:
summary/description generation runs first and still sees the text, and
create_clean_structure_for_description doesn't depend on text. From Codex
review of PR #272.

Claude-Session: https://claude.ai/code/session_01Kx5DgKbhK1N8autqXH8SmS
---
 pageindex/index/pipeline.py | 12 +++++++++---
 tests/test_pipeline.py      | 35 +++++++++++++++++++++++++++++++++++
 2 files changed, 44 insertions(+), 3 deletions(-)

diff --git a/pageindex/index/pipeline.py b/pageindex/index/pipeline.py
index 355b12f..6e213e7 100644
--- a/pageindex/index/pipeline.py
+++ b/pageindex/index/pipeline.py
@@ -99,9 +99,6 @@ def build_index(parsed: ParsedDocument, model: str = None, opt=None) -> dict:
         if opt.if_add_node_summary:
             _run_async(generate_summaries_for_structure(structure, model=opt.model))
 
-            if not opt.if_add_node_text and strategy != "level_based":
-                remove_structure_text(structure)
-
         result = {
             "doc_name": parsed.doc_name,
             "structure": structure,
@@ -113,6 +110,15 @@ def build_index(parsed: ParsedDocument, model: str = None, opt=None) -> dict:
                 clean_structure, model=opt.model
             )
 
+        # 'text' may have been populated for summary/description generation, or
+        # by build_tree_from_levels for the level_based (Markdown) path. Strip it
+        # LAST, for BOTH strategies, unless explicitly requested — otherwise a
+        # default index leaks each node's full text into get_document_structure /
+        # storage, inconsistent with if_add_node_text=False, the README, and the
+        # legacy md_to_tree.
+        if not opt.if_add_node_text:
+            remove_structure_text(structure)
+
         return result
 
 
diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py
index ce66b63..3325cf1 100644
--- a/tests/test_pipeline.py
+++ b/tests/test_pipeline.py
@@ -95,6 +95,41 @@ def test_null_logger_methods():
     logger.info({"key": "value"})
 
 
+def _structure_has_text(nodes) -> bool:
+    for n in nodes:
+        if "text" in n:
+            return True
+        if n.get("nodes") and _structure_has_text(n["nodes"]):
+            return True
+    return False
+
+
+def test_level_based_strips_text_by_default():
+    """Markdown (level_based) must honor if_add_node_text=False — build_tree_from_
+    levels seeds 'text', and it used to leak into the output/storage."""
+    from pageindex.config import IndexConfig
+    nodes = [
+        ContentNode(content="# Intro\nbody one", tokens=5, title="Intro", index=1, level=1),
+        ContentNode(content="## Sub\nbody two", tokens=5, title="Sub", index=2, level=2),
+    ]
+    parsed = ParsedDocument(doc_name="d", nodes=nodes)
+    # No summary/description -> no LLM calls.
+    opt = IndexConfig(if_add_node_summary=False, if_add_doc_description=False,
+                      if_add_node_text=False)
+    result = build_index(parsed, opt=opt)
+    assert not _structure_has_text(result["structure"])
+
+
+def test_level_based_keeps_text_when_requested():
+    from pageindex.config import IndexConfig
+    nodes = [ContentNode(content="# Intro\nbody", tokens=5, title="Intro", index=1, level=1)]
+    parsed = ParsedDocument(doc_name="d", nodes=nodes)
+    opt = IndexConfig(if_add_node_summary=False, if_add_doc_description=False,
+                      if_add_node_text=True)
+    result = build_index(parsed, opt=opt)
+    assert _structure_has_text(result["structure"])
+
+
 def test_check_title_appearance_tolerates_out_of_range_physical_index():
     """An LLM-emitted physical_index outside page_list must be marked 'no', not
     raise IndexError (which happens during task construction, outside the