chore: trim fix-rationale comments, restore --model help, drop stale gitignore entry

.gitignore

@@ -10,7 +10,6 @@ dist/
 venv/
 uv.lock
 
-# local SDK test-run artifacts (generated by demos; keep tracked example json)
+# local SDK test-run artifacts (generated by demos)
 examples/workspace/files/
 examples/workspace/*.db
-examples/documents/attention.pdf

examples/agentic_vectorless_rag_demo.py

@@ -139,8 +139,6 @@ def query_agent(collection, doc_id: str, prompt: str, model: str, verbose: bool
             print()
         return "" if not streamed_run.final_output else str(streamed_run.final_output)
 
-    # Only the detection is guarded, not the run, so a real error inside _run
-    # isn't misread as "no running loop".
     try:
         asyncio.get_running_loop()
     except RuntimeError:

pageindex/__init__.py

@@ -1,22 +1,15 @@
 # pageindex/__init__.py
-# Load .env explicitly, before anything else, so environment-based credentials
-# (OPENAI_API_KEY for local mode, PAGEINDEX_API_KEY that callers read via
-# os.environ for cloud mode) are populated by PageIndex itself — not left to
-# litellm's incidental dotenv loading, which would vanish if litellm changes or
-# its import is ever made lazy.
+# Load .env first so env-based credentials (OPENAI_API_KEY, PAGEINDEX_API_KEY) are set.
 from dotenv import load_dotenv as _load_dotenv
 _load_dotenv()
 
-# Backward compatibility: honor CHATGPT_API_KEY as an alias for OPENAI_API_KEY
-# (kept from the pre-SDK pageindex.utils). Runs after load_dotenv so a value in
-# .env is picked up too; only fills OPENAI_API_KEY when it isn't already set.
+# Backward compatibility: honor CHATGPT_API_KEY as an alias for OPENAI_API_KEY.
 import os as _os
 if not _os.getenv("OPENAI_API_KEY") and _os.getenv("CHATGPT_API_KEY"):
     _os.environ["OPENAI_API_KEY"] = _os.getenv("CHATGPT_API_KEY")
 
-# Upstream exports (backward compatibility). Import from the canonical
-# pageindex.index.* modules directly so `import pageindex` does NOT trip the
-# top-level deprecation shims (pageindex.page_index / .page_index_md / .utils).
+# Upstream exports (backward compatibility); import from the canonical index.*
+# modules so plain `import pageindex` doesn't trip the deprecation shims.
 from .index.page_index import *  # noqa: E402
 from .index.page_index_md import md_to_tree
 from .retrieve import get_document, get_document_structure, get_page_content

pageindex/index/pipeline.py

@@ -51,19 +51,12 @@ def _run_async(coro):
     import asyncio
     import concurrent.futures
     import contextvars
-    # Only the detection is guarded — NOT the run. If the coroutine's own work
-    # raises RuntimeError, letting it fall into `except RuntimeError` here would
-    # misfire the "no running loop" branch and mask the real error behind a
-    # bogus "asyncio.run() cannot be called from a running event loop".
     try:
         asyncio.get_running_loop()
     except RuntimeError:
-        # No running loop -- drive the coroutine directly.
         return asyncio.run(coro)
-    # Already inside an event loop -- run in a separate thread so we don't nest
-    # asyncio.run. Copy the current context so ContextVar-based settings (e.g.
-    # the max_concurrency_scope override set by build_index) propagate into the
-    # worker thread; .result() re-raises the worker's real exception unchanged.
+    # In a running loop: run in a worker thread, with the current context copied
+    # so ContextVar-based settings propagate.
     ctx = contextvars.copy_context()
     with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
         return pool.submit(ctx.run, asyncio.run, coro).result()
@@ -106,8 +99,7 @@ def build_index(parsed: ParsedDocument, model: str = None, opt=None) -> dict:
 
         if opt.if_add_node_summary:
             if strategy == "level_based":
-                # Markdown keeps the legacy summarizer: nodes under 200 tokens
-                # reuse their text instead of spending an LLM call.
+                # Markdown: legacy summarizer — nodes under 200 tokens reuse their text.
                 from .page_index_md import generate_summaries_for_structure_md
                 _run_async(generate_summaries_for_structure_md(
                     structure, summary_token_threshold=200, model=opt.model))
@@ -125,15 +117,7 @@ def build_index(parsed: ParsedDocument, model: str = None, opt=None) -> dict:
                 clean_structure, model=opt.model
             )
 
-        # 'text' is populated for level_based (Markdown, always) or for
-        # content_based when if_add_node_text/if_add_node_summary requested it.
-        # Strip it LAST, for BOTH strategies, unless explicitly requested —
-        # otherwise a default index leaks each node's full text into
-        # get_document_structure / storage, inconsistent with
-        # if_add_node_text=False, the README, and the legacy md_to_tree. Skip
-        # the walk entirely when text was never added in the first place
-        # (content_based with if_add_node_text=if_add_node_summary=False) —
-        # there's nothing to strip.
+        # Strip 'text' last unless explicitly requested; skip when it was never added.
         text_present = strategy == "level_based" or opt.if_add_node_text or opt.if_add_node_summary
         if text_present and not opt.if_add_node_text:
             remove_structure_text(structure)

pageindex/retrieve.py

@@ -15,9 +15,7 @@ except ImportError:
 # ── Helpers ──────────────────────────────────────────────────────────────────
 
 def _parse_pages(pages: str) -> list[int]:
-    """Parse a pages string like '5-7', '3,8', or '12' into a sorted list of ints.
-    Delegates to the canonical implementation so the two never drift again —
-    this one used to lack the p>=1 filter and the 1000-page DoS cap."""
+    """Parse a pages string like '5-7', '3,8', or '12' into a sorted list of ints."""
     return parse_pages(pages)
 
 
@@ -31,8 +29,7 @@ def _count_pages(doc_info: dict) -> int:
 
 
 def _get_pdf_page_content(doc_info: dict, page_nums: list[int]) -> list[dict]:
-    """Extract text for specific PDF pages (1-indexed). Prefer cached pages,
-    else delegate the file-read fallback to the canonical implementation."""
+    """Extract text for specific PDF pages (1-indexed). Prefer cached pages, fallback to PDF."""
     cached_pages = doc_info.get('pages')
     if cached_pages:
         page_map = {p['page']: p['content'] for p in cached_pages}
@@ -44,8 +41,7 @@ def _get_pdf_page_content(doc_info: dict, page_nums: list[int]) -> list[dict]:
 
 
 def _get_md_page_content(doc_info: dict, page_nums: list[int]) -> list[dict]:
-    """For Markdown documents, 'pages' are line numbers. Delegates to the
-    canonical implementation so the two never drift again."""
+    """For Markdown documents, 'pages' are line numbers."""
     return get_md_page_content(doc_info.get('structure', []), page_nums)
 
 

run_pageindex.py

@@ -2,10 +2,6 @@ import argparse
 import os
 import json
 from pageindex.index.page_index import *
-# Reuse the canonical yes/no coercion (as _cli_bool) instead of a second copy —
-# a bare ``--flag`` (no value) resolves to True via argparse's ``const``; an
-# explicit value keeps the legacy yes/no style working, so ``--flag no`` turns
-# it off. argparse only ever passes a str here (const/default bypass type=).
 from pageindex.index.page_index_md import md_to_tree
 from pageindex.index.utils import _coerce_bool as _cli_bool
 from pageindex.config import IndexConfig
@@ -17,7 +13,7 @@ if __name__ == "__main__":
     parser.add_argument('--pdf_path', type=str, help='Path to the PDF file')
     parser.add_argument('--md_path', type=str, help='Path to the Markdown file')
 
-    parser.add_argument('--model', type=str, default=None, help='Model to use')
+    parser.add_argument('--model', type=str, default=None, help='Model to use (overrides config.yaml)')
 
     parser.add_argument('--toc-check-pages', type=int, default=None,
                       help='Number of pages to check for table of contents (PDF only)')