diff --git a/.gitignore b/.gitignore index 4825407..ddfb4d7 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,6 @@ dist/ venv/ uv.lock -# local SDK test-run artifacts (generated by demos; keep tracked example json) +# local SDK test-run artifacts (generated by demos) examples/workspace/files/ examples/workspace/*.db -examples/documents/attention.pdf diff --git a/examples/agentic_vectorless_rag_demo.py b/examples/agentic_vectorless_rag_demo.py index 1526bee..ac4b58c 100644 --- a/examples/agentic_vectorless_rag_demo.py +++ b/examples/agentic_vectorless_rag_demo.py @@ -139,8 +139,6 @@ def query_agent(collection, doc_id: str, prompt: str, model: str, verbose: bool print() return "" if not streamed_run.final_output else str(streamed_run.final_output) - # Only the detection is guarded, not the run, so a real error inside _run - # isn't misread as "no running loop". try: asyncio.get_running_loop() except RuntimeError: diff --git a/pageindex/__init__.py b/pageindex/__init__.py index 4e15919..46971b4 100644 --- a/pageindex/__init__.py +++ b/pageindex/__init__.py @@ -1,22 +1,15 @@ # pageindex/__init__.py -# Load .env explicitly, before anything else, so environment-based credentials -# (OPENAI_API_KEY for local mode, PAGEINDEX_API_KEY that callers read via -# os.environ for cloud mode) are populated by PageIndex itself — not left to -# litellm's incidental dotenv loading, which would vanish if litellm changes or -# its import is ever made lazy. +# Load .env first so env-based credentials (OPENAI_API_KEY, PAGEINDEX_API_KEY) are set. from dotenv import load_dotenv as _load_dotenv _load_dotenv() -# Backward compatibility: honor CHATGPT_API_KEY as an alias for OPENAI_API_KEY -# (kept from the pre-SDK pageindex.utils). Runs after load_dotenv so a value in -# .env is picked up too; only fills OPENAI_API_KEY when it isn't already set. +# Backward compatibility: honor CHATGPT_API_KEY as an alias for OPENAI_API_KEY. import os as _os if not _os.getenv("OPENAI_API_KEY") and _os.getenv("CHATGPT_API_KEY"): _os.environ["OPENAI_API_KEY"] = _os.getenv("CHATGPT_API_KEY") -# Upstream exports (backward compatibility). Import from the canonical -# pageindex.index.* modules directly so `import pageindex` does NOT trip the -# top-level deprecation shims (pageindex.page_index / .page_index_md / .utils). +# Upstream exports (backward compatibility); import from the canonical index.* +# modules so plain `import pageindex` doesn't trip the deprecation shims. from .index.page_index import * # noqa: E402 from .index.page_index_md import md_to_tree from .retrieve import get_document, get_document_structure, get_page_content diff --git a/pageindex/index/pipeline.py b/pageindex/index/pipeline.py index 56e8964..13dbbc4 100644 --- a/pageindex/index/pipeline.py +++ b/pageindex/index/pipeline.py @@ -51,19 +51,12 @@ def _run_async(coro): import asyncio import concurrent.futures import contextvars - # Only the detection is guarded — NOT the run. If the coroutine's own work - # raises RuntimeError, letting it fall into `except RuntimeError` here would - # misfire the "no running loop" branch and mask the real error behind a - # bogus "asyncio.run() cannot be called from a running event loop". try: asyncio.get_running_loop() except RuntimeError: - # No running loop -- drive the coroutine directly. return asyncio.run(coro) - # Already inside an event loop -- run in a separate thread so we don't nest - # asyncio.run. Copy the current context so ContextVar-based settings (e.g. - # the max_concurrency_scope override set by build_index) propagate into the - # worker thread; .result() re-raises the worker's real exception unchanged. + # In a running loop: run in a worker thread, with the current context copied + # so ContextVar-based settings propagate. ctx = contextvars.copy_context() with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: return pool.submit(ctx.run, asyncio.run, coro).result() @@ -106,8 +99,7 @@ def build_index(parsed: ParsedDocument, model: str = None, opt=None) -> dict: if opt.if_add_node_summary: if strategy == "level_based": - # Markdown keeps the legacy summarizer: nodes under 200 tokens - # reuse their text instead of spending an LLM call. + # Markdown: legacy summarizer — nodes under 200 tokens reuse their text. from .page_index_md import generate_summaries_for_structure_md _run_async(generate_summaries_for_structure_md( structure, summary_token_threshold=200, model=opt.model)) @@ -125,15 +117,7 @@ def build_index(parsed: ParsedDocument, model: str = None, opt=None) -> dict: clean_structure, model=opt.model ) - # 'text' is populated for level_based (Markdown, always) or for - # content_based when if_add_node_text/if_add_node_summary requested it. - # Strip it LAST, for BOTH strategies, unless explicitly requested — - # otherwise a default index leaks each node's full text into - # get_document_structure / storage, inconsistent with - # if_add_node_text=False, the README, and the legacy md_to_tree. Skip - # the walk entirely when text was never added in the first place - # (content_based with if_add_node_text=if_add_node_summary=False) — - # there's nothing to strip. + # Strip 'text' last unless explicitly requested; skip when it was never added. text_present = strategy == "level_based" or opt.if_add_node_text or opt.if_add_node_summary if text_present and not opt.if_add_node_text: remove_structure_text(structure) diff --git a/pageindex/retrieve.py b/pageindex/retrieve.py index 72292eb..fb92469 100644 --- a/pageindex/retrieve.py +++ b/pageindex/retrieve.py @@ -15,9 +15,7 @@ except ImportError: # ── Helpers ────────────────────────────────────────────────────────────────── def _parse_pages(pages: str) -> list[int]: - """Parse a pages string like '5-7', '3,8', or '12' into a sorted list of ints. - Delegates to the canonical implementation so the two never drift again — - this one used to lack the p>=1 filter and the 1000-page DoS cap.""" + """Parse a pages string like '5-7', '3,8', or '12' into a sorted list of ints.""" return parse_pages(pages) @@ -31,8 +29,7 @@ def _count_pages(doc_info: dict) -> int: def _get_pdf_page_content(doc_info: dict, page_nums: list[int]) -> list[dict]: - """Extract text for specific PDF pages (1-indexed). Prefer cached pages, - else delegate the file-read fallback to the canonical implementation.""" + """Extract text for specific PDF pages (1-indexed). Prefer cached pages, fallback to PDF.""" cached_pages = doc_info.get('pages') if cached_pages: page_map = {p['page']: p['content'] for p in cached_pages} @@ -44,8 +41,7 @@ def _get_pdf_page_content(doc_info: dict, page_nums: list[int]) -> list[dict]: def _get_md_page_content(doc_info: dict, page_nums: list[int]) -> list[dict]: - """For Markdown documents, 'pages' are line numbers. Delegates to the - canonical implementation so the two never drift again.""" + """For Markdown documents, 'pages' are line numbers.""" return get_md_page_content(doc_info.get('structure', []), page_nums) diff --git a/run_pageindex.py b/run_pageindex.py index 38b6a29..1f72c5b 100644 --- a/run_pageindex.py +++ b/run_pageindex.py @@ -2,10 +2,6 @@ import argparse import os import json from pageindex.index.page_index import * -# Reuse the canonical yes/no coercion (as _cli_bool) instead of a second copy — -# a bare ``--flag`` (no value) resolves to True via argparse's ``const``; an -# explicit value keeps the legacy yes/no style working, so ``--flag no`` turns -# it off. argparse only ever passes a str here (const/default bypass type=). from pageindex.index.page_index_md import md_to_tree from pageindex.index.utils import _coerce_bool as _cli_bool from pageindex.config import IndexConfig @@ -17,7 +13,7 @@ if __name__ == "__main__": parser.add_argument('--pdf_path', type=str, help='Path to the PDF file') parser.add_argument('--md_path', type=str, help='Path to the Markdown file') - parser.add_argument('--model', type=str, default=None, help='Model to use') + parser.add_argument('--model', type=str, default=None, help='Model to use (overrides config.yaml)') parser.add_argument('--toc-check-pages', type=int, default=None, help='Number of pages to check for table of contents (PDF only)')