diff --git a/README.md b/README.md index adb7faf..38614ff 100644 --- a/README.md +++ b/README.md @@ -159,6 +159,7 @@ pip install pageindex from pageindex import PageIndexClient # Local mode — uses your LLM key (e.g. OPENAI_API_KEY in env). +# `model` drives indexing; agent QA uses `retrieve_model` (default: gpt-5.4). client = PageIndexClient(model="gpt-4o-2024-11-20") col = client.collection() @@ -238,13 +239,12 @@ You can customize the processing with additional optional arguments: --toc-check-pages Pages to check for table of contents (default: 20) --max-pages-per-node Max pages per node (default: 10) --max-tokens-per-node Max tokens per node (default: 20000) ---if-add-node-id Add node IDs (on by default; disable with: --if-add-node-id no) ---if-add-node-summary Add node summaries (on by default; disable with: --if-add-node-summary no) ---if-add-doc-description Add a document description (on by default; disable with: --if-add-doc-description no) ---if-add-node-text Add raw text to nodes (off by default; enable with: --if-add-node-text) +--if-add-node-id Add node ID (yes/no, default: yes) +--if-add-node-summary Add node summary (yes/no, default: yes) +--if-add-doc-description Add doc description (yes/no, default: no) +--if-add-node-text Add raw text to nodes (yes/no, default: no) ``` -These flags take no value by default (a bare `--if-add-node-id` turns it on); the -legacy `--if-add-node-id no` form still works for turning an option off. +A bare flag is shorthand for `yes` (e.g. `--if-add-node-id` turns the option on).
diff --git a/examples/agentic_vectorless_rag_demo.py b/examples/agentic_vectorless_rag_demo.py index c079e26..a1bbeaf 100644 --- a/examples/agentic_vectorless_rag_demo.py +++ b/examples/agentic_vectorless_rag_demo.py @@ -40,10 +40,10 @@ from openai.types.responses import ResponseTextDeltaEvent, ResponseReasoningSumm from pageindex import LocalClient -PDF_URL = "https://arxiv.org/pdf/1706.03762.pdf" +PDF_URL = "https://arxiv.org/pdf/2603.15031" _EXAMPLES_DIR = Path(__file__).parent -PDF_PATH = _EXAMPLES_DIR / "documents" / "attention.pdf" +PDF_PATH = _EXAMPLES_DIR / "documents" / "attention-residuals.pdf" WORKSPACE = _EXAMPLES_DIR / "workspace" MODEL = "gpt-4o-2024-11-20" # any LiteLLM-supported model @@ -199,6 +199,6 @@ if __name__ == "__main__": print("\n" + "=" * 60) print("Step 3: Agent Query (auto tool-use)") print("=" * 60) - question = "Explain the Transformer's self-attention in simple language." + question = "Explain Attention Residuals in simple language." print(f"\nQuestion: '{question}'") - query_agent(col, doc_id, question, MODEL, verbose=True) + query_agent(col, doc_id, question, client.retrieve_model, verbose=True) diff --git a/examples/cloud_demo.py b/examples/cloud_demo.py index 6455d7d..15aecd1 100644 --- a/examples/cloud_demo.py +++ b/examples/cloud_demo.py @@ -19,8 +19,8 @@ import requests from pageindex import CloudClient _EXAMPLES_DIR = Path(__file__).parent -PDF_URL = "https://arxiv.org/pdf/1706.03762.pdf" -PDF_PATH = _EXAMPLES_DIR / "documents" / "attention.pdf" +PDF_URL = "https://arxiv.org/pdf/2603.15031" +PDF_PATH = _EXAMPLES_DIR / "documents" / "attention-residuals.pdf" # Download PDF if needed if not PDF_PATH.exists(): diff --git a/examples/local_demo.py b/examples/local_demo.py index fcbc9bb..8f6d659 100644 --- a/examples/local_demo.py +++ b/examples/local_demo.py @@ -19,8 +19,8 @@ import requests from pageindex import LocalClient _EXAMPLES_DIR = Path(__file__).parent -PDF_URL = "https://arxiv.org/pdf/1706.03762.pdf" -PDF_PATH = _EXAMPLES_DIR / "documents" / "attention.pdf" +PDF_URL = "https://arxiv.org/pdf/2603.15031" +PDF_PATH = _EXAMPLES_DIR / "documents" / "attention-residuals.pdf" WORKSPACE = _EXAMPLES_DIR / "workspace" MODEL = "gpt-4o-2024-11-20" # any LiteLLM-supported model @@ -44,7 +44,7 @@ print(f"Indexed: {doc_id}\n") # Streaming query stream = col.query( - "What is the main architecture proposed in this paper and how does self-attention work?", + "Explain Attention Residuals in simple language.", stream=True, ) diff --git a/pageindex/client.py b/pageindex/client.py index a51df63..e7f16d8 100644 --- a/pageindex/client.py +++ b/pageindex/client.py @@ -101,6 +101,9 @@ class PageIndexClient: self._validate_llm_provider(opt.model) + self.model = opt.model + self.retrieve_model = _normalize_retrieve_model(opt.retrieve_model or self.model) + storage_path = Path(storage_path or ".pageindex").resolve() storage_path.mkdir(parents=True, exist_ok=True) @@ -111,7 +114,7 @@ class PageIndexClient: storage=storage_engine, files_dir=str(storage_path / "files"), model=opt.model, - retrieve_model=_normalize_retrieve_model(opt.retrieve_model or opt.model), + retrieve_model=self.retrieve_model, index_config=opt, ) diff --git a/pageindex/index/pipeline.py b/pageindex/index/pipeline.py index adcbfeb..56e8964 100644 --- a/pageindex/index/pipeline.py +++ b/pageindex/index/pipeline.py @@ -105,7 +105,14 @@ def build_index(parsed: ParsedDocument, model: str = None, opt=None) -> dict: add_node_text(structure, page_list) if opt.if_add_node_summary: - _run_async(generate_summaries_for_structure(structure, model=opt.model)) + if strategy == "level_based": + # Markdown keeps the legacy summarizer: nodes under 200 tokens + # reuse their text instead of spending an LLM call. + from .page_index_md import generate_summaries_for_structure_md + _run_async(generate_summaries_for_structure_md( + structure, summary_token_threshold=200, model=opt.model)) + else: + _run_async(generate_summaries_for_structure(structure, model=opt.model)) result = { "doc_name": parsed.doc_name, diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index e2412d0..da610e4 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -165,11 +165,11 @@ def test_build_index_scopes_llm_params_to_the_call(monkeypatch): set_llm_params(temperature=0) seen = {} - async def fake_generate_summaries(structure, model=None): + async def fake_generate_summaries(structure, summary_token_threshold=200, model=None): seen["llm_params"] = get_llm_params() monkeypatch.setattr( - "pageindex.index.utils.generate_summaries_for_structure", + "pageindex.index.page_index_md.generate_summaries_for_structure_md", fake_generate_summaries, )