mirror of
https://github.com/VectifyAI/PageIndex.git
synced 2026-07-15 21:11:05 +02:00
fix: address PR #272 review findings (directly-fixable items)
Verified against current dev; the compat/behavior decisions (#7 api_key semantics, #10 CLI flags, #11 doc-description default) are deferred. Crashes: - page_index(): snapshot args before importing IndexConfig — locals() was capturing the imported class and IndexConfig(extra='forbid') made every call raise ValidationError. - process_none_page_numbers: pop('page', None) instead of del (a TOC item without 'page' raised KeyError mid-pipeline). - pipeline._run_async: guard only the loop detection, not the run, so a real RuntimeError from the coroutine isn't masked as "asyncio.run() cannot be called from a running event loop". Silent-wrong / robustness: - LocalBackend.get_document_structure and the agent get_document / get_document_structure tools now surface a missing doc (raise / error-JSON) instead of returning empty, matching get_page_content and the cloud backend. - cloud delete_collection drops the cached folder_id. - cloud query raises on an empty collection instead of POSTing doc_id:[]. - LocalClient skips the API-key check for keyless providers (ollama, lm_studio, …) so keyless LiteLLM models aren't rejected at construction. Compat / cleanup: - md_to_tree coerces legacy 'yes'/'no' string flags (a bare 'no' was truthy). - FileTypeError also subclasses ValueError (0.2.x raised ValueError). - _validate_llm_provider no longer mutates global litellm.model_cost_map_url. - __all__ re-includes legacy exports (page_index, md_to_tree, get_*). - Rewrite examples/agentic_vectorless_rag_demo.py to the Collection API and use the in-repo attention.pdf (the old workspace=/client.index/client.documents API no longer exists). Adds tests/test_review_fixes.py (10 regressions). Full suite: 189 passed. Claude-Session: https://claude.ai/code/session_01Kx5DgKbhK1N8autqXH8SmS
This commit is contained in:
mountain
parent
703017e581
commit
cf7f5ce9bf
10 changed files with 250 additions and 56 deletions
|
|
@ -61,4 +61,11 @@ __all__ = [
|
|||
"IndexingError",
|
||||
"CloudAPIError",
|
||||
"FileTypeError",
|
||||
# Legacy top-level exports (pre-SDK API), kept so `from pageindex import *`
|
||||
# still binds them.
|
||||
"page_index",
|
||||
"md_to_tree",
|
||||
"get_document",
|
||||
"get_document_structure",
|
||||
"get_page_content",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -162,6 +162,9 @@ class CloudBackend:
|
|||
folder_id = self._get_folder_id(name)
|
||||
if folder_id:
|
||||
self._request("DELETE", f"/folder/{self._enc(folder_id)}/")
|
||||
# Drop the cached id so a later same-name op re-resolves instead of
|
||||
# reusing the now-deleted folder_id.
|
||||
self._folder_id_cache.pop(name, None)
|
||||
|
||||
# ── Document management ───────────────────────────────────────────────
|
||||
|
||||
|
|
@ -307,6 +310,8 @@ class CloudBackend:
|
|||
"doc_ids cannot be empty; pass None to query the whole collection"
|
||||
)
|
||||
doc_id = doc_ids if doc_ids else self._get_all_doc_ids(collection)
|
||||
if not doc_id:
|
||||
raise ValueError("collection has no documents to query")
|
||||
# A non-streaming completion returns nothing until generation
|
||||
# finishes, so it needs far more than the default 30s. retries=1:
|
||||
# retrying this non-idempotent call would redo the full server-side
|
||||
|
|
@ -340,6 +345,8 @@ class CloudBackend:
|
|||
"doc_ids cannot be empty; pass None to query the whole collection"
|
||||
)
|
||||
doc_id = doc_ids if doc_ids else self._get_all_doc_ids(collection)
|
||||
if not doc_id:
|
||||
raise ValueError("collection has no documents to query")
|
||||
headers = self._headers
|
||||
# Queue carries QueryEvent, an Exception to re-raise, or None (end).
|
||||
queue: asyncio.Queue[QueryEvent | Exception | None] = asyncio.Queue()
|
||||
|
|
|
|||
|
|
@ -190,6 +190,10 @@ class LocalBackend:
|
|||
LocalBackend._fill_node_text(node["nodes"], page_map)
|
||||
|
||||
def get_document_structure(self, collection: str, doc_id: str) -> list:
|
||||
# Parity with get_document / the cloud backend: a missing doc must raise,
|
||||
# not masquerade as an empty structure.
|
||||
if not self._storage.get_document(collection, doc_id):
|
||||
raise DocumentNotFoundError(f"Document {doc_id} not found")
|
||||
return self._storage.get_document_structure(collection, doc_id)
|
||||
|
||||
def get_page_content(self, collection: str, doc_id: str, pages: str) -> list:
|
||||
|
|
@ -255,7 +259,10 @@ class LocalBackend:
|
|||
rejection = _reject(doc_id)
|
||||
if rejection:
|
||||
return rejection
|
||||
return json.dumps(storage.get_document(col_name, doc_id))
|
||||
doc = storage.get_document(col_name, doc_id)
|
||||
if not doc:
|
||||
return json.dumps({"error": f"doc_id '{doc_id}' not found."})
|
||||
return json.dumps(doc)
|
||||
|
||||
@function_tool
|
||||
def get_document_structure(doc_id: str) -> str:
|
||||
|
|
@ -263,6 +270,8 @@ class LocalBackend:
|
|||
rejection = _reject(doc_id)
|
||||
if rejection:
|
||||
return rejection
|
||||
if not storage.get_document(col_name, doc_id):
|
||||
return json.dumps({"error": f"doc_id '{doc_id}' not found."})
|
||||
structure = storage.get_document_structure(col_name, doc_id)
|
||||
return json.dumps(remove_fields(structure, fields=["text"]), ensure_ascii=False)
|
||||
|
||||
|
|
@ -272,7 +281,10 @@ class LocalBackend:
|
|||
rejection = _reject(doc_id)
|
||||
if rejection:
|
||||
return rejection
|
||||
result = backend.get_page_content(col_name, doc_id, pages)
|
||||
try:
|
||||
result = backend.get_page_content(col_name, doc_id, pages)
|
||||
except DocumentNotFoundError:
|
||||
return json.dumps({"error": f"doc_id '{doc_id}' not found."})
|
||||
return json.dumps(result, ensure_ascii=False)
|
||||
|
||||
tools = [get_document, get_document_structure, get_page_content]
|
||||
|
|
|
|||
|
|
@ -113,14 +113,20 @@ class PageIndexClient:
|
|||
|
||||
@staticmethod
|
||||
def _validate_llm_provider(model: str) -> None:
|
||||
"""Validate model and check API key via litellm. Warns if key seems missing."""
|
||||
"""Validate the model string and require an API key for providers that
|
||||
need one. Local / keyless providers (ollama, lm_studio, …) are skipped so
|
||||
a keyless LiteLLM model isn't rejected at construction time."""
|
||||
try:
|
||||
import litellm
|
||||
litellm.model_cost_map_url = ""
|
||||
_, provider, _, _ = litellm.get_llm_provider(model=model)
|
||||
except Exception:
|
||||
return
|
||||
|
||||
# LiteLLM providers that run locally and need no API key.
|
||||
keyless = {"ollama", "ollama_chat", "lm_studio", "hosted_vllm", "vllm"}
|
||||
if provider in keyless:
|
||||
return
|
||||
|
||||
key = litellm.get_api_key(llm_provider=provider, dynamic_api_key=None)
|
||||
if not key:
|
||||
import os
|
||||
|
|
|
|||
|
|
@ -39,6 +39,10 @@ class CloudAPIError(PageIndexAPIError):
|
|||
self.status_code = status_code
|
||||
|
||||
|
||||
class FileTypeError(PageIndexError):
|
||||
"""Unsupported file type."""
|
||||
class FileTypeError(PageIndexError, ValueError):
|
||||
"""Unsupported file type.
|
||||
|
||||
Also subclasses ValueError so pre-SDK ``except ValueError`` around indexing
|
||||
(0.2.x raised ValueError for an unsupported file format) still catches it.
|
||||
"""
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -679,11 +679,11 @@ def process_none_page_numbers(toc_items, page_list, start_index=1, model=None):
|
|||
continue
|
||||
|
||||
item_copy = copy.deepcopy(item)
|
||||
del item_copy['page']
|
||||
item_copy.pop('page', None)
|
||||
result = add_page_number_to_toc(page_contents, item_copy, model)
|
||||
if isinstance(result[0]['physical_index'], str) and result[0]['physical_index'].startswith('<physical_index'):
|
||||
item['physical_index'] = int(result[0]['physical_index'].split('_')[-1].rstrip('>').strip())
|
||||
del item['page']
|
||||
item.pop('page', None)
|
||||
|
||||
return toc_items
|
||||
|
||||
|
|
@ -1113,11 +1113,14 @@ def page_index_main(doc, opt=None):
|
|||
def page_index(doc, model=None, toc_check_page_num=None, max_page_num_each_node=None, max_token_num_each_node=None,
|
||||
if_add_node_id=None, if_add_node_summary=None, if_add_doc_description=None, if_add_node_text=None):
|
||||
|
||||
from ..config import IndexConfig
|
||||
# Snapshot the call args BEFORE importing IndexConfig — otherwise the
|
||||
# imported class would be captured by locals() and rejected by
|
||||
# IndexConfig(extra="forbid").
|
||||
user_opt = {
|
||||
arg: value for arg, value in locals().items()
|
||||
if arg != "doc" and value is not None
|
||||
}
|
||||
from ..config import IndexConfig
|
||||
opt = IndexConfig(**user_opt)
|
||||
return page_index_main(doc, opt)
|
||||
|
||||
|
|
|
|||
|
|
@ -237,7 +237,21 @@ def clean_tree_for_output(tree_nodes):
|
|||
return cleaned_nodes
|
||||
|
||||
|
||||
def _coerce_bool(value):
|
||||
"""Coerce a legacy 'yes'/'no' string flag to bool (a bare 'no' is truthy)."""
|
||||
if isinstance(value, str):
|
||||
return value.strip().lower() in ("yes", "true", "1", "y", "on")
|
||||
return bool(value)
|
||||
|
||||
|
||||
async def md_to_tree(md_path, if_thinning=False, min_token_threshold=None, if_add_node_summary=False, summary_token_threshold=None, model=None, if_add_doc_description=False, if_add_node_text=False, if_add_node_id=True):
|
||||
# Accept legacy 'yes'/'no' string flags — a bare 'no' would otherwise be
|
||||
# truthy and wrongly enable the option.
|
||||
if_thinning = _coerce_bool(if_thinning)
|
||||
if_add_node_summary = _coerce_bool(if_add_node_summary)
|
||||
if_add_doc_description = _coerce_bool(if_add_doc_description)
|
||||
if_add_node_text = _coerce_bool(if_add_node_text)
|
||||
if_add_node_id = _coerce_bool(if_add_node_id)
|
||||
with open(md_path, 'r', encoding='utf-8') as f:
|
||||
markdown_content = f.read()
|
||||
line_count = markdown_content.count('\n') + 1
|
||||
|
|
|
|||
|
|
@ -44,17 +44,22 @@ def _run_async(coro):
|
|||
import asyncio
|
||||
import concurrent.futures
|
||||
import contextvars
|
||||
# Only the detection is guarded — NOT the run. If the coroutine's own work
|
||||
# raises RuntimeError, letting it fall into `except RuntimeError` here would
|
||||
# misfire the "no running loop" branch and mask the real error behind a
|
||||
# bogus "asyncio.run() cannot be called from a running event loop".
|
||||
try:
|
||||
asyncio.get_running_loop()
|
||||
# Already inside an event loop -- run in a separate thread. Copy the
|
||||
# current context so ContextVar-based settings (e.g. the
|
||||
# max_concurrency_scope override set by build_index) propagate into the
|
||||
# worker thread instead of silently falling back to the process default.
|
||||
ctx = contextvars.copy_context()
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
|
||||
return pool.submit(ctx.run, asyncio.run, coro).result()
|
||||
except RuntimeError:
|
||||
# No running loop -- drive the coroutine directly.
|
||||
return asyncio.run(coro)
|
||||
# Already inside an event loop -- run in a separate thread so we don't nest
|
||||
# asyncio.run. Copy the current context so ContextVar-based settings (e.g.
|
||||
# the max_concurrency_scope override set by build_index) propagate into the
|
||||
# worker thread; .result() re-raises the worker's real exception unchanged.
|
||||
ctx = contextvars.copy_context()
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
|
||||
return pool.submit(ctx.run, asyncio.run, coro).result()
|
||||
|
||||
|
||||
def build_index(parsed: ParsedDocument, model: str = None, opt=None) -> dict:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue