diff --git a/pageindex/page_index.py b/pageindex/page_index.py index aed38cf..ff74770 100644 --- a/pageindex/page_index.py +++ b/pageindex/page_index.py @@ -47,6 +47,19 @@ def _secure_doc_text(text: str) -> str: """Sanitize + delimiter-frame a PDF text block before LLM injection.""" return _wrap_doc_text(_sanitize_doc_text(text)) +_PHYSICAL_INDEX_MARKER_RE = re.compile(r"^$") + +def _parse_physical_index(raw): + if raw is None: + marker_match = _PHYSICAL_INDEX_MARKER_RE.match(str(raw).strip()) + if marker_match: + return int(marker_match.group(1)) + try: + return int(raw) + except (TypeError, ValueError) + return None + + def _validate_physical_indices(toc: list, total_pages: int, start_index: int = 1) -> list: """Nullify any physical_index the LLM produced that falls outside the real page range.""" max_idx = start_index + total_pages - 1 @@ -54,13 +67,11 @@ def _validate_physical_indices(toc: list, total_pages: int, start_index: int = 1 raw = entry.get("physical_index") if raw is None: continue - try: - val = int(raw) - except (TypeError, ValueError): - entry["physical_index"] = None - continue - if not (start_index <= val <= max_idx): + val = _parse_physical_index(raw) + if val is None or not (start_index <= val <= max_idx): entry["physical_index"] = None + else: + entry["physical_index"] = val return toc ################### check title in page ######################################################### @@ -216,8 +227,8 @@ def check_if_toc_transformation_is_complete(content, toc, model=None): prompt = ( prompt - + '/n Raw Table of contents:\n' + _secure_doc_text(content) - + '/n Cleaned Table of contents:\n' + _secure_doc_text(str(toc)) + + '\n Raw Table of contents:\n' + _secure_doc_text(content) + + '\n Cleaned Table of contents:\n' + _secure_doc_text(str(toc)) ) response = llm_completion(model=model, prompt=prompt) json_content = extract_json(response)