mirror of
https://github.com/VectifyAI/PageIndex.git
synced 2026-07-24 21:41:04 +02:00
feat: make pymupdf an optional [images] extra
The default install stays permissively licensed (PyMuPDF is AGPL). Text extraction is PyPDF2 everywhere; image extraction requires pip install "pageindex[images]" and degrades to text-only with a one-time warning when PyMuPDF is absent.
This commit is contained in:
parent
2688d15b16
commit
472d871ac3
4 changed files with 37 additions and 11 deletions
|
|
@ -153,6 +153,8 @@ A unified `PageIndexClient` powers both local self-hosted and cloud-managed mode
|
|||
pip install pageindex
|
||||
```
|
||||
|
||||
Local-mode image extraction uses PyMuPDF (AGPL-licensed) and is an optional extra: `pip install "pageindex[images]"`. Without it, local indexing is text-only.
|
||||
|
||||
### Quick start
|
||||
|
||||
```python
|
||||
|
|
|
|||
|
|
@ -9,7 +9,6 @@ import re
|
|||
import asyncio
|
||||
import threading
|
||||
import PyPDF2
|
||||
import pymupdf
|
||||
import yaml
|
||||
from datetime import datetime
|
||||
from io import BytesIO
|
||||
|
|
@ -817,6 +816,7 @@ def get_page_tokens(pdf_path, model=None, pdf_parser="PyPDF2"):
|
|||
page_list.append((page_text, token_length))
|
||||
return page_list
|
||||
elif pdf_parser == "PyMuPDF":
|
||||
import pymupdf # optional dependency: pip install pageindex[images]
|
||||
if isinstance(pdf_path, BytesIO):
|
||||
pdf_stream = pdf_path
|
||||
doc = pymupdf.open(stream=pdf_stream, filetype="pdf")
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import logging
|
||||
import PyPDF2
|
||||
import pymupdf
|
||||
from pathlib import Path
|
||||
from .protocol import ContentNode, ParsedDocument
|
||||
from ..tokens import count_tokens
|
||||
|
|
@ -7,6 +7,8 @@ from ..tokens import count_tokens
|
|||
# Minimum image dimension to keep (skip icons/artifacts)
|
||||
_MIN_IMAGE_SIZE = 32
|
||||
|
||||
_warned_no_pymupdf = False
|
||||
|
||||
|
||||
class PdfParser:
|
||||
def supported_extensions(self) -> list[str]:
|
||||
|
|
@ -17,16 +19,10 @@ class PdfParser:
|
|||
model = kwargs.get("model")
|
||||
images_dir = kwargs.get("images_dir")
|
||||
|
||||
# Images are extracted with PyMuPDF (PyPDF2 cannot); text stays with
|
||||
# Images are extracted with PyMuPDF (optional extra); text stays with
|
||||
# PyPDF2 below so the extracted text — and therefore the tree — matches
|
||||
# the CLI / pre-SDK default.
|
||||
page_images: dict[int, list[dict]] = {}
|
||||
if images_dir:
|
||||
with pymupdf.open(str(path)) as doc:
|
||||
for i, page in enumerate(doc):
|
||||
images = self._extract_page_images(page, i + 1, images_dir)
|
||||
if images:
|
||||
page_images[i + 1] = images
|
||||
page_images = self._extract_images(path, images_dir) if images_dir else {}
|
||||
|
||||
reader = PyPDF2.PdfReader(str(path))
|
||||
nodes = []
|
||||
|
|
@ -49,9 +45,33 @@ class PdfParser:
|
|||
return ParsedDocument(doc_name=path.stem, nodes=nodes,
|
||||
metadata={"page_count": len(reader.pages)})
|
||||
|
||||
@staticmethod
|
||||
def _extract_images(path: Path, images_dir: str) -> dict[int, list[dict]]:
|
||||
"""Extract images per page. Requires the optional PyMuPDF dependency;
|
||||
without it, indexing proceeds text-only."""
|
||||
global _warned_no_pymupdf
|
||||
try:
|
||||
import pymupdf
|
||||
except ImportError:
|
||||
if not _warned_no_pymupdf:
|
||||
logging.getLogger(__name__).warning(
|
||||
"PyMuPDF is not installed; skipping image extraction. "
|
||||
'Install with: pip install "pageindex[images]"')
|
||||
_warned_no_pymupdf = True
|
||||
return {}
|
||||
|
||||
page_images: dict[int, list[dict]] = {}
|
||||
with pymupdf.open(str(path)) as doc:
|
||||
for i, page in enumerate(doc):
|
||||
images = PdfParser._extract_page_images(page, i + 1, images_dir)
|
||||
if images:
|
||||
page_images[i + 1] = images
|
||||
return page_images
|
||||
|
||||
@staticmethod
|
||||
def _extract_page_images(page, page_num: int, images_dir: str) -> list[dict]:
|
||||
"""Save a page's images to disk and return their metadata."""
|
||||
import pymupdf
|
||||
images_path = Path(images_dir)
|
||||
images_path.mkdir(parents=True, exist_ok=True)
|
||||
# Store an absolute path so the  reference resolves
|
||||
|
|
|
|||
|
|
@ -22,7 +22,8 @@ packages = [{include = "pageindex"}]
|
|||
[tool.poetry.dependencies]
|
||||
python = ">=3.10"
|
||||
litellm = ">=1.83.0"
|
||||
pymupdf = ">=1.26.0"
|
||||
# AGPL-licensed; optional so the default install stays permissively licensed.
|
||||
pymupdf = {version = ">=1.26.0", optional = true}
|
||||
PyPDF2 = ">=3.0.0"
|
||||
python-dotenv = ">=1.0.0"
|
||||
pyyaml = ">=6.0"
|
||||
|
|
@ -33,6 +34,9 @@ httpx = {extras = ["socks"], version = ">=0.28.1"}
|
|||
typing-extensions = ">=4.9.0"
|
||||
pydantic = ">=2.5.0,<3.0.0"
|
||||
|
||||
[tool.poetry.extras]
|
||||
images = ["pymupdf"]
|
||||
|
||||
[tool.poetry.group.dev.dependencies]
|
||||
pytest = ">=7.0"
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue