mirror of
https://github.com/VectifyAI/PageIndex.git
synced 2026-07-18 21:21:05 +02:00
fix: dedup race, cwd-relative image paths, unencoded legacy URLs
- SQLite dedup race: add UNIQUE(collection_name, file_hash) and switch save_document to a plain INSERT. add_document now catches the IntegrityError from a concurrent add of the same content, cleans up its managed files, and returns the winning doc_id — instead of two doc_ids for one file (each having paid for its own LLM indexing). - PDF image paths: store the absolute path to each extracted image instead of a path relative to the indexing process's cwd. The  references broke as soon as a query ran from a different directory. - LegacyCloudAPI: URL-encode doc_id / retrieval_id path segments (added _enc()), matching CloudBackend. An id containing '/', '?', '#' or a space previously hit the wrong endpoint or produced a malformed URL. Adds regression tests: UNIQUE enforcement, the add-race resolving to the winner, absolute image paths, and encoded legacy URLs. Claude-Session: https://claude.ai/code/session_01Kx5DgKbhK1N8autqXH8SmS
This commit is contained in:
parent
fe36e25773
commit
b3616f76a2
8 changed files with 123 additions and 15 deletions
|
|
@ -1,6 +1,7 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import urllib.parse
|
||||
from typing import Any, Iterator
|
||||
|
||||
import requests
|
||||
|
|
@ -17,6 +18,11 @@ class LegacyCloudAPI:
|
|||
self.api_key = api_key
|
||||
self.base_url = base_url or self.BASE_URL
|
||||
|
||||
@staticmethod
|
||||
def _enc(value: str) -> str:
|
||||
"""URL-encode a path segment (ids may contain / ? # or spaces)."""
|
||||
return urllib.parse.quote(str(value), safe="")
|
||||
|
||||
def _headers(self) -> dict[str, str]:
|
||||
return {"api_key": self.api_key}
|
||||
|
||||
|
|
@ -71,7 +77,7 @@ class LegacyCloudAPI:
|
|||
|
||||
response = self._request(
|
||||
"GET",
|
||||
f"/doc/{doc_id}/?type=ocr&format={format}",
|
||||
f"/doc/{self._enc(doc_id)}/?type=ocr&format={format}",
|
||||
"Failed to get OCR result",
|
||||
)
|
||||
return response.json()
|
||||
|
|
@ -79,7 +85,7 @@ class LegacyCloudAPI:
|
|||
def get_tree(self, doc_id: str, node_summary: bool = False) -> dict[str, Any]:
|
||||
response = self._request(
|
||||
"GET",
|
||||
f"/doc/{doc_id}/?type=tree&summary={node_summary}",
|
||||
f"/doc/{self._enc(doc_id)}/?type=tree&summary={node_summary}",
|
||||
"Failed to get tree result",
|
||||
)
|
||||
return response.json()
|
||||
|
|
@ -112,7 +118,7 @@ class LegacyCloudAPI:
|
|||
def get_retrieval(self, retrieval_id: str) -> dict[str, Any]:
|
||||
response = self._request(
|
||||
"GET",
|
||||
f"/retrieval/{retrieval_id}/",
|
||||
f"/retrieval/{self._enc(retrieval_id)}/",
|
||||
"Failed to get retrieval result",
|
||||
)
|
||||
return response.json()
|
||||
|
|
@ -203,7 +209,7 @@ class LegacyCloudAPI:
|
|||
def get_document(self, doc_id: str) -> dict[str, Any]:
|
||||
response = self._request(
|
||||
"GET",
|
||||
f"/doc/{doc_id}/metadata/",
|
||||
f"/doc/{self._enc(doc_id)}/metadata/",
|
||||
"Failed to get document metadata",
|
||||
)
|
||||
return response.json()
|
||||
|
|
@ -211,7 +217,7 @@ class LegacyCloudAPI:
|
|||
def delete_document(self, doc_id: str) -> dict[str, Any]:
|
||||
response = self._request(
|
||||
"DELETE",
|
||||
f"/doc/{doc_id}/",
|
||||
f"/doc/{self._enc(doc_id)}/",
|
||||
"Failed to delete document",
|
||||
)
|
||||
return response.json()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue