mirror of
https://github.com/VectifyAI/PageIndex.git
synced 2026-07-18 21:21:05 +02:00
fix: dedup race, cwd-relative image paths, unencoded legacy URLs
- SQLite dedup race: add UNIQUE(collection_name, file_hash) and switch save_document to a plain INSERT. add_document now catches the IntegrityError from a concurrent add of the same content, cleans up its managed files, and returns the winning doc_id — instead of two doc_ids for one file (each having paid for its own LLM indexing). - PDF image paths: store the absolute path to each extracted image instead of a path relative to the indexing process's cwd. The  references broke as soon as a query ran from a different directory. - LegacyCloudAPI: URL-encode doc_id / retrieval_id path segments (added _enc()), matching CloudBackend. An id containing '/', '?', '#' or a space previously hit the wrong endpoint or produced a malformed URL. Adds regression tests: UNIQUE enforcement, the add-race resolving to the winner, absolute image paths, and encoded legacy URLs. Claude-Session: https://claude.ai/code/session_01Kx5DgKbhK1N8autqXH8SmS
This commit is contained in:
parent
fe36e25773
commit
b3616f76a2
8 changed files with 123 additions and 15 deletions
|
|
@ -47,7 +47,8 @@ class SQLiteStorage:
|
|||
doc_type TEXT NOT NULL,
|
||||
structure JSON,
|
||||
pages JSON,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE(collection_name, file_hash)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_docs_collection ON documents(collection_name);
|
||||
CREATE INDEX IF NOT EXISTS idx_docs_hash ON documents(collection_name, file_hash);
|
||||
|
|
@ -76,8 +77,11 @@ class SQLiteStorage:
|
|||
|
||||
def save_document(self, collection: str, doc_id: str, doc: dict) -> None:
|
||||
conn = self._get_conn()
|
||||
# Plain INSERT (doc_id is a fresh uuid, never pre-existing). A duplicate
|
||||
# (collection_name, file_hash) raises sqlite3.IntegrityError, which the
|
||||
# caller uses to resolve a concurrent add-of-same-file race.
|
||||
conn.execute(
|
||||
"""INSERT OR REPLACE INTO documents
|
||||
"""INSERT INTO documents
|
||||
(doc_id, collection_name, doc_name, doc_description, file_path, file_hash, doc_type, structure, pages)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
|
||||
(doc_id, collection, doc.get("doc_name"), doc.get("doc_description"),
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue