mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-07-22 03:31:02 +02:00
1b61ad1581
- Document schema (knowledge/document.py): Added document_id field for streaming retrieval - Librarian operations: - add-child-document for extracted PDF pages - list-children to get child documents - stream-document for chunked content retrieval - Cascade delete removes children when parent is deleted - list-documents filters children by default - PDF decoder (decoding/pdf/pdf_decoder.py): Updated to stream large documents from librarian API to temp file - Librarian service (librarian/service.py): Sends document_id instead of content for large PDFs (>2MB) - Deprecated tools (load_pdf.py, load_text.py): Added deprecation warnings directing users to tg-add-library-document + tg-start-library-processing
35 lines
915 B
Python
35 lines
915 B
Python
from dataclasses import dataclass
|
|
|
|
from ..core.metadata import Metadata
|
|
from ..core.topic import topic
|
|
|
|
############################################################################
|
|
|
|
# PDF docs etc.
|
|
@dataclass
|
|
class Document:
|
|
metadata: Metadata | None = None
|
|
data: bytes = b""
|
|
# For large document streaming: if document_id is set, the receiver should
|
|
# fetch content from librarian instead of using inline data
|
|
document_id: str = ""
|
|
|
|
############################################################################
|
|
|
|
# Text documents / text from PDF
|
|
|
|
@dataclass
|
|
class TextDocument:
|
|
metadata: Metadata | None = None
|
|
text: bytes = b""
|
|
|
|
############################################################################
|
|
|
|
# Chunks of text
|
|
|
|
@dataclass
|
|
class Chunk:
|
|
metadata: Metadata | None = None
|
|
chunk: bytes = b""
|
|
|
|
############################################################################
|