mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-07-09 05:12:12 +02:00
4fb0b4d8e8
Remove Pulsar-specific concepts from application code so that the pub/sub backend is swappable via configuration. Rename translators: - to_pulsar/from_pulsar → decode/encode across all translator classes, dispatch handlers, and tests (55+ files) - from_response_with_completion → encode_with_completion - Remove pulsar.schema.Record from translator base class Queue naming (CLASS:TOPICSPACE:TOPIC): - Replace topic() helper with queue() using new format: flow:tg:name, request:tg:name, response:tg:name, state:tg:name - Queue class implies persistence/TTL (no QoS in names) - Update Pulsar backend map_topic() to parse new format - Librarian queues use flow class (persistent, for chunking) - Config push uses state class (persistent, last-value) - Remove 15 dead topic imports from schema files - Update init_trustgraph.py namespace: config → state Confine Pulsar to pulsar_backend.py: - Delete legacy PulsarClient class from pubsub.py - Move add_args to add_pubsub_args() with standalone flag for CLI tools (defaults to localhost) - PulsarBackendConsumer.receive() catches _pulsar.Timeout, raises standard TimeoutError - Remove Pulsar imports from: async_processor, flow_processor, log_level, all 11 client files, 4 storage writers, gateway service, gateway config receiver - Remove log_level/LoggerLevel from client API - Rewrite tg-monitor-prompts to use backend abstraction - Update tg-dump-queues to use add_pubsub_args Also: pubsub-abstraction.md tech spec covering problem statement, design goals, as-is requirements, candidate broker assessment, approach, and implementation order.
194 lines
6.1 KiB
Python
194 lines
6.1 KiB
Python
import base64
|
|
from typing import Dict, Any
|
|
from ...schema import Document, TextDocument, Chunk, DocumentEmbeddings, ChunkEmbeddings
|
|
from .base import SendTranslator
|
|
|
|
|
|
def _decode_text_payload(payload: str | bytes, charset: str) -> str:
|
|
"""
|
|
Decode text-load payloads.
|
|
|
|
Historical clients send base64-encoded text, but direct REST callers may
|
|
send raw UTF-8 text. Support both so Unicode text-load requests do not fail
|
|
at the gateway translation layer.
|
|
"""
|
|
if isinstance(payload, bytes):
|
|
if not payload.isascii():
|
|
return payload.decode(charset)
|
|
candidate = payload.decode("ascii")
|
|
else:
|
|
if not payload.isascii():
|
|
return payload
|
|
candidate = payload
|
|
|
|
try:
|
|
return base64.b64decode(candidate, validate=True).decode(charset)
|
|
except (ValueError, UnicodeDecodeError):
|
|
return candidate
|
|
|
|
|
|
class DocumentTranslator(SendTranslator):
|
|
"""Translator for Document schema objects (PDF docs etc.)"""
|
|
|
|
def decode(self, data: Dict[str, Any]) -> Document:
|
|
# Handle base64 content validation
|
|
doc = base64.b64decode(data["data"])
|
|
|
|
from ...schema import Metadata
|
|
return Document(
|
|
metadata=Metadata(
|
|
id=data.get("id"),
|
|
root=data.get("root", ""),
|
|
user=data.get("user", "trustgraph"),
|
|
collection=data.get("collection", "default"),
|
|
),
|
|
data=base64.b64encode(doc).decode("utf-8")
|
|
)
|
|
|
|
def encode(self, obj: Document) -> Dict[str, Any]:
|
|
result = {
|
|
"data": obj.data
|
|
}
|
|
|
|
if obj.metadata:
|
|
metadata_dict = {}
|
|
if obj.metadata.id:
|
|
metadata_dict["id"] = obj.metadata.id
|
|
if obj.metadata.root:
|
|
metadata_dict["root"] = obj.metadata.root
|
|
if obj.metadata.user:
|
|
metadata_dict["user"] = obj.metadata.user
|
|
if obj.metadata.collection:
|
|
metadata_dict["collection"] = obj.metadata.collection
|
|
|
|
result["metadata"] = metadata_dict
|
|
|
|
return result
|
|
|
|
|
|
class TextDocumentTranslator(SendTranslator):
|
|
"""Translator for TextDocument schema objects"""
|
|
|
|
def decode(self, data: Dict[str, Any]) -> TextDocument:
|
|
charset = data.get("charset", "utf-8")
|
|
|
|
text = _decode_text_payload(data["text"], charset)
|
|
|
|
from ...schema import Metadata
|
|
return TextDocument(
|
|
metadata=Metadata(
|
|
id=data.get("id"),
|
|
root=data.get("root", ""),
|
|
user=data.get("user", "trustgraph"),
|
|
collection=data.get("collection", "default"),
|
|
),
|
|
text=text.encode("utf-8")
|
|
)
|
|
|
|
def encode(self, obj: TextDocument) -> Dict[str, Any]:
|
|
result = {
|
|
"text": obj.text.decode("utf-8") if isinstance(obj.text, bytes) else obj.text
|
|
}
|
|
|
|
if obj.metadata:
|
|
metadata_dict = {}
|
|
if obj.metadata.id:
|
|
metadata_dict["id"] = obj.metadata.id
|
|
if obj.metadata.root:
|
|
metadata_dict["root"] = obj.metadata.root
|
|
if obj.metadata.user:
|
|
metadata_dict["user"] = obj.metadata.user
|
|
if obj.metadata.collection:
|
|
metadata_dict["collection"] = obj.metadata.collection
|
|
|
|
result["metadata"] = metadata_dict
|
|
|
|
return result
|
|
|
|
|
|
class ChunkTranslator(SendTranslator):
|
|
"""Translator for Chunk schema objects"""
|
|
|
|
def decode(self, data: Dict[str, Any]) -> Chunk:
|
|
from ...schema import Metadata
|
|
return Chunk(
|
|
metadata=Metadata(
|
|
id=data.get("id"),
|
|
root=data.get("root", ""),
|
|
user=data.get("user", "trustgraph"),
|
|
collection=data.get("collection", "default"),
|
|
),
|
|
chunk=data["chunk"].encode("utf-8") if isinstance(data["chunk"], str) else data["chunk"]
|
|
)
|
|
|
|
def encode(self, obj: Chunk) -> Dict[str, Any]:
|
|
result = {
|
|
"chunk": obj.chunk.decode("utf-8") if isinstance(obj.chunk, bytes) else obj.chunk
|
|
}
|
|
|
|
if obj.metadata:
|
|
metadata_dict = {}
|
|
if obj.metadata.id:
|
|
metadata_dict["id"] = obj.metadata.id
|
|
if obj.metadata.root:
|
|
metadata_dict["root"] = obj.metadata.root
|
|
if obj.metadata.user:
|
|
metadata_dict["user"] = obj.metadata.user
|
|
if obj.metadata.collection:
|
|
metadata_dict["collection"] = obj.metadata.collection
|
|
|
|
result["metadata"] = metadata_dict
|
|
|
|
return result
|
|
|
|
|
|
class DocumentEmbeddingsTranslator(SendTranslator):
|
|
"""Translator for DocumentEmbeddings schema objects"""
|
|
|
|
def decode(self, data: Dict[str, Any]) -> DocumentEmbeddings:
|
|
metadata = data.get("metadata", {})
|
|
|
|
chunks = [
|
|
ChunkEmbeddings(
|
|
chunk_id=chunk["chunk_id"],
|
|
vectors=chunk["vectors"]
|
|
)
|
|
for chunk in data.get("chunks", [])
|
|
]
|
|
|
|
from ...schema import Metadata
|
|
return DocumentEmbeddings(
|
|
metadata=Metadata(
|
|
id=metadata.get("id"),
|
|
root=metadata.get("root", ""),
|
|
user=metadata.get("user", "trustgraph"),
|
|
collection=metadata.get("collection", "default"),
|
|
),
|
|
chunks=chunks
|
|
)
|
|
|
|
def encode(self, obj: DocumentEmbeddings) -> Dict[str, Any]:
|
|
result = {
|
|
"chunks": [
|
|
{
|
|
"chunk_id": chunk.chunk_id,
|
|
"vector": chunk.vector
|
|
}
|
|
for chunk in obj.chunks
|
|
]
|
|
}
|
|
|
|
if obj.metadata:
|
|
metadata_dict = {}
|
|
if obj.metadata.id:
|
|
metadata_dict["id"] = obj.metadata.id
|
|
if obj.metadata.root:
|
|
metadata_dict["root"] = obj.metadata.root
|
|
if obj.metadata.user:
|
|
metadata_dict["user"] = obj.metadata.user
|
|
if obj.metadata.collection:
|
|
metadata_dict["collection"] = obj.metadata.collection
|
|
|
|
result["metadata"] = metadata_dict
|
|
|
|
return result
|