apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-04-25 00:16:23 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Pub/sub abstraction: decouple from Pulsar (#751)

Browse source 
Remove Pulsar-specific concepts from application code so that
the pub/sub backend is swappable via configuration.

Rename translators:
- to_pulsar/from_pulsar → decode/encode across all translator
  classes, dispatch handlers, and tests (55+ files)
- from_response_with_completion → encode_with_completion
- Remove pulsar.schema.Record from translator base class

Queue naming (CLASS:TOPICSPACE:TOPIC):
- Replace topic() helper with queue() using new format:
  flow:tg:name, request:tg:name, response:tg:name, state:tg:name
- Queue class implies persistence/TTL (no QoS in names)
- Update Pulsar backend map_topic() to parse new format
- Librarian queues use flow class (persistent, for chunking)
- Config push uses state class (persistent, last-value)
- Remove 15 dead topic imports from schema files
- Update init_trustgraph.py namespace: config → state

Confine Pulsar to pulsar_backend.py:
- Delete legacy PulsarClient class from pubsub.py
- Move add_args to add_pubsub_args() with standalone flag
  for CLI tools (defaults to localhost)
- PulsarBackendConsumer.receive() catches _pulsar.Timeout,
  raises standard TimeoutError
- Remove Pulsar imports from: async_processor, flow_processor,
  log_level, all 11 client files, 4 storage writers, gateway
  service, gateway config receiver
- Remove log_level/LoggerLevel from client API
- Rewrite tg-monitor-prompts to use backend abstraction
- Update tg-dump-queues to use add_pubsub_args

Also: pubsub-abstraction.md tech spec covering problem statement,
design goals, as-is requirements, candidate broker assessment,
approach, and implementation order.
This commit is contained in:
cybermaggedon 2026-04-01 20:16:53 +01:00 • committed by GitHub
parent dbf8daa74a
commit 4fb0b4d8e8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
106 changed files with 1269 additions and 788 deletions
Download patch file Download diff file Expand all files Collapse all files

2
trustgraph-base/trustgraph/base/__init__.py
View file

@ -1,5 +1,5 @@
from . pubsub import PulsarClient, get_pubsub
from . pubsub import get_pubsub, add_pubsub_args
from . async_processor import AsyncProcessor
from . consumer import Consumer
from . producer import Producer

17
trustgraph-base/trustgraph/base/async_processor.py
View file

@ -6,7 +6,6 @@
import asyncio
import argparse
import _pulsar
import time
import uuid
import logging
@ -15,7 +14,7 @@ from prometheus_client import start_http_server, Info
from .. schema import ConfigPush, config_push_queue
from .. log_level import LogLevel
from . pubsub import PulsarClient, get_pubsub
from . pubsub import get_pubsub, add_pubsub_args
from . producer import Producer
from . consumer import Consumer
from . metrics import ProcessorMetrics, ConsumerMetrics
@ -223,8 +222,8 @@ class AsyncProcessor:
logger.info("Keyboard interrupt.")
return
except _pulsar.Interrupted:
logger.info("Pulsar Interrupted.")
except KeyboardInterrupt:
logger.info("Interrupted.")
return
# Exceptions from a taskgroup come in as an exception group
@ -250,15 +249,7 @@ class AsyncProcessor:
@staticmethod
def add_args(parser):
# Pub/sub backend selection
parser.add_argument(
'--pubsub-backend',
default=os.getenv('PUBSUB_BACKEND', 'pulsar'),
choices=['pulsar', 'mqtt'],
help='Pub/sub backend (default: pulsar, env: PUBSUB_BACKEND)',
)
PulsarClient.add_args(parser)
add_pubsub_args(parser)
add_logging_args(parser)
parser.add_argument(

2
trustgraph-base/trustgraph/base/flow_processor.py
View file

@ -6,8 +6,6 @@
import json
import logging
from pulsar.schema import JsonSchema
from .. schema import Error
from .. schema import config_request_queue, config_response_queue
from .. schema import config_push_queue

118
trustgraph-base/trustgraph/base/pubsub.py
View file

@ -1,110 +1,72 @@
import os
import pulsar
import _pulsar
import uuid
from pulsar.schema import JsonSchema
import logging
from .. log_level import LogLevel
from .pulsar_backend import PulsarBackend
logger = logging.getLogger(__name__)
# Default connection settings from environment
DEFAULT_PULSAR_HOST = os.getenv("PULSAR_HOST", 'pulsar://pulsar:6650')
DEFAULT_PULSAR_API_KEY = os.getenv("PULSAR_API_KEY", None)
def get_pubsub(**config):
"""
Factory function to create a pub/sub backend based on configuration.
Args:
config: Configuration dictionary from command-line args
Must include 'pubsub_backend' key
config: Configuration dictionary from command-line args.
Key 'pubsub_backend' selects the backend (default: 'pulsar').
Returns:
Backend instance (PulsarBackend, MQTTBackend, etc.)
Example:
backend = get_pubsub(
pubsub_backend='pulsar',
pulsar_host='pulsar://localhost:6650'
)
Backend instance implementing the PubSubBackend protocol.
"""
backend_type = config.get('pubsub_backend', 'pulsar')
if backend_type == 'pulsar':
from .pulsar_backend import PulsarBackend
return PulsarBackend(
host=config.get('pulsar_host', PulsarClient.default_pulsar_host),
api_key=config.get('pulsar_api_key', PulsarClient.default_pulsar_api_key),
host=config.get('pulsar_host', DEFAULT_PULSAR_HOST),
api_key=config.get('pulsar_api_key', DEFAULT_PULSAR_API_KEY),
listener=config.get('pulsar_listener'),
)
elif backend_type == 'mqtt':
# TODO: Implement MQTT backend
raise NotImplementedError("MQTT backend not yet implemented")
else:
raise ValueError(f"Unknown pub/sub backend: {backend_type}")
class PulsarClient:
STANDALONE_PULSAR_HOST = 'pulsar://localhost:6650'
default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://pulsar:6650')
default_pulsar_api_key = os.getenv("PULSAR_API_KEY", None)
def __init__(self, **params):
def add_pubsub_args(parser, standalone=False):
"""Add pub/sub CLI arguments to an argument parser.
self.client = None
Args:
parser: argparse.ArgumentParser
standalone: If True, default host is localhost (for CLI tools
that run outside containers)
"""
host = STANDALONE_PULSAR_HOST if standalone else DEFAULT_PULSAR_HOST
listener_default = 'localhost' if standalone else None
pulsar_host = params.get("pulsar_host", self.default_pulsar_host)
pulsar_listener = params.get("pulsar_listener", None)
pulsar_api_key = params.get(
"pulsar_api_key",
self.default_pulsar_api_key
)
# Hard-code Pulsar logging to ERROR level to minimize noise
parser.add_argument(
'--pubsub-backend',
default=os.getenv('PUBSUB_BACKEND', 'pulsar'),
help='Pub/sub backend (default: pulsar, env: PUBSUB_BACKEND)',
)
self.pulsar_host = pulsar_host
self.pulsar_api_key = pulsar_api_key
parser.add_argument(
'-p', '--pulsar-host',
default=host,
help=f'Pulsar host (default: {host})',
)
if pulsar_api_key:
auth = pulsar.AuthenticationToken(pulsar_api_key)
self.client = pulsar.Client(
pulsar_host,
authentication=auth,
logger=pulsar.ConsoleLogger(_pulsar.LoggerLevel.Error)
)
else:
self.client = pulsar.Client(
pulsar_host,
listener_name=pulsar_listener,
logger=pulsar.ConsoleLogger(_pulsar.LoggerLevel.Error)
)
parser.add_argument(
'--pulsar-api-key',
default=DEFAULT_PULSAR_API_KEY,
help='Pulsar API key',
)
self.pulsar_listener = pulsar_listener
def close(self):
self.client.close()
def __del__(self):
if hasattr(self, "client"):
if self.client:
self.client.close()
@staticmethod
def add_args(parser):
parser.add_argument(
'-p', '--pulsar-host',
default=__class__.default_pulsar_host,
help=f'Pulsar host (default: {__class__.default_pulsar_host})',
)
parser.add_argument(
'--pulsar-api-key',
default=__class__.default_pulsar_api_key,
help=f'Pulsar API key',
)
parser.add_argument(
'--pulsar-listener',
help=f'Pulsar listener (default: none)',
)
parser.add_argument(
'--pulsar-listener',
default=listener_default,
help=f'Pulsar listener (default: {listener_default or "none"})',
)

47
trustgraph-base/trustgraph/base/pulsar_backend.py
View file

@ -181,8 +181,11 @@ class PulsarBackendConsumer:
self._schema_cls = schema_cls
def receive(self, timeout_millis: int = 2000) -> Message:
"""Receive a message."""
pulsar_msg = self._consumer.receive(timeout_millis=timeout_millis)
"""Receive a message. Raises TimeoutError if no message available."""
try:
pulsar_msg = self._consumer.receive(timeout_millis=timeout_millis)
except _pulsar.Timeout:
raise TimeoutError("No message received within timeout")
return PulsarMessage(pulsar_msg, self._schema_cls)
def acknowledge(self, message: Message) -> None:
@ -237,38 +240,44 @@ class PulsarBackend:
self.client = pulsar.Client(**client_args)
logger.info(f"Pulsar client connected to {host}")
def map_topic(self, generic_topic: str) -> str:
def map_topic(self, queue_id: str) -> str:
"""
Map generic topic format to Pulsar URI.
Map queue identifier to Pulsar URI.
Format: qos/tenant/namespace/queue
Example: q1/tg/flow/my-queue -> persistent://tg/flow/my-queue
Format: class:topicspace:topic
Example: flow:tg:text-completion-request -> persistent://tg/flow/text-completion-request
Args:
generic_topic: Generic topic string or already-formatted Pulsar URI
queue_id: Queue identifier string or already-formatted Pulsar URI
Returns:
Pulsar topic URI
"""
# If already a Pulsar URI, return as-is
if '://' in generic_topic:
return generic_topic
if '://' in queue_id:
return queue_id
parts = generic_topic.split('/', 3)
if len(parts) != 4:
raise ValueError(f"Invalid topic format: {generic_topic}, expected qos/tenant/namespace/queue")
parts = queue_id.split(':', 2)
if len(parts) != 3:
raise ValueError(
f"Invalid queue format: {queue_id}, "
f"expected class:topicspace:topic"
)
qos, tenant, namespace, queue = parts
cls, topicspace, topic = parts
# Map QoS to persistence
if qos == 'q0':
persistence = 'non-persistent'
elif qos in ['q1', 'q2']:
# Map class to Pulsar persistence and namespace
if cls in ('flow', 'state'):
persistence = 'persistent'
elif cls in ('request', 'response'):
persistence = 'non-persistent'
else:
raise ValueError(f"Invalid QoS level: {qos}, expected q0, q1, or q2")
raise ValueError(
f"Invalid queue class: {cls}, "
f"expected flow, request, response, or state"
)
return f"{persistence}://{tenant}/{namespace}/{queue}"
return f"{persistence}://{topicspace}/{cls}/{topic}"
def create_producer(self, topic: str, schema: type, **options) -> BackendProducer:
"""

8
trustgraph-base/trustgraph/clients/agent_client.py
View file

@ -1,5 +1,4 @@
import _pulsar
from .. schema import AgentRequest, AgentResponse
from .. schema import agent_request_queue
@ -7,15 +6,11 @@ from .. schema import agent_response_queue
from . base import BaseClient
# Ugly
ERROR=_pulsar.LoggerLevel.Error
WARN=_pulsar.LoggerLevel.Warn
INFO=_pulsar.LoggerLevel.Info
DEBUG=_pulsar.LoggerLevel.Debug
class AgentClient(BaseClient):
def __init__(
self, log_level=ERROR,
self,
subscriber=None,
input_queue=None,
output_queue=None,
@ -27,7 +22,6 @@ class AgentClient(BaseClient):
if output_queue is None: output_queue = agent_response_queue
super(AgentClient, self).__init__(
log_level=log_level,
subscriber=subscriber,
input_queue=input_queue,
output_queue=output_queue,

14
trustgraph-base/trustgraph/clients/base.py
View file

@ -1,10 +1,6 @@
import pulsar
import _pulsar
import hashlib
import uuid
import time
from pulsar.schema import JsonSchema
from .. exceptions import *
from ..base.pubsub import get_pubsub
@ -12,16 +8,11 @@ from ..base.pubsub import get_pubsub
# Default timeout for a request/response. In seconds.
DEFAULT_TIMEOUT=300
# Ugly
ERROR=_pulsar.LoggerLevel.Error
WARN=_pulsar.LoggerLevel.Warn
INFO=_pulsar.LoggerLevel.Info
DEBUG=_pulsar.LoggerLevel.Debug
class BaseClient:
def __init__(
self, log_level=ERROR,
self,
subscriber=None,
input_queue=None,
output_queue=None,
@ -87,7 +78,7 @@ class BaseClient:
try:
msg = self.consumer.receive(timeout_millis=2500)
except pulsar.exceptions.Timeout:
except TimeoutError:
continue
mid = msg.properties()["id"]
@ -139,4 +130,3 @@ class BaseClient:
if hasattr(self, "backend"):
self.backend.close()

8
trustgraph-base/trustgraph/clients/config_client.py
View file

@ -1,5 +1,4 @@
import _pulsar
import json
import dataclasses
@ -9,10 +8,6 @@ from .. schema import config_response_queue
from . base import BaseClient
# Ugly
ERROR=_pulsar.LoggerLevel.Error
WARN=_pulsar.LoggerLevel.Warn
INFO=_pulsar.LoggerLevel.Info
DEBUG=_pulsar.LoggerLevel.Debug
@dataclasses.dataclass
class Definition:
@ -34,7 +29,7 @@ class Topic:
class ConfigClient(BaseClient):
def __init__(
self, log_level=ERROR,
self,
subscriber=None,
input_queue=None,
output_queue=None,
@ -50,7 +45,6 @@ class ConfigClient(BaseClient):
output_queue = config_response_queue
super(ConfigClient, self).__init__(
log_level=log_level,
subscriber=subscriber,
input_queue=input_queue,
output_queue=output_queue,

8
trustgraph-base/trustgraph/clients/document_embeddings_client.py
View file

@ -1,5 +1,4 @@
import _pulsar
from .. schema import DocumentEmbeddingsRequest, DocumentEmbeddingsResponse
from .. schema import document_embeddings_request_queue
@ -7,15 +6,11 @@ from .. schema import document_embeddings_response_queue
from . base import BaseClient
# Ugly
ERROR=_pulsar.LoggerLevel.Error
WARN=_pulsar.LoggerLevel.Warn
INFO=_pulsar.LoggerLevel.Info
DEBUG=_pulsar.LoggerLevel.Debug
class DocumentEmbeddingsClient(BaseClient):
def __init__(
self, log_level=ERROR,
self,
subscriber=None,
input_queue=None,
output_queue=None,
@ -30,7 +25,6 @@ class DocumentEmbeddingsClient(BaseClient):
output_queue = document_embeddings_response_queue
super(DocumentEmbeddingsClient, self).__init__(
log_level=log_level,
subscriber=subscriber,
input_queue=input_queue,
output_queue=output_queue,

7
trustgraph-base/trustgraph/clients/document_rag_client.py
View file

@ -1,21 +1,15 @@
import _pulsar
from .. schema import DocumentRagQuery, DocumentRagResponse
from .. schema import document_rag_request_queue, document_rag_response_queue
from . base import BaseClient
# Ugly
ERROR=_pulsar.LoggerLevel.Error
WARN=_pulsar.LoggerLevel.Warn
INFO=_pulsar.LoggerLevel.Info
DEBUG=_pulsar.LoggerLevel.Debug
class DocumentRagClient(BaseClient):
def __init__(
self,
log_level=ERROR,
subscriber=None,
input_queue=None,
output_queue=None,
@ -30,7 +24,6 @@ class DocumentRagClient(BaseClient):
output_queue = document_rag_response_queue
super(DocumentRagClient, self).__init__(
log_level=log_level,
subscriber=subscriber,
input_queue=input_queue,
output_queue=output_queue,

9
trustgraph-base/trustgraph/clients/embeddings_client.py
View file

@ -1,20 +1,14 @@
from pulsar.schema import JsonSchema
from .. schema import EmbeddingsRequest, EmbeddingsResponse
from . base import BaseClient
import _pulsar
# Ugly
ERROR=_pulsar.LoggerLevel.Error
WARN=_pulsar.LoggerLevel.Warn
INFO=_pulsar.LoggerLevel.Info
DEBUG=_pulsar.LoggerLevel.Debug
class EmbeddingsClient(BaseClient):
def __init__(
self, log_level=ERROR,
self,
input_queue=None,
output_queue=None,
subscriber=None,
@ -23,7 +17,6 @@ class EmbeddingsClient(BaseClient):
):
super(EmbeddingsClient, self).__init__(
log_level=log_level,
subscriber=subscriber,
input_queue=input_queue,
output_queue=output_queue,

8
trustgraph-base/trustgraph/clients/graph_embeddings_client.py
View file

@ -1,5 +1,4 @@
import _pulsar
from .. schema import GraphEmbeddingsRequest, GraphEmbeddingsResponse
from .. schema import graph_embeddings_request_queue
@ -7,15 +6,11 @@ from .. schema import graph_embeddings_response_queue
from . base import BaseClient
# Ugly
ERROR=_pulsar.LoggerLevel.Error
WARN=_pulsar.LoggerLevel.Warn
INFO=_pulsar.LoggerLevel.Info
DEBUG=_pulsar.LoggerLevel.Debug
class GraphEmbeddingsClient(BaseClient):
def __init__(
self, log_level=ERROR,
self,
subscriber=None,
input_queue=None,
output_queue=None,
@ -30,7 +25,6 @@ class GraphEmbeddingsClient(BaseClient):
output_queue = graph_embeddings_response_queue
super(GraphEmbeddingsClient, self).__init__(
log_level=log_level,
subscriber=subscriber,
input_queue=input_queue,
output_queue=output_queue,

7
trustgraph-base/trustgraph/clients/graph_rag_client.py
View file

@ -1,21 +1,15 @@
import _pulsar
from .. schema import GraphRagQuery, GraphRagResponse
from .. schema import graph_rag_request_queue, graph_rag_response_queue
from . base import BaseClient
# Ugly
ERROR=_pulsar.LoggerLevel.Error
WARN=_pulsar.LoggerLevel.Warn
INFO=_pulsar.LoggerLevel.Info
DEBUG=_pulsar.LoggerLevel.Debug
class GraphRagClient(BaseClient):
def __init__(
self,
log_level=ERROR,
subscriber=None,
input_queue=None,
output_queue=None,
@ -30,7 +24,6 @@ class GraphRagClient(BaseClient):
output_queue = graph_rag_response_queue
super(GraphRagClient, self).__init__(
log_level=log_level,
subscriber=subscriber,
input_queue=input_queue,
output_queue=output_queue,

8
trustgraph-base/trustgraph/clients/llm_client.py
View file

@ -1,5 +1,4 @@
import _pulsar
from .. schema import TextCompletionRequest, TextCompletionResponse
from .. schema import text_completion_request_queue
@ -8,15 +7,11 @@ from . base import BaseClient
from .. exceptions import LlmError
# Ugly
ERROR=_pulsar.LoggerLevel.Error
WARN=_pulsar.LoggerLevel.Warn
INFO=_pulsar.LoggerLevel.Info
DEBUG=_pulsar.LoggerLevel.Debug
class LlmClient(BaseClient):
def __init__(
self, log_level=ERROR,
self,
subscriber=None,
input_queue=None,
output_queue=None,
@ -28,7 +23,6 @@ class LlmClient(BaseClient):
if output_queue is None: output_queue = text_completion_response_queue
super(LlmClient, self).__init__(
log_level=log_level,
subscriber=subscriber,
input_queue=input_queue,
output_queue=output_queue,

8
trustgraph-base/trustgraph/clients/prompt_client.py
View file

@ -1,5 +1,4 @@
import _pulsar
import json
import dataclasses
@ -9,10 +8,6 @@ from .. schema import prompt_response_queue
from . base import BaseClient
# Ugly
ERROR=_pulsar.LoggerLevel.Error
WARN=_pulsar.LoggerLevel.Warn
INFO=_pulsar.LoggerLevel.Info
DEBUG=_pulsar.LoggerLevel.Debug
@dataclasses.dataclass
class Definition:
@ -34,7 +29,7 @@ class Topic:
class PromptClient(BaseClient):
def __init__(
self, log_level=ERROR,
self,
subscriber=None,
input_queue=None,
output_queue=None,
@ -49,7 +44,6 @@ class PromptClient(BaseClient):
output_queue = prompt_response_queue
super(PromptClient, self).__init__(
log_level=log_level,
subscriber=subscriber,
input_queue=input_queue,
output_queue=output_queue,

8
trustgraph-base/trustgraph/clients/row_embeddings_client.py
View file

@ -1,5 +1,4 @@
import _pulsar
from .. schema import RowEmbeddingsRequest, RowEmbeddingsResponse
from .. schema import row_embeddings_request_queue
@ -7,15 +6,11 @@ from .. schema import row_embeddings_response_queue
from . base import BaseClient
# Ugly
ERROR=_pulsar.LoggerLevel.Error
WARN=_pulsar.LoggerLevel.Warn
INFO=_pulsar.LoggerLevel.Info
DEBUG=_pulsar.LoggerLevel.Debug
class RowEmbeddingsClient(BaseClient):
def __init__(
self, log_level=ERROR,
self,
subscriber=None,
input_queue=None,
output_queue=None,
@ -30,7 +25,6 @@ class RowEmbeddingsClient(BaseClient):
output_queue = row_embeddings_response_queue
super(RowEmbeddingsClient, self).__init__(
log_level=log_level,
subscriber=subscriber,
input_queue=input_queue,
output_queue=output_queue,

8
trustgraph-base/trustgraph/clients/triples_query_client.py
View file

@ -1,6 +1,5 @@
#!/usr/bin/env python3
import _pulsar
from .. schema import TriplesQueryRequest, TriplesQueryResponse, Term, IRI, LITERAL
from .. schema import triples_request_queue
@ -8,15 +7,11 @@ from .. schema import triples_response_queue
from . base import BaseClient
# Ugly
ERROR=_pulsar.LoggerLevel.Error
WARN=_pulsar.LoggerLevel.Warn
INFO=_pulsar.LoggerLevel.Info
DEBUG=_pulsar.LoggerLevel.Debug
class TriplesQueryClient(BaseClient):
def __init__(
self, log_level=ERROR,
self,
subscriber=None,
input_queue=None,
output_queue=None,
@ -31,7 +26,6 @@ class TriplesQueryClient(BaseClient):
output_queue = triples_response_queue
super(TriplesQueryClient, self).__init__(
log_level=log_level,
subscriber=subscriber,
input_queue=input_queue,
output_queue=output_queue,

10
trustgraph-base/trustgraph/log_level.py
View file

@ -1,6 +1,6 @@
from enum import Enum
import _pulsar
class LogLevel(Enum):
DEBUG = 'debug'
@ -10,11 +10,3 @@ class LogLevel(Enum):
def __str__(self):
return self.value
def to_pulsar(self):
if self == LogLevel.DEBUG: return _pulsar.LoggerLevel.Debug
if self == LogLevel.INFO: return _pulsar.LoggerLevel.Info
if self == LogLevel.WARN: return _pulsar.LoggerLevel.Warn
if self == LogLevel.ERROR: return _pulsar.LoggerLevel.Error
raise RuntimeError("Log level mismatch")

12
trustgraph-base/trustgraph/messaging/translators/agent.py
View file

@ -6,7 +6,7 @@ from .base import MessageTranslator
class AgentRequestTranslator(MessageTranslator):
"""Translator for AgentRequest schema objects"""
def to_pulsar(self, data: Dict[str, Any]) -> AgentRequest:
def decode(self, data: Dict[str, Any]) -> AgentRequest:
return AgentRequest(
question=data["question"],
state=data.get("state", None),
@ -26,7 +26,7 @@ class AgentRequestTranslator(MessageTranslator):
expected_siblings=data.get("expected_siblings", 0),
)
def from_pulsar(self, obj: AgentRequest) -> Dict[str, Any]:
def encode(self, obj: AgentRequest) -> Dict[str, Any]:
return {
"question": obj.question,
"state": obj.state,
@ -50,10 +50,10 @@ class AgentRequestTranslator(MessageTranslator):
class AgentResponseTranslator(MessageTranslator):
"""Translator for AgentResponse schema objects"""
def to_pulsar(self, data: Dict[str, Any]) -> AgentResponse:
def decode(self, data: Dict[str, Any]) -> AgentResponse:
raise NotImplementedError("Response translation to Pulsar not typically needed")
def from_pulsar(self, obj: AgentResponse) -> Dict[str, Any]:
def encode(self, obj: AgentResponse) -> Dict[str, Any]:
result = {}
if obj.chunk_type:
@ -81,7 +81,7 @@ class AgentResponseTranslator(MessageTranslator):
return result
def from_response_with_completion(self, obj: AgentResponse) -> Tuple[Dict[str, Any], bool]:
def encode_with_completion(self, obj: AgentResponse) -> Tuple[Dict[str, Any], bool]:
"""Returns (response_dict, is_final)"""
is_final = getattr(obj, 'end_of_dialog', False)
return self.from_pulsar(obj), is_final
return self.encode(obj), is_final

47
trustgraph-base/trustgraph/messaging/translators/base.py
View file

@ -1,43 +1,46 @@
from abc import ABC, abstractmethod
from typing import Dict, Any, Tuple
from pulsar.schema import Record
class Translator(ABC):
"""Base class for bidirectional Pulsar ↔ dict translation"""
"""Base class for bidirectional schema ↔ dict translation.
Translates between external API dicts (JSON from HTTP/WebSocket)
and internal schema objects (dataclasses).
"""
@abstractmethod
def to_pulsar(self, data: Dict[str, Any]) -> Record:
"""Convert dict to Pulsar schema object"""
def decode(self, data: Dict[str, Any]) -> Any:
"""Convert external dict to schema object."""
pass
@abstractmethod
def from_pulsar(self, obj: Record) -> Dict[str, Any]:
"""Convert Pulsar schema object to dict"""
@abstractmethod
def encode(self, obj: Any) -> Dict[str, Any]:
"""Convert schema object to external dict."""
pass
class MessageTranslator(Translator):
"""For complete request/response message translation"""
def from_response_with_completion(self, obj: Record) -> Tuple[Dict[str, Any], bool]:
"""Returns (response_dict, is_final) - for streaming responses"""
return self.from_pulsar(obj), True
"""For complete request/response message translation."""
def encode_with_completion(self, obj: Any) -> Tuple[Dict[str, Any], bool]:
"""Returns (response_dict, is_final) — for streaming responses."""
return self.encode(obj), True
class SendTranslator(Translator):
"""For fire-and-forget send operations (like ServiceSender)"""
def from_pulsar(self, obj: Record) -> Dict[str, Any]:
"""Usually not needed for send-only operations"""
raise NotImplementedError("Send translators typically don't need from_pulsar")
"""For fire-and-forget send operations."""
def encode(self, obj: Any) -> Dict[str, Any]:
"""Usually not needed for send-only operations."""
raise NotImplementedError("Send translators don't need encode")
def handle_optional_fields(obj: Record, fields: list) -> Dict[str, Any]:
"""Helper to extract optional fields from Pulsar object"""
def handle_optional_fields(obj: Any, fields: list) -> Dict[str, Any]:
"""Helper to extract optional fields from a schema object."""
result = {}
for field in fields:
value = getattr(obj, field, None)
if value is not None:
result[field] = value
return result
return result

8
trustgraph-base/trustgraph/messaging/translators/collection.py
View file

@ -6,7 +6,7 @@ from .base import MessageTranslator
class CollectionManagementRequestTranslator(MessageTranslator):
"""Translator for CollectionManagementRequest schema objects"""
def to_pulsar(self, data: Dict[str, Any]) -> CollectionManagementRequest:
def decode(self, data: Dict[str, Any]) -> CollectionManagementRequest:
return CollectionManagementRequest(
operation=data.get("operation"),
user=data.get("user"),
@ -19,7 +19,7 @@ class CollectionManagementRequestTranslator(MessageTranslator):
limit=data.get("limit")
)
def from_pulsar(self, obj: CollectionManagementRequest) -> Dict[str, Any]:
def encode(self, obj: CollectionManagementRequest) -> Dict[str, Any]:
result = {}
if obj.operation is not None:
@ -47,7 +47,7 @@ class CollectionManagementRequestTranslator(MessageTranslator):
class CollectionManagementResponseTranslator(MessageTranslator):
"""Translator for CollectionManagementResponse schema objects"""
def to_pulsar(self, data: Dict[str, Any]) -> CollectionManagementResponse:
def decode(self, data: Dict[str, Any]) -> CollectionManagementResponse:
# Handle error
error = None
@ -76,7 +76,7 @@ class CollectionManagementResponseTranslator(MessageTranslator):
collections=collections
)
def from_pulsar(self, obj: CollectionManagementResponse) -> Dict[str, Any]:
def encode(self, obj: CollectionManagementResponse) -> Dict[str, Any]:
result = {}
print("COLLECTIONMGMT", obj, flush=True)

12
trustgraph-base/trustgraph/messaging/translators/config.py
View file

@ -6,7 +6,7 @@ from .base import MessageTranslator
class ConfigRequestTranslator(MessageTranslator):
"""Translator for ConfigRequest schema objects"""
def to_pulsar(self, data: Dict[str, Any]) -> ConfigRequest:
def decode(self, data: Dict[str, Any]) -> ConfigRequest:
keys = None
if "keys" in data:
keys = [
@ -35,7 +35,7 @@ class ConfigRequestTranslator(MessageTranslator):
values=values
)
def from_pulsar(self, obj: ConfigRequest) -> Dict[str, Any]:
def encode(self, obj: ConfigRequest) -> Dict[str, Any]:
result = {}
if obj.operation is not None:
@ -69,10 +69,10 @@ class ConfigRequestTranslator(MessageTranslator):
class ConfigResponseTranslator(MessageTranslator):
"""Translator for ConfigResponse schema objects"""
def to_pulsar(self, data: Dict[str, Any]) -> ConfigResponse:
def decode(self, data: Dict[str, Any]) -> ConfigResponse:
raise NotImplementedError("Response translation to Pulsar not typically needed")
def from_pulsar(self, obj: ConfigResponse) -> Dict[str, Any]:
def encode(self, obj: ConfigResponse) -> Dict[str, Any]:
result = {}
if obj.version is not None:
@ -96,6 +96,6 @@ class ConfigResponseTranslator(MessageTranslator):
return result
def from_response_with_completion(self, obj: ConfigResponse) -> Tuple[Dict[str, Any], bool]:
def encode_with_completion(self, obj: ConfigResponse) -> Tuple[Dict[str, Any], bool]:
"""Returns (response_dict, is_final)"""
return self.from_pulsar(obj), True
return self.encode(obj), True

12
trustgraph-base/trustgraph/messaging/translators/diagnosis.py
View file

@ -7,7 +7,7 @@ from .base import MessageTranslator
class StructuredDataDiagnosisRequestTranslator(MessageTranslator):
"""Translator for StructuredDataDiagnosisRequest schema objects"""
def to_pulsar(self, data: Dict[str, Any]) -> StructuredDataDiagnosisRequest:
def decode(self, data: Dict[str, Any]) -> StructuredDataDiagnosisRequest:
return StructuredDataDiagnosisRequest(
operation=data["operation"],
sample=data["sample"],
@ -16,7 +16,7 @@ class StructuredDataDiagnosisRequestTranslator(MessageTranslator):
options=data.get("options", {})
)
def from_pulsar(self, obj: StructuredDataDiagnosisRequest) -> Dict[str, Any]:
def encode(self, obj: StructuredDataDiagnosisRequest) -> Dict[str, Any]:
result = {
"operation": obj.operation,
"sample": obj.sample,
@ -36,10 +36,10 @@ class StructuredDataDiagnosisRequestTranslator(MessageTranslator):
class StructuredDataDiagnosisResponseTranslator(MessageTranslator):
"""Translator for StructuredDataDiagnosisResponse schema objects"""
def to_pulsar(self, data: Dict[str, Any]) -> StructuredDataDiagnosisResponse:
def decode(self, data: Dict[str, Any]) -> StructuredDataDiagnosisResponse:
raise NotImplementedError("Response translation to Pulsar not typically needed")
def from_pulsar(self, obj: StructuredDataDiagnosisResponse) -> Dict[str, Any]:
def encode(self, obj: StructuredDataDiagnosisResponse) -> Dict[str, Any]:
result = {
"operation": obj.operation
}
@ -64,6 +64,6 @@ class StructuredDataDiagnosisResponseTranslator(MessageTranslator):
return result
def from_response_with_completion(self, obj: StructuredDataDiagnosisResponse) -> Tuple[Dict[str, Any], bool]:
def encode_with_completion(self, obj: StructuredDataDiagnosisResponse) -> Tuple[Dict[str, Any], bool]:
"""Returns (response_dict, is_final)"""
return self.from_pulsar(obj), True
return self.encode(obj), True

16
trustgraph-base/trustgraph/messaging/translators/document_loading.py
View file

@ -30,7 +30,7 @@ def _decode_text_payload(payload: str | bytes, charset: str) -> str:
class DocumentTranslator(SendTranslator):
"""Translator for Document schema objects (PDF docs etc.)"""
def to_pulsar(self, data: Dict[str, Any]) -> Document:
def decode(self, data: Dict[str, Any]) -> Document:
# Handle base64 content validation
doc = base64.b64decode(data["data"])
@ -45,7 +45,7 @@ class DocumentTranslator(SendTranslator):
data=base64.b64encode(doc).decode("utf-8")
)
def from_pulsar(self, obj: Document) -> Dict[str, Any]:
def encode(self, obj: Document) -> Dict[str, Any]:
result = {
"data": obj.data
}
@ -69,7 +69,7 @@ class DocumentTranslator(SendTranslator):
class TextDocumentTranslator(SendTranslator):
"""Translator for TextDocument schema objects"""
def to_pulsar(self, data: Dict[str, Any]) -> TextDocument:
def decode(self, data: Dict[str, Any]) -> TextDocument:
charset = data.get("charset", "utf-8")
text = _decode_text_payload(data["text"], charset)
@ -85,7 +85,7 @@ class TextDocumentTranslator(SendTranslator):
text=text.encode("utf-8")
)
def from_pulsar(self, obj: TextDocument) -> Dict[str, Any]:
def encode(self, obj: TextDocument) -> Dict[str, Any]:
result = {
"text": obj.text.decode("utf-8") if isinstance(obj.text, bytes) else obj.text
}
@ -109,7 +109,7 @@ class TextDocumentTranslator(SendTranslator):
class ChunkTranslator(SendTranslator):
"""Translator for Chunk schema objects"""
def to_pulsar(self, data: Dict[str, Any]) -> Chunk:
def decode(self, data: Dict[str, Any]) -> Chunk:
from ...schema import Metadata
return Chunk(
metadata=Metadata(
@ -121,7 +121,7 @@ class ChunkTranslator(SendTranslator):
chunk=data["chunk"].encode("utf-8") if isinstance(data["chunk"], str) else data["chunk"]
)
def from_pulsar(self, obj: Chunk) -> Dict[str, Any]:
def encode(self, obj: Chunk) -> Dict[str, Any]:
result = {
"chunk": obj.chunk.decode("utf-8") if isinstance(obj.chunk, bytes) else obj.chunk
}
@ -145,7 +145,7 @@ class ChunkTranslator(SendTranslator):
class DocumentEmbeddingsTranslator(SendTranslator):
"""Translator for DocumentEmbeddings schema objects"""
def to_pulsar(self, data: Dict[str, Any]) -> DocumentEmbeddings:
def decode(self, data: Dict[str, Any]) -> DocumentEmbeddings:
metadata = data.get("metadata", {})
chunks = [
@ -167,7 +167,7 @@ class DocumentEmbeddingsTranslator(SendTranslator):
chunks=chunks
)
def from_pulsar(self, obj: DocumentEmbeddings) -> Dict[str, Any]:
def encode(self, obj: DocumentEmbeddings) -> Dict[str, Any]:
result = {
"chunks": [
{

12
trustgraph-base/trustgraph/messaging/translators/embeddings.py
View file

@ -6,12 +6,12 @@ from .base import MessageTranslator
class EmbeddingsRequestTranslator(MessageTranslator):
"""Translator for EmbeddingsRequest schema objects"""
def to_pulsar(self, data: Dict[str, Any]) -> EmbeddingsRequest:
def decode(self, data: Dict[str, Any]) -> EmbeddingsRequest:
return EmbeddingsRequest(
texts=data["texts"]
)
def from_pulsar(self, obj: EmbeddingsRequest) -> Dict[str, Any]:
def encode(self, obj: EmbeddingsRequest) -> Dict[str, Any]:
return {
"texts": obj.texts
}
@ -20,14 +20,14 @@ class EmbeddingsRequestTranslator(MessageTranslator):
class EmbeddingsResponseTranslator(MessageTranslator):
"""Translator for EmbeddingsResponse schema objects"""
def to_pulsar(self, data: Dict[str, Any]) -> EmbeddingsResponse:
def decode(self, data: Dict[str, Any]) -> EmbeddingsResponse:
raise NotImplementedError("Response translation to Pulsar not typically needed")
def from_pulsar(self, obj: EmbeddingsResponse) -> Dict[str, Any]:
def encode(self, obj: EmbeddingsResponse) -> Dict[str, Any]:
return {
"vectors": obj.vectors
}
def from_response_with_completion(self, obj: EmbeddingsResponse) -> Tuple[Dict[str, Any], bool]:
def encode_with_completion(self, obj: EmbeddingsResponse) -> Tuple[Dict[str, Any], bool]:
"""Returns (response_dict, is_final)"""
return self.from_pulsar(obj), True
return self.encode(obj), True

38
trustgraph-base/trustgraph/messaging/translators/embeddings_query.py
View file

@ -11,7 +11,7 @@ from .primitives import ValueTranslator
class DocumentEmbeddingsRequestTranslator(MessageTranslator):
"""Translator for DocumentEmbeddingsRequest schema objects"""
def to_pulsar(self, data: Dict[str, Any]) -> DocumentEmbeddingsRequest:
def decode(self, data: Dict[str, Any]) -> DocumentEmbeddingsRequest:
return DocumentEmbeddingsRequest(
vector=data["vector"],
limit=int(data.get("limit", 10)),
@ -19,7 +19,7 @@ class DocumentEmbeddingsRequestTranslator(MessageTranslator):
collection=data.get("collection", "default")
)
def from_pulsar(self, obj: DocumentEmbeddingsRequest) -> Dict[str, Any]:
def encode(self, obj: DocumentEmbeddingsRequest) -> Dict[str, Any]:
return {
"vector": obj.vector,
"limit": obj.limit,
@ -31,10 +31,10 @@ class DocumentEmbeddingsRequestTranslator(MessageTranslator):
class DocumentEmbeddingsResponseTranslator(MessageTranslator):
"""Translator for DocumentEmbeddingsResponse schema objects"""
def to_pulsar(self, data: Dict[str, Any]) -> DocumentEmbeddingsResponse:
def decode(self, data: Dict[str, Any]) -> DocumentEmbeddingsResponse:
raise NotImplementedError("Response translation to Pulsar not typically needed")
def from_pulsar(self, obj: DocumentEmbeddingsResponse) -> Dict[str, Any]:
def encode(self, obj: DocumentEmbeddingsResponse) -> Dict[str, Any]:
result = {}
if obj.chunks is not None:
@ -48,15 +48,15 @@ class DocumentEmbeddingsResponseTranslator(MessageTranslator):
return result
def from_response_with_completion(self, obj: DocumentEmbeddingsResponse) -> Tuple[Dict[str, Any], bool]:
def encode_with_completion(self, obj: DocumentEmbeddingsResponse) -> Tuple[Dict[str, Any], bool]:
"""Returns (response_dict, is_final)"""
return self.from_pulsar(obj), True
return self.encode(obj), True
class GraphEmbeddingsRequestTranslator(MessageTranslator):
"""Translator for GraphEmbeddingsRequest schema objects"""
def to_pulsar(self, data: Dict[str, Any]) -> GraphEmbeddingsRequest:
def decode(self, data: Dict[str, Any]) -> GraphEmbeddingsRequest:
return GraphEmbeddingsRequest(
vector=data["vector"],
limit=int(data.get("limit", 10)),
@ -64,7 +64,7 @@ class GraphEmbeddingsRequestTranslator(MessageTranslator):
collection=data.get("collection", "default")
)
def from_pulsar(self, obj: GraphEmbeddingsRequest) -> Dict[str, Any]:
def encode(self, obj: GraphEmbeddingsRequest) -> Dict[str, Any]:
return {
"vector": obj.vector,
"limit": obj.limit,
@ -79,16 +79,16 @@ class GraphEmbeddingsResponseTranslator(MessageTranslator):
def __init__(self):
self.value_translator = ValueTranslator()
def to_pulsar(self, data: Dict[str, Any]) -> GraphEmbeddingsResponse:
def decode(self, data: Dict[str, Any]) -> GraphEmbeddingsResponse:
raise NotImplementedError("Response translation to Pulsar not typically needed")
def from_pulsar(self, obj: GraphEmbeddingsResponse) -> Dict[str, Any]:
def encode(self, obj: GraphEmbeddingsResponse) -> Dict[str, Any]:
result = {}
if obj.entities is not None:
result["entities"] = [
{
"entity": self.value_translator.from_pulsar(match.entity),
"entity": self.value_translator.encode(match.entity),
"score": match.score
}
for match in obj.entities
@ -96,15 +96,15 @@ class GraphEmbeddingsResponseTranslator(MessageTranslator):
return result
def from_response_with_completion(self, obj: GraphEmbeddingsResponse) -> Tuple[Dict[str, Any], bool]:
def encode_with_completion(self, obj: GraphEmbeddingsResponse) -> Tuple[Dict[str, Any], bool]:
"""Returns (response_dict, is_final)"""
return self.from_pulsar(obj), True
return self.encode(obj), True
class RowEmbeddingsRequestTranslator(MessageTranslator):
"""Translator for RowEmbeddingsRequest schema objects"""
def to_pulsar(self, data: Dict[str, Any]) -> RowEmbeddingsRequest:
def decode(self, data: Dict[str, Any]) -> RowEmbeddingsRequest:
return RowEmbeddingsRequest(
vector=data["vector"],
limit=int(data.get("limit", 10)),
@ -114,7 +114,7 @@ class RowEmbeddingsRequestTranslator(MessageTranslator):
index_name=data.get("index_name")
)
def from_pulsar(self, obj: RowEmbeddingsRequest) -> Dict[str, Any]:
def encode(self, obj: RowEmbeddingsRequest) -> Dict[str, Any]:
result = {
"vector": obj.vector,
"limit": obj.limit,
@ -130,10 +130,10 @@ class RowEmbeddingsRequestTranslator(MessageTranslator):
class RowEmbeddingsResponseTranslator(MessageTranslator):
"""Translator for RowEmbeddingsResponse schema objects"""
def to_pulsar(self, data: Dict[str, Any]) -> RowEmbeddingsResponse:
def decode(self, data: Dict[str, Any]) -> RowEmbeddingsResponse:
raise NotImplementedError("Response translation to Pulsar not typically needed")
def from_pulsar(self, obj: RowEmbeddingsResponse) -> Dict[str, Any]:
def encode(self, obj: RowEmbeddingsResponse) -> Dict[str, Any]:
result = {}
if obj.error is not None:
@ -155,6 +155,6 @@ class RowEmbeddingsResponseTranslator(MessageTranslator):
return result
def from_response_with_completion(self, obj: RowEmbeddingsResponse) -> Tuple[Dict[str, Any], bool]:
def encode_with_completion(self, obj: RowEmbeddingsResponse) -> Tuple[Dict[str, Any], bool]:
"""Returns (response_dict, is_final)"""
return self.from_pulsar(obj), True
return self.encode(obj), True

12
trustgraph-base/trustgraph/messaging/translators/flow.py
View file

@ -6,7 +6,7 @@ from .base import MessageTranslator
class FlowRequestTranslator(MessageTranslator):
"""Translator for FlowRequest schema objects"""
def to_pulsar(self, data: Dict[str, Any]) -> FlowRequest:
def decode(self, data: Dict[str, Any]) -> FlowRequest:
return FlowRequest(
operation=data.get("operation"),
blueprint_name=data.get("blueprint-name"),
@ -16,7 +16,7 @@ class FlowRequestTranslator(MessageTranslator):
parameters=data.get("parameters")
)
def from_pulsar(self, obj: FlowRequest) -> Dict[str, Any]:
def encode(self, obj: FlowRequest) -> Dict[str, Any]:
result = {}
if obj.operation is not None:
@ -38,10 +38,10 @@ class FlowRequestTranslator(MessageTranslator):
class FlowResponseTranslator(MessageTranslator):
"""Translator for FlowResponse schema objects"""
def to_pulsar(self, data: Dict[str, Any]) -> FlowResponse:
def decode(self, data: Dict[str, Any]) -> FlowResponse:
raise NotImplementedError("Response translation to Pulsar not typically needed")
def from_pulsar(self, obj: FlowResponse) -> Dict[str, Any]:
def encode(self, obj: FlowResponse) -> Dict[str, Any]:
result = {}
if obj.blueprint_names is not None:
@ -59,6 +59,6 @@ class FlowResponseTranslator(MessageTranslator):
return result
def from_response_with_completion(self, obj: FlowResponse) -> Tuple[Dict[str, Any], bool]:
def encode_with_completion(self, obj: FlowResponse) -> Tuple[Dict[str, Any], bool]:
"""Returns (response_dict, is_final)"""
return self.from_pulsar(obj), True
return self.encode(obj), True

24
trustgraph-base/trustgraph/messaging/translators/knowledge.py
View file

@ -14,7 +14,7 @@ class KnowledgeRequestTranslator(MessageTranslator):
self.value_translator = ValueTranslator()
self.subgraph_translator = SubgraphTranslator()
def to_pulsar(self, data: Dict[str, Any]) -> KnowledgeRequest:
def decode(self, data: Dict[str, Any]) -> KnowledgeRequest:
triples = None
if "triples" in data:
triples = Triples(
@ -24,7 +24,7 @@ class KnowledgeRequestTranslator(MessageTranslator):
user=data["triples"]["metadata"]["user"],
collection=data["triples"]["metadata"]["collection"]
),
triples=self.subgraph_translator.to_pulsar(data["triples"]["triples"]),
triples=self.subgraph_translator.decode(data["triples"]["triples"]),
)
graph_embeddings = None
@ -38,7 +38,7 @@ class KnowledgeRequestTranslator(MessageTranslator):
),
entities=[
EntityEmbeddings(
entity=self.value_translator.to_pulsar(ent["entity"]),
entity=self.value_translator.decode(ent["entity"]),
vectors=ent["vectors"],
)
for ent in data["graph-embeddings"]["entities"]
@ -55,7 +55,7 @@ class KnowledgeRequestTranslator(MessageTranslator):
graph_embeddings=graph_embeddings,
)
def from_pulsar(self, obj: KnowledgeRequest) -> Dict[str, Any]:
def encode(self, obj: KnowledgeRequest) -> Dict[str, Any]:
result = {}
if obj.operation:
@ -77,7 +77,7 @@ class KnowledgeRequestTranslator(MessageTranslator):
"user": obj.triples.metadata.user,
"collection": obj.triples.metadata.collection,
},
"triples": self.subgraph_translator.from_pulsar(obj.triples.triples),
"triples": self.subgraph_translator.encode(obj.triples.triples),
}
if obj.graph_embeddings:
@ -91,7 +91,7 @@ class KnowledgeRequestTranslator(MessageTranslator):
"entities": [
{
"vector": entity.vector,
"entity": self.value_translator.from_pulsar(entity.entity),
"entity": self.value_translator.encode(entity.entity),
}
for entity in obj.graph_embeddings.entities
],
@ -107,10 +107,10 @@ class KnowledgeResponseTranslator(MessageTranslator):
self.value_translator = ValueTranslator()
self.subgraph_translator = SubgraphTranslator()
def to_pulsar(self, data: Dict[str, Any]) -> KnowledgeResponse:
def decode(self, data: Dict[str, Any]) -> KnowledgeResponse:
raise NotImplementedError("Response translation to Pulsar not typically needed")
def from_pulsar(self, obj: KnowledgeResponse) -> Dict[str, Any]:
def encode(self, obj: KnowledgeResponse) -> Dict[str, Any]:
# Response to list operation
if obj.ids is not None:
return {"ids": obj.ids}
@ -125,7 +125,7 @@ class KnowledgeResponseTranslator(MessageTranslator):
"user": obj.triples.metadata.user,
"collection": obj.triples.metadata.collection,
},
"triples": self.subgraph_translator.from_pulsar(obj.triples.triples),
"triples": self.subgraph_translator.encode(obj.triples.triples),
}
}
@ -142,7 +142,7 @@ class KnowledgeResponseTranslator(MessageTranslator):
"entities": [
{
"vector": entity.vector,
"entity": self.value_translator.from_pulsar(entity.entity),
"entity": self.value_translator.encode(entity.entity),
}
for entity in obj.graph_embeddings.entities
],
@ -156,9 +156,9 @@ class KnowledgeResponseTranslator(MessageTranslator):
# Empty response (successful delete)
return {}
def from_response_with_completion(self, obj: KnowledgeResponse) -> Tuple[Dict[str, Any], bool]:
def encode_with_completion(self, obj: KnowledgeResponse) -> Tuple[Dict[str, Any], bool]:
"""Returns (response_dict, is_final)"""
response = self.from_pulsar(obj)
response = self.encode(obj)
# Check if this is a final response
is_final = (

26
trustgraph-base/trustgraph/messaging/translators/library.py
View file

@ -11,16 +11,16 @@ class LibraryRequestTranslator(MessageTranslator):
self.doc_metadata_translator = DocumentMetadataTranslator()
self.proc_metadata_translator = ProcessingMetadataTranslator()
def to_pulsar(self, data: Dict[str, Any]) -> LibrarianRequest:
def decode(self, data: Dict[str, Any]) -> LibrarianRequest:
# Document metadata
doc_metadata = None
if "document-metadata" in data:
doc_metadata = self.doc_metadata_translator.to_pulsar(data["document-metadata"])
doc_metadata = self.doc_metadata_translator.decode(data["document-metadata"])
# Processing metadata
proc_metadata = None
if "processing-metadata" in data:
proc_metadata = self.proc_metadata_translator.to_pulsar(data["processing-metadata"])
proc_metadata = self.proc_metadata_translator.decode(data["processing-metadata"])
# Criteria
criteria = []
@ -61,7 +61,7 @@ class LibraryRequestTranslator(MessageTranslator):
include_children=data.get("include-children", False),
)
def from_pulsar(self, obj: LibrarianRequest) -> Dict[str, Any]:
def encode(self, obj: LibrarianRequest) -> Dict[str, Any]:
result = {}
if obj.operation:
@ -71,9 +71,9 @@ class LibraryRequestTranslator(MessageTranslator):
if obj.processing_id:
result["processing-id"] = obj.processing_id
if obj.document_metadata:
result["document-metadata"] = self.doc_metadata_translator.from_pulsar(obj.document_metadata)
result["document-metadata"] = self.doc_metadata_translator.encode(obj.document_metadata)
if obj.processing_metadata:
result["processing-metadata"] = self.proc_metadata_translator.from_pulsar(obj.processing_metadata)
result["processing-metadata"] = self.proc_metadata_translator.encode(obj.processing_metadata)
if obj.content:
result["content"] = obj.content.decode("utf-8") if isinstance(obj.content, bytes) else obj.content
if obj.user:
@ -100,10 +100,10 @@ class LibraryResponseTranslator(MessageTranslator):
self.doc_metadata_translator = DocumentMetadataTranslator()
self.proc_metadata_translator = ProcessingMetadataTranslator()
def to_pulsar(self, data: Dict[str, Any]) -> LibrarianResponse:
def decode(self, data: Dict[str, Any]) -> LibrarianResponse:
raise NotImplementedError("Response translation to Pulsar not typically needed")
def from_pulsar(self, obj: LibrarianResponse) -> Dict[str, Any]:
def encode(self, obj: LibrarianResponse) -> Dict[str, Any]:
result = {}
if obj.error:
@ -113,20 +113,20 @@ class LibraryResponseTranslator(MessageTranslator):
}
if obj.document_metadata:
result["document-metadata"] = self.doc_metadata_translator.from_pulsar(obj.document_metadata)
result["document-metadata"] = self.doc_metadata_translator.encode(obj.document_metadata)
if obj.content:
result["content"] = obj.content.decode("utf-8") if isinstance(obj.content, bytes) else obj.content
if obj.document_metadatas is not None:
result["document-metadatas"] = [
self.doc_metadata_translator.from_pulsar(dm)
self.doc_metadata_translator.encode(dm)
for dm in obj.document_metadatas
]
if obj.processing_metadatas is not None:
result["processing-metadatas"] = [
self.proc_metadata_translator.from_pulsar(pm)
self.proc_metadata_translator.encode(pm)
for pm in obj.processing_metadatas
]
@ -172,6 +172,6 @@ class LibraryResponseTranslator(MessageTranslator):
return result
def from_response_with_completion(self, obj: LibrarianResponse) -> Tuple[Dict[str, Any], bool]:
def encode_with_completion(self, obj: LibrarianResponse) -> Tuple[Dict[str, Any], bool]:
"""Returns (response_dict, is_final)"""
return self.from_pulsar(obj), obj.is_final
return self.encode(obj), obj.is_final

12
trustgraph-base/trustgraph/messaging/translators/metadata.py
View file

@ -10,7 +10,7 @@ class DocumentMetadataTranslator(Translator):
def __init__(self):
self.subgraph_translator = SubgraphTranslator()
def to_pulsar(self, data: Dict[str, Any]) -> DocumentMetadata:
def decode(self, data: Dict[str, Any]) -> DocumentMetadata:
metadata = data.get("metadata", [])
return DocumentMetadata(
id=data.get("id"),
@ -18,14 +18,14 @@ class DocumentMetadataTranslator(Translator):
kind=data.get("kind"),
title=data.get("title"),
comments=data.get("comments"),
metadata=self.subgraph_translator.to_pulsar(metadata) if metadata is not None else [],
metadata=self.subgraph_translator.decode(metadata) if metadata is not None else [],
user=data.get("user"),
tags=data.get("tags"),
parent_id=data.get("parent-id", ""),
document_type=data.get("document-type", "source"),
)
def from_pulsar(self, obj: DocumentMetadata) -> Dict[str, Any]:
def encode(self, obj: DocumentMetadata) -> Dict[str, Any]:
result = {}
if obj.id:
@ -39,7 +39,7 @@ class DocumentMetadataTranslator(Translator):
if obj.comments:
result["comments"] = obj.comments
if obj.metadata is not None:
result["metadata"] = self.subgraph_translator.from_pulsar(obj.metadata)
result["metadata"] = self.subgraph_translator.encode(obj.metadata)
if obj.user:
result["user"] = obj.user
if obj.tags is not None:
@ -55,7 +55,7 @@ class DocumentMetadataTranslator(Translator):
class ProcessingMetadataTranslator(Translator):
"""Translator for ProcessingMetadata schema objects"""
def to_pulsar(self, data: Dict[str, Any]) -> ProcessingMetadata:
def decode(self, data: Dict[str, Any]) -> ProcessingMetadata:
return ProcessingMetadata(
id=data.get("id"),
document_id=data.get("document-id"),
@ -66,7 +66,7 @@ class ProcessingMetadataTranslator(Translator):
tags=data.get("tags")
)
def from_pulsar(self, obj: ProcessingMetadata) -> Dict[str, Any]:
def encode(self, obj: ProcessingMetadata) -> Dict[str, Any]:
result = {}
if obj.id:

12
trustgraph-base/trustgraph/messaging/translators/nlp_query.py
View file

@ -6,13 +6,13 @@ from .base import MessageTranslator
class QuestionToStructuredQueryRequestTranslator(MessageTranslator):
"""Translator for QuestionToStructuredQueryRequest schema objects"""
def to_pulsar(self, data: Dict[str, Any]) -> QuestionToStructuredQueryRequest:
def decode(self, data: Dict[str, Any]) -> QuestionToStructuredQueryRequest:
return QuestionToStructuredQueryRequest(
question=data.get("question", ""),
max_results=data.get("max_results", 100)
)
def from_pulsar(self, obj: QuestionToStructuredQueryRequest) -> Dict[str, Any]:
def encode(self, obj: QuestionToStructuredQueryRequest) -> Dict[str, Any]:
return {
"question": obj.question,
"max_results": obj.max_results
@ -22,10 +22,10 @@ class QuestionToStructuredQueryRequestTranslator(MessageTranslator):
class QuestionToStructuredQueryResponseTranslator(MessageTranslator):
"""Translator for QuestionToStructuredQueryResponse schema objects"""
def to_pulsar(self, data: Dict[str, Any]) -> QuestionToStructuredQueryResponse:
def decode(self, data: Dict[str, Any]) -> QuestionToStructuredQueryResponse:
raise NotImplementedError("Response translation to Pulsar not typically needed")
def from_pulsar(self, obj: QuestionToStructuredQueryResponse) -> Dict[str, Any]:
def encode(self, obj: QuestionToStructuredQueryResponse) -> Dict[str, Any]:
result = {
"graphql_query": obj.graphql_query,
"variables": dict(obj.variables) if obj.variables else {},
@ -42,6 +42,6 @@ class QuestionToStructuredQueryResponseTranslator(MessageTranslator):
return result
def from_response_with_completion(self, obj: QuestionToStructuredQueryResponse) -> Tuple[Dict[str, Any], bool]:
def encode_with_completion(self, obj: QuestionToStructuredQueryResponse) -> Tuple[Dict[str, Any], bool]:
"""Returns (response_dict, is_final)"""
return self.from_pulsar(obj), True
return self.encode(obj), True

56
trustgraph-base/trustgraph/messaging/translators/primitives.py
View file

@ -17,7 +17,7 @@ class TermTranslator(Translator):
- "tr": triple (for TRIPLE type, nested)
"""
def to_pulsar(self, data: Dict[str, Any]) -> Term:
def decode(self, data: Dict[str, Any]) -> Term:
term_type = data.get("t", "")
if term_type == IRI:
@ -38,7 +38,7 @@ class TermTranslator(Translator):
# Nested triple - use TripleTranslator
triple_data = data.get("tr")
if triple_data:
triple = _triple_translator_to_pulsar(triple_data)
triple = _triple_translator_decode(triple_data)
else:
triple = None
return Term(type=TRIPLE, triple=triple)
@ -47,7 +47,7 @@ class TermTranslator(Translator):
# Unknown or empty type
return Term(type=term_type)
def from_pulsar(self, obj: Term) -> Dict[str, Any]:
def encode(self, obj: Term) -> Dict[str, Any]:
result: Dict[str, Any] = {"t": obj.type}
if obj.type == IRI:
@ -65,33 +65,33 @@ class TermTranslator(Translator):
elif obj.type == TRIPLE:
if obj.triple:
result["tr"] = _triple_translator_from_pulsar(obj.triple)
result["tr"] = _triple_translator_encode(obj.triple)
return result
# Module-level helper functions to avoid circular instantiation
def _triple_translator_to_pulsar(data: Dict[str, Any]) -> Triple:
def _triple_translator_decode(data: Dict[str, Any]) -> Triple:
term_translator = TermTranslator()
return Triple(
s=term_translator.to_pulsar(data["s"]) if data.get("s") else None,
p=term_translator.to_pulsar(data["p"]) if data.get("p") else None,
o=term_translator.to_pulsar(data["o"]) if data.get("o") else None,
s=term_translator.decode(data["s"]) if data.get("s") else None,
p=term_translator.decode(data["p"]) if data.get("p") else None,
o=term_translator.decode(data["o"]) if data.get("o") else None,
g=data.get("g"),
)
def _triple_translator_from_pulsar(obj: Triple) -> Dict[str, Any]:
def _triple_translator_encode(obj: Triple) -> Dict[str, Any]:
"""Convert Triple object to wire format dict."""
term_translator = TermTranslator()
result: Dict[str, Any] = {}
if obj.s:
result["s"] = term_translator.from_pulsar(obj.s)
result["s"] = term_translator.encode(obj.s)
if obj.p:
result["p"] = term_translator.from_pulsar(obj.p)
result["p"] = term_translator.encode(obj.p)
if obj.o:
result["o"] = term_translator.from_pulsar(obj.o)
result["o"] = term_translator.encode(obj.o)
if obj.g:
result["g"] = obj.g
@ -104,23 +104,23 @@ class TripleTranslator(Translator):
def __init__(self):
self.term_translator = TermTranslator()
def to_pulsar(self, data: Dict[str, Any]) -> Triple:
def decode(self, data: Dict[str, Any]) -> Triple:
return Triple(
s=self.term_translator.to_pulsar(data["s"]) if data.get("s") else None,
p=self.term_translator.to_pulsar(data["p"]) if data.get("p") else None,
o=self.term_translator.to_pulsar(data["o"]) if data.get("o") else None,
s=self.term_translator.decode(data["s"]) if data.get("s") else None,
p=self.term_translator.decode(data["p"]) if data.get("p") else None,
o=self.term_translator.decode(data["o"]) if data.get("o") else None,
g=data.get("g"),
)
def from_pulsar(self, obj: Triple) -> Dict[str, Any]:
def encode(self, obj: Triple) -> Dict[str, Any]:
result: Dict[str, Any] = {}
if obj.s:
result["s"] = self.term_translator.from_pulsar(obj.s)
result["s"] = self.term_translator.encode(obj.s)
if obj.p:
result["p"] = self.term_translator.from_pulsar(obj.p)
result["p"] = self.term_translator.encode(obj.p)
if obj.o:
result["o"] = self.term_translator.from_pulsar(obj.o)
result["o"] = self.term_translator.encode(obj.o)
if obj.g:
result["g"] = obj.g
@ -137,17 +137,17 @@ class SubgraphTranslator(Translator):
def __init__(self):
self.triple_translator = TripleTranslator()
def to_pulsar(self, data: List[Dict[str, Any]]) -> List[Triple]:
return [self.triple_translator.to_pulsar(t) for t in data]
def decode(self, data: List[Dict[str, Any]]) -> List[Triple]:
return [self.triple_translator.decode(t) for t in data]
def from_pulsar(self, obj: List[Triple]) -> List[Dict[str, Any]]:
return [self.triple_translator.from_pulsar(t) for t in obj]
def encode(self, obj: List[Triple]) -> List[Dict[str, Any]]:
return [self.triple_translator.encode(t) for t in obj]
class RowSchemaTranslator(Translator):
"""Translator for RowSchema objects"""
def to_pulsar(self, data: Dict[str, Any]) -> RowSchema:
def decode(self, data: Dict[str, Any]) -> RowSchema:
"""Convert dict to RowSchema Pulsar object"""
fields = []
for field_data in data.get("fields", []):
@ -169,7 +169,7 @@ class RowSchemaTranslator(Translator):
fields=fields
)
def from_pulsar(self, obj: RowSchema) -> Dict[str, Any]:
def encode(self, obj: RowSchema) -> Dict[str, Any]:
"""Convert RowSchema Pulsar object to JSON-serializable dictionary"""
result = {
"name": obj.name,
@ -200,7 +200,7 @@ class RowSchemaTranslator(Translator):
class FieldTranslator(Translator):
"""Translator for Field objects"""
def to_pulsar(self, data: Dict[str, Any]) -> Field:
def decode(self, data: Dict[str, Any]) -> Field:
"""Convert dict to Field Pulsar object"""
return Field(
name=data.get("name", ""),
@ -213,7 +213,7 @@ class FieldTranslator(Translator):
enum_values=data.get("enum_values", [])
)
def from_pulsar(self, obj: Field) -> Dict[str, Any]:
def encode(self, obj: Field) -> Dict[str, Any]:
"""Convert Field Pulsar object to JSON-serializable dictionary"""
result = {
"name": obj.name,

12
trustgraph-base/trustgraph/messaging/translators/prompt.py
View file

@ -7,7 +7,7 @@ from .base import MessageTranslator
class PromptRequestTranslator(MessageTranslator):
"""Translator for PromptRequest schema objects"""
def to_pulsar(self, data: Dict[str, Any]) -> PromptRequest:
def decode(self, data: Dict[str, Any]) -> PromptRequest:
# Handle both "terms" and "variables" input keys
terms = data.get("terms", {})
if "variables" in data:
@ -23,7 +23,7 @@ class PromptRequestTranslator(MessageTranslator):
streaming=data.get("streaming", False)
)
def from_pulsar(self, obj: PromptRequest) -> Dict[str, Any]:
def encode(self, obj: PromptRequest) -> Dict[str, Any]:
result = {}
if obj.id:
@ -37,10 +37,10 @@ class PromptRequestTranslator(MessageTranslator):
class PromptResponseTranslator(MessageTranslator):
"""Translator for PromptResponse schema objects"""
def to_pulsar(self, data: Dict[str, Any]) -> PromptResponse:
def decode(self, data: Dict[str, Any]) -> PromptResponse:
raise NotImplementedError("Response translation to Pulsar not typically needed")
def from_pulsar(self, obj: PromptResponse) -> Dict[str, Any]:
def encode(self, obj: PromptResponse) -> Dict[str, Any]:
result = {}
# Include text field if present (even if empty string)
@ -55,8 +55,8 @@ class PromptResponseTranslator(MessageTranslator):
return result
def from_response_with_completion(self, obj: PromptResponse) -> Tuple[Dict[str, Any], bool]:
def encode_with_completion(self, obj: PromptResponse) -> Tuple[Dict[str, Any], bool]:
"""Returns (response_dict, is_final)"""
# Check end_of_stream field to determine if this is the final message
is_final = getattr(obj, 'end_of_stream', True)
return self.from_pulsar(obj), is_final
return self.encode(obj), is_final

24
trustgraph-base/trustgraph/messaging/translators/retrieval.py
View file

@ -6,7 +6,7 @@ from .base import MessageTranslator
class DocumentRagRequestTranslator(MessageTranslator):
"""Translator for DocumentRagQuery schema objects"""
def to_pulsar(self, data: Dict[str, Any]) -> DocumentRagQuery:
def decode(self, data: Dict[str, Any]) -> DocumentRagQuery:
return DocumentRagQuery(
query=data["query"],
user=data.get("user", "trustgraph"),
@ -15,7 +15,7 @@ class DocumentRagRequestTranslator(MessageTranslator):
streaming=data.get("streaming", False)
)
def from_pulsar(self, obj: DocumentRagQuery) -> Dict[str, Any]:
def encode(self, obj: DocumentRagQuery) -> Dict[str, Any]:
return {
"query": obj.query,
"user": obj.user,
@ -28,10 +28,10 @@ class DocumentRagRequestTranslator(MessageTranslator):
class DocumentRagResponseTranslator(MessageTranslator):
"""Translator for DocumentRagResponse schema objects"""
def to_pulsar(self, data: Dict[str, Any]) -> DocumentRagResponse:
def decode(self, data: Dict[str, Any]) -> DocumentRagResponse:
raise NotImplementedError("Response translation to Pulsar not typically needed")
def from_pulsar(self, obj: DocumentRagResponse) -> Dict[str, Any]:
def encode(self, obj: DocumentRagResponse) -> Dict[str, Any]:
result = {}
# Include message_type for distinguishing chunk vs explain messages
@ -65,17 +65,17 @@ class DocumentRagResponseTranslator(MessageTranslator):
return result
def from_response_with_completion(self, obj: DocumentRagResponse) -> Tuple[Dict[str, Any], bool]:
def encode_with_completion(self, obj: DocumentRagResponse) -> Tuple[Dict[str, Any], bool]:
"""Returns (response_dict, is_final)"""
# Session is complete when end_of_session is True
is_final = getattr(obj, 'end_of_session', False)
return self.from_pulsar(obj), is_final
return self.encode(obj), is_final
class GraphRagRequestTranslator(MessageTranslator):
"""Translator for GraphRagQuery schema objects"""
def to_pulsar(self, data: Dict[str, Any]) -> GraphRagQuery:
def decode(self, data: Dict[str, Any]) -> GraphRagQuery:
return GraphRagQuery(
query=data["query"],
user=data.get("user", "trustgraph"),
@ -89,7 +89,7 @@ class GraphRagRequestTranslator(MessageTranslator):
streaming=data.get("streaming", False)
)
def from_pulsar(self, obj: GraphRagQuery) -> Dict[str, Any]:
def encode(self, obj: GraphRagQuery) -> Dict[str, Any]:
return {
"query": obj.query,
"user": obj.user,
@ -107,10 +107,10 @@ class GraphRagRequestTranslator(MessageTranslator):
class GraphRagResponseTranslator(MessageTranslator):
"""Translator for GraphRagResponse schema objects"""
def to_pulsar(self, data: Dict[str, Any]) -> GraphRagResponse:
def decode(self, data: Dict[str, Any]) -> GraphRagResponse:
raise NotImplementedError("Response translation to Pulsar not typically needed")
def from_pulsar(self, obj: GraphRagResponse) -> Dict[str, Any]:
def encode(self, obj: GraphRagResponse) -> Dict[str, Any]:
result = {}
# Include message_type
@ -144,8 +144,8 @@ class GraphRagResponseTranslator(MessageTranslator):
return result
def from_response_with_completion(self, obj: GraphRagResponse) -> Tuple[Dict[str, Any], bool]:
def encode_with_completion(self, obj: GraphRagResponse) -> Tuple[Dict[str, Any], bool]:
"""Returns (response_dict, is_final)"""
# Session is complete when end_of_session is True
is_final = getattr(obj, 'end_of_session', False)
return self.from_pulsar(obj), is_final
return self.encode(obj), is_final

12
trustgraph-base/trustgraph/messaging/translators/rows_query.py
View file

@ -7,7 +7,7 @@ import json
class RowsQueryRequestTranslator(MessageTranslator):
"""Translator for RowsQueryRequest schema objects"""
def to_pulsar(self, data: Dict[str, Any]) -> RowsQueryRequest:
def decode(self, data: Dict[str, Any]) -> RowsQueryRequest:
return RowsQueryRequest(
user=data.get("user", "trustgraph"),
collection=data.get("collection", "default"),
@ -16,7 +16,7 @@ class RowsQueryRequestTranslator(MessageTranslator):
operation_name=data.get("operation_name", None)
)
def from_pulsar(self, obj: RowsQueryRequest) -> Dict[str, Any]:
def encode(self, obj: RowsQueryRequest) -> Dict[str, Any]:
result = {
"user": obj.user,
"collection": obj.collection,
@ -33,10 +33,10 @@ class RowsQueryRequestTranslator(MessageTranslator):
class RowsQueryResponseTranslator(MessageTranslator):
"""Translator for RowsQueryResponse schema objects"""
def to_pulsar(self, data: Dict[str, Any]) -> RowsQueryResponse:
def decode(self, data: Dict[str, Any]) -> RowsQueryResponse:
raise NotImplementedError("Response translation to Pulsar not typically needed")
def from_pulsar(self, obj: RowsQueryResponse) -> Dict[str, Any]:
def encode(self, obj: RowsQueryResponse) -> Dict[str, Any]:
result = {}
# Handle GraphQL response data
@ -74,6 +74,6 @@ class RowsQueryResponseTranslator(MessageTranslator):
return result
def from_response_with_completion(self, obj: RowsQueryResponse) -> Tuple[Dict[str, Any], bool]:
def encode_with_completion(self, obj: RowsQueryResponse) -> Tuple[Dict[str, Any], bool]:
"""Returns (response_dict, is_final)"""
return self.from_pulsar(obj), True
return self.encode(obj), True

12
trustgraph-base/trustgraph/messaging/translators/structured_query.py
View file

@ -7,14 +7,14 @@ import json
class StructuredQueryRequestTranslator(MessageTranslator):
"""Translator for StructuredQueryRequest schema objects"""
def to_pulsar(self, data: Dict[str, Any]) -> StructuredQueryRequest:
def decode(self, data: Dict[str, Any]) -> StructuredQueryRequest:
return StructuredQueryRequest(
question=data.get("question", ""),
user=data.get("user", "trustgraph"), # Default fallback
collection=data.get("collection", "default") # Default fallback
)
def from_pulsar(self, obj: StructuredQueryRequest) -> Dict[str, Any]:
def encode(self, obj: StructuredQueryRequest) -> Dict[str, Any]:
return {
"question": obj.question,
"user": obj.user,
@ -25,10 +25,10 @@ class StructuredQueryRequestTranslator(MessageTranslator):
class StructuredQueryResponseTranslator(MessageTranslator):
"""Translator for StructuredQueryResponse schema objects"""
def to_pulsar(self, data: Dict[str, Any]) -> StructuredQueryResponse:
def decode(self, data: Dict[str, Any]) -> StructuredQueryResponse:
raise NotImplementedError("Response translation to Pulsar not typically needed")
def from_pulsar(self, obj: StructuredQueryResponse) -> Dict[str, Any]:
def encode(self, obj: StructuredQueryResponse) -> Dict[str, Any]:
result = {}
# Handle structured query response data
@ -55,6 +55,6 @@ class StructuredQueryResponseTranslator(MessageTranslator):
return result
def from_response_with_completion(self, obj: StructuredQueryResponse) -> Tuple[Dict[str, Any], bool]:
def encode_with_completion(self, obj: StructuredQueryResponse) -> Tuple[Dict[str, Any], bool]:
"""Returns (response_dict, is_final)"""
return self.from_pulsar(obj), True
return self.encode(obj), True

12
trustgraph-base/trustgraph/messaging/translators/text_completion.py
View file

@ -6,14 +6,14 @@ from .base import MessageTranslator
class TextCompletionRequestTranslator(MessageTranslator):
"""Translator for TextCompletionRequest schema objects"""
def to_pulsar(self, data: Dict[str, Any]) -> TextCompletionRequest:
def decode(self, data: Dict[str, Any]) -> TextCompletionRequest:
return TextCompletionRequest(
system=data["system"],
prompt=data["prompt"],
streaming=data.get("streaming", False)
)
def from_pulsar(self, obj: TextCompletionRequest) -> Dict[str, Any]:
def encode(self, obj: TextCompletionRequest) -> Dict[str, Any]:
return {
"system": obj.system,
"prompt": obj.prompt
@ -23,10 +23,10 @@ class TextCompletionRequestTranslator(MessageTranslator):
class TextCompletionResponseTranslator(MessageTranslator):
"""Translator for TextCompletionResponse schema objects"""
def to_pulsar(self, data: Dict[str, Any]) -> TextCompletionResponse:
def decode(self, data: Dict[str, Any]) -> TextCompletionResponse:
raise NotImplementedError("Response translation to Pulsar not typically needed")
def from_pulsar(self, obj: TextCompletionResponse) -> Dict[str, Any]:
def encode(self, obj: TextCompletionResponse) -> Dict[str, Any]:
result = {"response": obj.response}
if obj.in_token:
@ -41,8 +41,8 @@ class TextCompletionResponseTranslator(MessageTranslator):
return result
def from_response_with_completion(self, obj: TextCompletionResponse) -> Tuple[Dict[str, Any], bool]:
def encode_with_completion(self, obj: TextCompletionResponse) -> Tuple[Dict[str, Any], bool]:
"""Returns (response_dict, is_final)"""
# Check end_of_stream field to determine if this is the final message
is_final = getattr(obj, 'end_of_stream', True)
return self.from_pulsar(obj), is_final
return self.encode(obj), is_final

12
trustgraph-base/trustgraph/messaging/translators/tool.py
View file

@ -6,7 +6,7 @@ from .base import MessageTranslator
class ToolRequestTranslator(MessageTranslator):
"""Translator for ToolRequest schema objects"""
def to_pulsar(self, data: Dict[str, Any]) -> ToolRequest:
def decode(self, data: Dict[str, Any]) -> ToolRequest:
# Handle both "name" and "parameters" input keys
name = data.get("name", "")
if "parameters" in data:
@ -19,7 +19,7 @@ class ToolRequestTranslator(MessageTranslator):
parameters = parameters,
)
def from_pulsar(self, obj: ToolRequest) -> Dict[str, Any]:
def encode(self, obj: ToolRequest) -> Dict[str, Any]:
result = {}
if obj.name:
@ -32,10 +32,10 @@ class ToolRequestTranslator(MessageTranslator):
class ToolResponseTranslator(MessageTranslator):
"""Translator for ToolResponse schema objects"""
def to_pulsar(self, data: Dict[str, Any]) -> ToolResponse:
def decode(self, data: Dict[str, Any]) -> ToolResponse:
raise NotImplementedError("Response translation to Pulsar not typically needed")
def from_pulsar(self, obj: ToolResponse) -> Dict[str, Any]:
def encode(self, obj: ToolResponse) -> Dict[str, Any]:
result = {}
@ -46,6 +46,6 @@ class ToolResponseTranslator(MessageTranslator):
return result
def from_response_with_completion(self, obj: ToolResponse) -> Tuple[Dict[str, Any], bool]:
def encode_with_completion(self, obj: ToolResponse) -> Tuple[Dict[str, Any], bool]:
"""Returns (response_dict, is_final)"""
return self.from_pulsar(obj), True
return self.encode(obj), True

26
trustgraph-base/trustgraph/messaging/translators/triples.py
View file

@ -10,10 +10,10 @@ class TriplesQueryRequestTranslator(MessageTranslator):
def __init__(self):
self.value_translator = ValueTranslator()
def to_pulsar(self, data: Dict[str, Any]) -> TriplesQueryRequest:
s = self.value_translator.to_pulsar(data["s"]) if "s" in data else None
p = self.value_translator.to_pulsar(data["p"]) if "p" in data else None
o = self.value_translator.to_pulsar(data["o"]) if "o" in data else None
def decode(self, data: Dict[str, Any]) -> TriplesQueryRequest:
s = self.value_translator.decode(data["s"]) if "s" in data else None
p = self.value_translator.decode(data["p"]) if "p" in data else None
o = self.value_translator.decode(data["o"]) if "o" in data else None
g = data.get("g") # None=default graph, "*"=all graphs
return TriplesQueryRequest(
@ -28,7 +28,7 @@ class TriplesQueryRequestTranslator(MessageTranslator):
batch_size=int(data.get("batch-size", 20)),
)
def from_pulsar(self, obj: TriplesQueryRequest) -> Dict[str, Any]:
def encode(self, obj: TriplesQueryRequest) -> Dict[str, Any]:
result = {
"limit": obj.limit,
"user": obj.user,
@ -38,11 +38,11 @@ class TriplesQueryRequestTranslator(MessageTranslator):
}
if obj.s:
result["s"] = self.value_translator.from_pulsar(obj.s)
result["s"] = self.value_translator.encode(obj.s)
if obj.p:
result["p"] = self.value_translator.from_pulsar(obj.p)
result["p"] = self.value_translator.encode(obj.p)
if obj.o:
result["o"] = self.value_translator.from_pulsar(obj.o)
result["o"] = self.value_translator.encode(obj.o)
if obj.g is not None:
result["g"] = obj.g
@ -55,14 +55,14 @@ class TriplesQueryResponseTranslator(MessageTranslator):
def __init__(self):
self.subgraph_translator = SubgraphTranslator()
def to_pulsar(self, data: Dict[str, Any]) -> TriplesQueryResponse:
def decode(self, data: Dict[str, Any]) -> TriplesQueryResponse:
raise NotImplementedError("Response translation to Pulsar not typically needed")
def from_pulsar(self, obj: TriplesQueryResponse) -> Dict[str, Any]:
def encode(self, obj: TriplesQueryResponse) -> Dict[str, Any]:
return {
"response": self.subgraph_translator.from_pulsar(obj.triples)
"response": self.subgraph_translator.encode(obj.triples)
}
def from_response_with_completion(self, obj: TriplesQueryResponse) -> Tuple[Dict[str, Any], bool]:
def encode_with_completion(self, obj: TriplesQueryResponse) -> Tuple[Dict[str, Any], bool]:
"""Returns (response_dict, is_final)"""
return self.from_pulsar(obj), obj.is_final
return self.encode(obj), obj.is_final

31
trustgraph-base/trustgraph/schema/core/topic.py
View file

@ -1,23 +1,26 @@
def topic(queue_name, qos='q1', tenant='tg', namespace='flow'):
def queue(topic, cls='flow', topicspace='tg'):
"""
Create a generic topic identifier that can be mapped by backends.
Create a queue identifier in CLASS:TOPICSPACE:TOPIC format.
Args:
queue_name: The queue/topic name
qos: Quality of service
- 'q0' = best-effort (no ack)
- 'q1' = at-least-once (ack required)
- 'q2' = exactly-once (two-phase ack)
tenant: Tenant identifier for multi-tenancy
namespace: Namespace within tenant
topic: The logical queue name (e.g. 'config', 'librarian')
cls: Queue class determining operational characteristics:
- 'flow' = persistent processing pipeline queue
- 'request' = non-persistent, short TTL request queue
- 'response' = non-persistent, short TTL response queue
- 'state' = persistent, last-value state broadcast
topicspace: Deployment isolation prefix (default: 'tg')
Returns:
Generic topic string: qos/tenant/namespace/queue_name
Queue identifier string: cls:topicspace:topic
Examples:
topic('my-queue') # q1/tg/flow/my-queue
topic('config', qos='q2', namespace='config') # q2/tg/config/config
queue('text-completion-request')
# flow:tg:text-completion-request
queue('config', cls='request')
# request:tg:config
queue('config', cls='state')
# state:tg:config
"""
return f"{qos}/{tenant}/{namespace}/{queue_name}"
return f"{cls}:{topicspace}:{topic}"

1
trustgraph-base/trustgraph/schema/knowledge/document.py
View file

@ -1,7 +1,6 @@
from dataclasses import dataclass
from ..core.metadata import Metadata
from ..core.topic import topic
############################################################################

1
trustgraph-base/trustgraph/schema/knowledge/embeddings.py
View file

@ -2,7 +2,6 @@ from dataclasses import dataclass, field
from ..core.metadata import Metadata
from ..core.primitives import Term, RowSchema
from ..core.topic import topic
############################################################################

1
trustgraph-base/trustgraph/schema/knowledge/graph.py
View file

@ -2,7 +2,6 @@ from dataclasses import dataclass, field
from ..core.primitives import Term, Triple
from ..core.metadata import Metadata
from ..core.topic import topic
############################################################################

10
trustgraph-base/trustgraph/schema/knowledge/knowledge.py
View file

@ -1,6 +1,6 @@
from dataclasses import dataclass, field
from ..core.primitives import Triple, Error
from ..core.topic import topic
from ..core.topic import queue
from ..core.metadata import Metadata
from .document import Document, TextDocument
from .graph import Triples
@ -52,9 +52,5 @@ class KnowledgeResponse:
triples: Triples | None = None
graph_embeddings: GraphEmbeddings | None = None
knowledge_request_queue = topic(
'knowledge', qos='q0', namespace='request'
)
knowledge_response_queue = topic(
'knowledge', qos='q0', namespace='response',
)
knowledge_request_queue = queue('knowledge', cls='request')
knowledge_response_queue = queue('knowledge', cls='response')

1
trustgraph-base/trustgraph/schema/knowledge/nlp.py
View file

@ -1,6 +1,5 @@
from dataclasses import dataclass
from ..core.topic import topic
############################################################################

1
trustgraph-base/trustgraph/schema/knowledge/object.py
View file

@ -1,7 +1,6 @@
from dataclasses import dataclass, field
from ..core.metadata import Metadata
from ..core.topic import topic
############################################################################

1
trustgraph-base/trustgraph/schema/knowledge/rows.py
View file

@ -2,7 +2,6 @@ from dataclasses import dataclass, field
from ..core.metadata import Metadata
from ..core.primitives import RowSchema
from ..core.topic import topic
############################################################################

1
trustgraph-base/trustgraph/schema/knowledge/structured.py
View file

@ -1,7 +1,6 @@
from dataclasses import dataclass, field
from ..core.metadata import Metadata
from ..core.topic import topic
############################################################################

1
trustgraph-base/trustgraph/schema/services/agent.py
View file

@ -2,7 +2,6 @@
from dataclasses import dataclass, field
from typing import Optional
from ..core.topic import topic
from ..core.primitives import Error
############################################################################

10
trustgraph-base/trustgraph/schema/services/collection.py
View file

@ -2,7 +2,7 @@ from dataclasses import dataclass, field
from datetime import datetime
from ..core.primitives import Error
from ..core.topic import topic
from ..core.topic import queue
############################################################################
@ -50,10 +50,6 @@ class CollectionManagementResponse:
# Topics
collection_request_queue = topic(
'collection', qos='q0', namespace='request'
)
collection_response_queue = topic(
'collection', qos='q0', namespace='response'
)
collection_request_queue = queue('collection', cls='request')
collection_response_queue = queue('collection', cls='response')

14
trustgraph-base/trustgraph/schema/services/config.py
View file

@ -1,7 +1,7 @@
from dataclasses import dataclass, field
from ..core.topic import topic
from ..core.topic import queue
from ..core.primitives import Error
############################################################################
@ -60,15 +60,9 @@ class ConfigPush:
version: int = 0
config: dict[str, dict[str, str]] = field(default_factory=dict)
config_request_queue = topic(
'config', qos='q0', namespace='request'
)
config_response_queue = topic(
'config', qos='q0', namespace='response'
)
config_push_queue = topic(
'config', qos='q2', namespace='config'
)
config_request_queue = queue('config', cls='request')
config_response_queue = queue('config', cls='response')
config_push_queue = queue('config', cls='state')
############################################################################

10
trustgraph-base/trustgraph/schema/services/flow.py
View file

@ -1,7 +1,7 @@
from dataclasses import dataclass, field
from ..core.topic import topic
from ..core.topic import queue
from ..core.primitives import Error
############################################################################
@ -61,12 +61,8 @@ class FlowResponse:
# Everything
error: Error | None = None
flow_request_queue = topic(
'flow', qos='q0', namespace='request'
)
flow_response_queue = topic(
'flow', qos='q0', namespace='response'
)
flow_request_queue = queue('flow', cls='request')
flow_response_queue = queue('flow', cls='response')
############################################################################

10
trustgraph-base/trustgraph/schema/services/library.py
View file

@ -1,6 +1,6 @@
from dataclasses import dataclass, field
from ..core.primitives import Triple, Error
from ..core.topic import topic
from ..core.topic import queue
from ..core.metadata import Metadata
# Note: Document imports will be updated after knowledge schemas are converted
@ -220,9 +220,5 @@ class LibrarianResponse:
# FIXME: Is this right? Using persistence on librarian so that
# message chunking works
librarian_request_queue = topic(
'librarian', qos='q1', namespace='request'
)
librarian_response_queue = topic(
'librarian', qos='q1', namespace='response',
)
librarian_request_queue = queue('librarian-request', cls='flow')
librarian_response_queue = queue('librarian-response', cls='flow')

1
trustgraph-base/trustgraph/schema/services/llm.py
View file

@ -1,7 +1,6 @@
from dataclasses import dataclass, field
from ..core.topic import topic
from ..core.primitives import Error
############################################################################

1
trustgraph-base/trustgraph/schema/services/lookup.py
View file

@ -1,7 +1,6 @@
from dataclasses import dataclass
from ..core.primitives import Error, Term, Triple
from ..core.topic import topic
from ..core.metadata import Metadata
############################################################################

1
trustgraph-base/trustgraph/schema/services/nlp_query.py
View file

@ -1,7 +1,6 @@
from dataclasses import dataclass, field
from ..core.primitives import Error
from ..core.topic import topic
############################################################################

1
trustgraph-base/trustgraph/schema/services/prompt.py
View file

@ -1,7 +1,6 @@
from dataclasses import dataclass, field
from ..core.primitives import Error
from ..core.topic import topic
############################################################################

18
trustgraph-base/trustgraph/schema/services/query.py
View file

@ -1,7 +1,7 @@
from dataclasses import dataclass, field
from ..core.primitives import Error, Term, Triple
from ..core.topic import topic
from ..core.topic import queue
############################################################################
@ -69,12 +69,8 @@ class DocumentEmbeddingsResponse:
error: Error | None = None
chunks: list[ChunkMatch] = field(default_factory=list)
document_embeddings_request_queue = topic(
"document-embeddings-request", qos='q0', tenant='trustgraph', namespace='flow'
)
document_embeddings_response_queue = topic(
"document-embeddings-response", qos='q0', tenant='trustgraph', namespace='flow'
)
document_embeddings_request_queue = queue('document-embeddings', cls='request')
document_embeddings_response_queue = queue('document-embeddings', cls='response')
############################################################################
@ -104,9 +100,5 @@ class RowEmbeddingsResponse:
error: Error | None = None
matches: list[RowIndexMatch] = field(default_factory=list)
row_embeddings_request_queue = topic(
"row-embeddings-request", qos='q0', tenant='trustgraph', namespace='flow'
)
row_embeddings_response_queue = topic(
"row-embeddings-response", qos='q0', tenant='trustgraph', namespace='flow'
)
row_embeddings_request_queue = queue('row-embeddings', cls='request')
row_embeddings_response_queue = queue('row-embeddings', cls='response')

1
trustgraph-base/trustgraph/schema/services/retrieval.py
View file

@ -1,5 +1,4 @@
from dataclasses import dataclass
from ..core.topic import topic
from ..core.primitives import Error, Term
############################################################################

1
trustgraph-base/trustgraph/schema/services/rows_query.py
View file

@ -2,7 +2,6 @@ from dataclasses import dataclass, field
from typing import Optional
from ..core.primitives import Error
from ..core.topic import topic
############################################################################

1
trustgraph-base/trustgraph/schema/services/structured_query.py
View file

@ -1,7 +1,6 @@
from dataclasses import dataclass, field
from ..core.primitives import Error
from ..core.topic import topic
############################################################################