mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-07-05 11:22:11 +02:00
Pub/sub abstraction: decouple from Pulsar (#751)
Remove Pulsar-specific concepts from application code so that the pub/sub backend is swappable via configuration. Rename translators: - to_pulsar/from_pulsar → decode/encode across all translator classes, dispatch handlers, and tests (55+ files) - from_response_with_completion → encode_with_completion - Remove pulsar.schema.Record from translator base class Queue naming (CLASS:TOPICSPACE:TOPIC): - Replace topic() helper with queue() using new format: flow:tg:name, request:tg:name, response:tg:name, state:tg:name - Queue class implies persistence/TTL (no QoS in names) - Update Pulsar backend map_topic() to parse new format - Librarian queues use flow class (persistent, for chunking) - Config push uses state class (persistent, last-value) - Remove 15 dead topic imports from schema files - Update init_trustgraph.py namespace: config → state Confine Pulsar to pulsar_backend.py: - Delete legacy PulsarClient class from pubsub.py - Move add_args to add_pubsub_args() with standalone flag for CLI tools (defaults to localhost) - PulsarBackendConsumer.receive() catches _pulsar.Timeout, raises standard TimeoutError - Remove Pulsar imports from: async_processor, flow_processor, log_level, all 11 client files, 4 storage writers, gateway service, gateway config receiver - Remove log_level/LoggerLevel from client API - Rewrite tg-monitor-prompts to use backend abstraction - Update tg-dump-queues to use add_pubsub_args Also: pubsub-abstraction.md tech spec covering problem statement, design goals, as-is requirements, candidate broker assessment, approach, and implementation order.
This commit is contained in:
parent
dbf8daa74a
commit
4fb0b4d8e8
106 changed files with 1269 additions and 788 deletions
|
|
@ -2,7 +2,6 @@
|
|||
from dataclasses import dataclass, field
|
||||
from typing import Optional
|
||||
|
||||
from ..core.topic import topic
|
||||
from ..core.primitives import Error
|
||||
|
||||
############################################################################
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ from dataclasses import dataclass, field
|
|||
from datetime import datetime
|
||||
|
||||
from ..core.primitives import Error
|
||||
from ..core.topic import topic
|
||||
from ..core.topic import queue
|
||||
|
||||
############################################################################
|
||||
|
||||
|
|
@ -50,10 +50,6 @@ class CollectionManagementResponse:
|
|||
|
||||
# Topics
|
||||
|
||||
collection_request_queue = topic(
|
||||
'collection', qos='q0', namespace='request'
|
||||
)
|
||||
collection_response_queue = topic(
|
||||
'collection', qos='q0', namespace='response'
|
||||
)
|
||||
collection_request_queue = queue('collection', cls='request')
|
||||
collection_response_queue = queue('collection', cls='response')
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from ..core.topic import topic
|
||||
from ..core.topic import queue
|
||||
from ..core.primitives import Error
|
||||
|
||||
############################################################################
|
||||
|
|
@ -60,15 +60,9 @@ class ConfigPush:
|
|||
version: int = 0
|
||||
config: dict[str, dict[str, str]] = field(default_factory=dict)
|
||||
|
||||
config_request_queue = topic(
|
||||
'config', qos='q0', namespace='request'
|
||||
)
|
||||
config_response_queue = topic(
|
||||
'config', qos='q0', namespace='response'
|
||||
)
|
||||
config_push_queue = topic(
|
||||
'config', qos='q2', namespace='config'
|
||||
)
|
||||
config_request_queue = queue('config', cls='request')
|
||||
config_response_queue = queue('config', cls='response')
|
||||
config_push_queue = queue('config', cls='state')
|
||||
|
||||
############################################################################
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from ..core.topic import topic
|
||||
from ..core.topic import queue
|
||||
from ..core.primitives import Error
|
||||
|
||||
############################################################################
|
||||
|
|
@ -61,12 +61,8 @@ class FlowResponse:
|
|||
# Everything
|
||||
error: Error | None = None
|
||||
|
||||
flow_request_queue = topic(
|
||||
'flow', qos='q0', namespace='request'
|
||||
)
|
||||
flow_response_queue = topic(
|
||||
'flow', qos='q0', namespace='response'
|
||||
)
|
||||
flow_request_queue = queue('flow', cls='request')
|
||||
flow_response_queue = queue('flow', cls='response')
|
||||
|
||||
############################################################################
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from dataclasses import dataclass, field
|
||||
from ..core.primitives import Triple, Error
|
||||
from ..core.topic import topic
|
||||
from ..core.topic import queue
|
||||
from ..core.metadata import Metadata
|
||||
# Note: Document imports will be updated after knowledge schemas are converted
|
||||
|
||||
|
|
@ -220,9 +220,5 @@ class LibrarianResponse:
|
|||
# FIXME: Is this right? Using persistence on librarian so that
|
||||
# message chunking works
|
||||
|
||||
librarian_request_queue = topic(
|
||||
'librarian', qos='q1', namespace='request'
|
||||
)
|
||||
librarian_response_queue = topic(
|
||||
'librarian', qos='q1', namespace='response',
|
||||
)
|
||||
librarian_request_queue = queue('librarian-request', cls='flow')
|
||||
librarian_response_queue = queue('librarian-response', cls='flow')
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from ..core.topic import topic
|
||||
from ..core.primitives import Error
|
||||
|
||||
############################################################################
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
from dataclasses import dataclass
|
||||
|
||||
from ..core.primitives import Error, Term, Triple
|
||||
from ..core.topic import topic
|
||||
from ..core.metadata import Metadata
|
||||
|
||||
############################################################################
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
from dataclasses import dataclass, field
|
||||
|
||||
from ..core.primitives import Error
|
||||
from ..core.topic import topic
|
||||
|
||||
############################################################################
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
from dataclasses import dataclass, field
|
||||
|
||||
from ..core.primitives import Error
|
||||
from ..core.topic import topic
|
||||
|
||||
############################################################################
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
from dataclasses import dataclass, field
|
||||
|
||||
from ..core.primitives import Error, Term, Triple
|
||||
from ..core.topic import topic
|
||||
from ..core.topic import queue
|
||||
|
||||
############################################################################
|
||||
|
||||
|
|
@ -69,12 +69,8 @@ class DocumentEmbeddingsResponse:
|
|||
error: Error | None = None
|
||||
chunks: list[ChunkMatch] = field(default_factory=list)
|
||||
|
||||
document_embeddings_request_queue = topic(
|
||||
"document-embeddings-request", qos='q0', tenant='trustgraph', namespace='flow'
|
||||
)
|
||||
document_embeddings_response_queue = topic(
|
||||
"document-embeddings-response", qos='q0', tenant='trustgraph', namespace='flow'
|
||||
)
|
||||
document_embeddings_request_queue = queue('document-embeddings', cls='request')
|
||||
document_embeddings_response_queue = queue('document-embeddings', cls='response')
|
||||
|
||||
############################################################################
|
||||
|
||||
|
|
@ -104,9 +100,5 @@ class RowEmbeddingsResponse:
|
|||
error: Error | None = None
|
||||
matches: list[RowIndexMatch] = field(default_factory=list)
|
||||
|
||||
row_embeddings_request_queue = topic(
|
||||
"row-embeddings-request", qos='q0', tenant='trustgraph', namespace='flow'
|
||||
)
|
||||
row_embeddings_response_queue = topic(
|
||||
"row-embeddings-response", qos='q0', tenant='trustgraph', namespace='flow'
|
||||
)
|
||||
row_embeddings_request_queue = queue('row-embeddings', cls='request')
|
||||
row_embeddings_response_queue = queue('row-embeddings', cls='response')
|
||||
|
|
@ -1,5 +1,4 @@
|
|||
from dataclasses import dataclass
|
||||
from ..core.topic import topic
|
||||
from ..core.primitives import Error, Term
|
||||
|
||||
############################################################################
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@ from dataclasses import dataclass, field
|
|||
from typing import Optional
|
||||
|
||||
from ..core.primitives import Error
|
||||
from ..core.topic import topic
|
||||
|
||||
############################################################################
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
from dataclasses import dataclass, field
|
||||
|
||||
from ..core.primitives import Error
|
||||
from ..core.topic import topic
|
||||
|
||||
############################################################################
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue