mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-07-01 09:29:38 +02:00
24f0190ce7
Adds a RabbitMQ backend as an alternative to Pulsar, selectable via PUBSUB_BACKEND=rabbitmq. Both backends implement the same PubSubBackend protocol — no application code changes needed to switch. RabbitMQ topology: - Single topic exchange per topicspace (e.g. 'tg') - Routing key derived from queue class and topic name - Shared consumers: named queue bound to exchange (competing, round-robin) - Exclusive consumers: anonymous auto-delete queue (broadcast, each gets every message). Used by Subscriber and config push consumer. - Thread-local producer connections (pika is not thread-safe) - Push-based consumption via basic_consume with process_data_events for heartbeat processing Consumer model changes: - Consumer class creates one backend consumer per concurrent task (required for pika thread safety, harmless for Pulsar) - Consumer class accepts consumer_type parameter - Subscriber passes consumer_type='exclusive' for broadcast semantics - Config push consumer uses consumer_type='exclusive' so every processor instance receives config updates - handle_one_from_queue receives consumer as parameter for correct per-connection ack/nack LibrarianClient: - New shared client class replacing duplicated librarian request-response code across 6+ services (chunking, decoders, RAG, etc.) - Uses stream-document instead of get-document-content for fetching document content in 1MB chunks (avoids broker message size limits) - Standalone object (self.librarian = LibrarianClient(...)) not a mixin - get-document-content marked deprecated in schema and OpenAPI spec Serialisation: - Extracted dataclass_to_dict/dict_to_dataclass to shared serialization.py (used by both Pulsar and RabbitMQ backends) Librarian queues: - Changed from flow class (persistent) back to request/response class now that stream-document eliminates large single messages - API upload chunk size reduced from 5MB to 3MB to stay under broker limits after base64 encoding Factory and CLI: - get_pubsub() handles 'rabbitmq' backend with RabbitMQ connection params - add_pubsub_args() includes RabbitMQ options (host, port, credentials) - add_pubsub_args(standalone=True) defaults to localhost for CLI tools - init_trustgraph skips Pulsar admin setup for non-Pulsar backends - tg-dump-queues and tg-monitor-prompts use backend abstraction - BaseClient and ConfigClient accept generic pubsub config
42 lines
2.1 KiB
Python
42 lines
2.1 KiB
Python
|
|
from . pubsub import get_pubsub, add_pubsub_args
|
|
from . async_processor import AsyncProcessor
|
|
from . consumer import Consumer
|
|
from . producer import Producer
|
|
from . publisher import Publisher
|
|
from . subscriber import Subscriber
|
|
from . metrics import ProcessorMetrics, ConsumerMetrics, ProducerMetrics
|
|
from . logging import add_logging_args, setup_logging
|
|
from . flow_processor import FlowProcessor
|
|
from . consumer_spec import ConsumerSpec
|
|
from . parameter_spec import ParameterSpec
|
|
from . producer_spec import ProducerSpec
|
|
from . subscriber_spec import SubscriberSpec
|
|
from . request_response_spec import RequestResponseSpec
|
|
from . llm_service import LlmService, LlmResult, LlmChunk
|
|
from . librarian_client import LibrarianClient
|
|
from . chunking_service import ChunkingService
|
|
from . embeddings_service import EmbeddingsService
|
|
from . embeddings_client import EmbeddingsClientSpec
|
|
from . text_completion_client import TextCompletionClientSpec
|
|
from . prompt_client import PromptClientSpec
|
|
from . triples_store_service import TriplesStoreService
|
|
from . graph_embeddings_store_service import GraphEmbeddingsStoreService
|
|
from . document_embeddings_store_service import DocumentEmbeddingsStoreService
|
|
from . triples_query_service import TriplesQueryService
|
|
from . graph_embeddings_query_service import GraphEmbeddingsQueryService
|
|
from . document_embeddings_query_service import DocumentEmbeddingsQueryService
|
|
from . graph_embeddings_client import GraphEmbeddingsClientSpec
|
|
from . triples_client import TriplesClientSpec
|
|
from . document_embeddings_client import DocumentEmbeddingsClientSpec
|
|
from . agent_service import AgentService
|
|
from . graph_rag_client import GraphRagClientSpec
|
|
from . tool_service import ToolService
|
|
from . tool_client import ToolClientSpec
|
|
from . dynamic_tool_service import DynamicToolService
|
|
from . tool_service_client import ToolServiceClientSpec
|
|
from . agent_client import AgentClientSpec
|
|
from . structured_query_client import StructuredQueryClientSpec
|
|
from . row_embeddings_query_client import RowEmbeddingsQueryClientSpec
|
|
from . collection_config_handler import CollectionConfigHandler
|
|
|