refactor: use one fanout exchange per topic instead of shared topic exchange (#827)

The RabbitMQ backend used a single topic exchange per topicspace
with routing keys to differentiate logical topics. This meant the
flow service had to manually create named queues for every
processor-topic pair, including producer-side topics — creating
phantom queues that accumulated unread message copies indefinitely.

Replace with one fanout exchange per logical topic. Consumers now
declare and bind their own queues on connect. The flow service
manages topic lifecycle (create/delete exchanges) rather than queue
lifecycle, and only collects unique topic identifiers instead of
per-processor (topic, subscription) pairs.

Backend API: create_queue/delete_queue/ensure_queue replaced with
create_topic/delete_topic/ensure_topic (subscription parameter
removed).
This commit is contained in:
cybermaggedon 2026-04-17 18:01:35 +01:00 committed by GitHub
parent 391b9076f3
commit 3505bfdd25
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 190 additions and 228 deletions

View file

@ -121,7 +121,7 @@ class PubSubBackend(Protocol):
Create a producer for a topic.
Args:
topic: Generic topic format (qos/tenant/namespace/queue)
topic: Queue identifier in class:topicspace:topic format
schema: Dataclass type for messages
**options: Backend-specific options (e.g., chunking_enabled)
@ -159,59 +159,55 @@ class PubSubBackend(Protocol):
"""
...
async def create_queue(self, topic: str, subscription: str) -> None:
async def create_topic(self, topic: str) -> None:
"""
Pre-create a queue so it exists before any consumer connects.
Create the broker-side resources for a logical topic.
The topic and subscription together identify the queue, mirroring
create_consumer where the queue name is derived from both.
For RabbitMQ this creates a fanout exchange. For Pulsar this is
a no-op (topics auto-create on first use).
Idempotent creating an already-existing queue succeeds silently.
Idempotent creating an already-existing topic succeeds silently.
Args:
topic: Queue identifier in class:topicspace:topic format
subscription: Subscription/consumer group name
topic: Topic identifier in class:topicspace:topic format
"""
...
async def delete_queue(self, topic: str, subscription: str) -> None:
async def delete_topic(self, topic: str) -> None:
"""
Delete a queue and any messages it contains.
Delete a topic and discard any in-flight messages.
The topic and subscription together identify the queue, mirroring
create_consumer where the queue name is derived from both.
For RabbitMQ this deletes the fanout exchange; consumer queues
lose their binding and drain naturally.
Idempotent deleting a non-existent queue succeeds silently.
Idempotent deleting a non-existent topic succeeds silently.
Args:
topic: Queue identifier in class:topicspace:topic format
subscription: Subscription/consumer group name
topic: Topic identifier in class:topicspace:topic format
"""
...
async def queue_exists(self, topic: str, subscription: str) -> bool:
async def topic_exists(self, topic: str) -> bool:
"""
Check whether a queue exists.
Check whether a topic exists.
Args:
topic: Queue identifier in class:topicspace:topic format
subscription: Subscription/consumer group name
topic: Topic identifier in class:topicspace:topic format
Returns:
True if the queue exists, False otherwise.
True if the topic exists, False otherwise.
"""
...
async def ensure_queue(self, topic: str, subscription: str) -> None:
async def ensure_topic(self, topic: str) -> None:
"""
Ensure a queue exists, creating it if necessary.
Ensure a topic exists, creating it if necessary.
Convenience wrapper checks existence, creates if missing.
Used by system services on startup.
Used by the flow service and system services on startup.
Args:
topic: Queue identifier in class:topicspace:topic format
subscription: Subscription/consumer group name
topic: Topic identifier in class:topicspace:topic format
"""
...

View file

@ -266,22 +266,22 @@ class PulsarBackend:
return PulsarBackendConsumer(pulsar_consumer, schema)
async def create_queue(self, topic: str, subscription: str) -> None:
async def create_topic(self, topic: str) -> None:
"""No-op — Pulsar auto-creates topics on first use.
TODO: Use admin REST API for explicit persistent topic creation."""
pass
async def delete_queue(self, topic: str, subscription: str) -> None:
async def delete_topic(self, topic: str) -> None:
"""No-op — to be replaced with admin REST API calls.
TODO: Delete subscription and persistent topic via admin API."""
TODO: Delete persistent topic via admin API."""
pass
async def queue_exists(self, topic: str, subscription: str) -> bool:
async def topic_exists(self, topic: str) -> bool:
"""Returns True — Pulsar auto-creates on subscribe.
TODO: Use admin REST API for actual existence check."""
return True
async def ensure_queue(self, topic: str, subscription: str) -> None:
async def ensure_topic(self, topic: str) -> None:
"""No-op — Pulsar auto-creates topics on first use.
TODO: Use admin REST API for explicit creation."""
pass

View file

@ -1,22 +1,24 @@
"""
RabbitMQ backend implementation for pub/sub abstraction.
Uses a single topic exchange per topicspace. The logical queue name
becomes the routing key. Consumer behavior is determined by the
subscription name:
Each logical topic maps to its own fanout exchange. The exchange name
encodes the full topic identity:
- Same subscription + same topic = shared queue (competing consumers)
- Different subscriptions = separate queues (broadcast / fan-out)
class:topicspace:topic exchange topicspace.class.topic
This mirrors Pulsar's subscription model using idiomatic RabbitMQ.
Producers publish to the exchange with an empty routing key.
Consumers declare and bind their own queues:
- flow / request: named durable/non-durable queue (competing consumers)
- response / notify: anonymous exclusive auto-delete queue (per-subscriber)
The flow service manages topic lifecycle (create/delete exchanges).
Consumers manage their own queue lifecycle (declare + bind on connect).
Architecture:
Producer --> [tg exchange] --routing key--> [named queue] --> Consumer
--routing key--> [named queue] --> Consumer
--routing key--> [exclusive q] --> Subscriber
Uses basic_consume (push) instead of basic_get (polling) for
efficient message delivery.
Producer --> [fanout exchange] --> [named queue] --> Consumer
--> [named queue] --> Consumer
--> [exclusive queue] --> Subscriber
"""
import asyncio
@ -58,18 +60,16 @@ class RabbitMQMessage:
class RabbitMQBackendProducer:
"""Publishes messages to a topic exchange with a routing key.
"""Publishes messages to a fanout exchange.
Uses thread-local connections so each thread gets its own
connection/channel. This avoids wire corruption from concurrent
threads writing to the same socket (pika is not thread-safe).
"""
def __init__(self, connection_params, exchange_name, routing_key,
durable):
def __init__(self, connection_params, exchange_name, durable):
self._connection_params = connection_params
self._exchange_name = exchange_name
self._routing_key = routing_key
self._durable = durable
self._local = threading.local()
@ -90,7 +90,7 @@ class RabbitMQBackendProducer:
chan = conn.channel()
chan.exchange_declare(
exchange=self._exchange_name,
exchange_type='topic',
exchange_type='fanout',
durable=True,
)
self._local.connection = conn
@ -113,7 +113,7 @@ class RabbitMQBackendProducer:
channel = self._get_channel()
channel.basic_publish(
exchange=self._exchange_name,
routing_key=self._routing_key,
routing_key='',
body=json_data.encode('utf-8'),
properties=amqp_properties,
)
@ -144,19 +144,17 @@ class RabbitMQBackendProducer:
class RabbitMQBackendConsumer:
"""Consumes from a queue bound to a topic exchange.
"""Consumes from a queue bound to a fanout exchange.
Uses basic_consume (push model) with messages delivered to an
internal thread-safe queue. process_data_events() drives both
message delivery and heartbeat processing.
"""
def __init__(self, connection_params, exchange_name, routing_key,
queue_name, schema_cls, durable, exclusive=False,
auto_delete=False):
def __init__(self, connection_params, exchange_name, queue_name,
schema_cls, durable, exclusive=False, auto_delete=False):
self._connection_params = connection_params
self._exchange_name = exchange_name
self._routing_key = routing_key
self._queue_name = queue_name
self._schema_cls = schema_cls
self._durable = durable
@ -171,17 +169,16 @@ class RabbitMQBackendConsumer:
self._connection = pika.BlockingConnection(self._connection_params)
self._channel = self._connection.channel()
# Declare the topic exchange (idempotent, also done by producers)
# Declare the fanout exchange (idempotent)
self._channel.exchange_declare(
exchange=self._exchange_name,
exchange_type='topic',
exchange_type='fanout',
durable=True,
)
if self._exclusive:
# Anonymous ephemeral queue (response/notify class).
# These are per-consumer and must be created here — the
# broker assigns the name.
# Per-consumer, broker assigns the name.
result = self._channel.queue_declare(
queue='',
durable=False,
@ -189,20 +186,22 @@ class RabbitMQBackendConsumer:
auto_delete=True,
)
self._queue_name = result.method.queue
self._channel.queue_bind(
queue=self._queue_name,
exchange=self._exchange_name,
routing_key=self._routing_key,
)
else:
# Named queue (flow/request class). Queue must already
# exist — created by the flow service or ensure_queue.
# We just verify it exists and bind to consume.
# Named queue (flow/request class).
# Consumer owns its queue — declare and bind here.
self._channel.queue_declare(
queue=self._queue_name, passive=True,
queue=self._queue_name,
durable=self._durable,
exclusive=False,
auto_delete=False,
)
# Bind queue to the fanout exchange
self._channel.queue_bind(
queue=self._queue_name,
exchange=self._exchange_name,
)
self._channel.basic_qos(prefetch_count=1)
# Register push-based consumer
@ -318,7 +317,7 @@ class RabbitMQBackendConsumer:
class RabbitMQBackend:
"""RabbitMQ pub/sub backend using a topic exchange per topicspace."""
"""RabbitMQ pub/sub backend using one fanout exchange per topic."""
def __init__(self, host='localhost', port=5672, username='guest',
password='guest', vhost='/'):
@ -331,20 +330,23 @@ class RabbitMQBackend:
)
logger.info(f"RabbitMQ backend: {host}:{port} vhost={vhost}")
def _parse_queue_id(self, queue_id: str) -> tuple[str, str, str, bool]:
def _parse_topic(self, topic_id: str) -> tuple[str, str, bool]:
"""
Parse queue identifier into exchange, routing key, and durability.
Parse topic identifier into exchange name and durability.
Format: class:topicspace:topic
Returns: (exchange_name, routing_key, class, durable)
"""
if ':' not in queue_id:
return 'tg', queue_id, 'flow', False
Returns: (exchange_name, class, durable)
parts = queue_id.split(':', 2)
The exchange name encodes the full topic identity:
class:topicspace:topic topicspace.class.topic
"""
if ':' not in topic_id:
return f'tg.flow.{topic_id}', 'flow', True
parts = topic_id.split(':', 2)
if len(parts) != 3:
raise ValueError(
f"Invalid queue format: {queue_id}, "
f"Invalid topic format: {topic_id}, "
f"expected class:topicspace:topic"
)
@ -356,36 +358,28 @@ class RabbitMQBackend:
durable = False
else:
raise ValueError(
f"Invalid queue class: {cls}, "
f"Invalid topic class: {cls}, "
f"expected flow, request, response, or notify"
)
# Exchange per topicspace, routing key includes class
exchange_name = topicspace
routing_key = f"{cls}.{topic}"
exchange_name = f"{topicspace}.{cls}.{topic}"
return exchange_name, routing_key, cls, durable
# Keep map_queue_name for backward compatibility with tests
def map_queue_name(self, queue_id: str) -> tuple[str, bool]:
exchange, routing_key, cls, durable = self._parse_queue_id(queue_id)
return f"{exchange}.{routing_key}", durable
return exchange_name, cls, durable
def create_producer(self, topic: str, schema: type,
**options) -> BackendProducer:
exchange, routing_key, cls, durable = self._parse_queue_id(topic)
exchange, cls, durable = self._parse_topic(topic)
logger.debug(
f"Creating producer: exchange={exchange}, "
f"routing_key={routing_key}"
f"Creating producer: exchange={exchange}"
)
return RabbitMQBackendProducer(
self._connection_params, exchange, routing_key, durable,
self._connection_params, exchange, durable,
)
def create_consumer(self, topic: str, subscription: str, schema: type,
initial_position: str = 'latest',
**options) -> BackendConsumer:
"""Create a consumer with a queue bound to the topic exchange.
"""Create a consumer with a queue bound to the topic's exchange.
Behaviour is determined by the topic's class prefix:
- flow: named durable queue, competing consumers (round-robin)
@ -393,7 +387,7 @@ class RabbitMQBackend:
- response: anonymous ephemeral queue, per-subscriber (auto-delete)
- notify: anonymous ephemeral queue, per-subscriber (auto-delete)
"""
exchange, routing_key, cls, durable = self._parse_queue_id(topic)
exchange, cls, durable = self._parse_topic(topic)
if cls in ('response', 'notify'):
# Per-subscriber: anonymous queue, auto-deleted on disconnect
@ -403,45 +397,33 @@ class RabbitMQBackend:
auto_delete = True
else:
# Shared: named queue, competing consumers
queue_name = f"{exchange}.{routing_key}.{subscription}"
queue_name = f"{exchange}.{subscription}"
queue_durable = durable
exclusive = False
auto_delete = False
logger.debug(
f"Creating consumer: exchange={exchange}, "
f"routing_key={routing_key}, queue={queue_name or '(anonymous)'}, "
f"cls={cls}"
f"queue={queue_name or '(anonymous)'}, cls={cls}"
)
return RabbitMQBackendConsumer(
self._connection_params, exchange, routing_key,
self._connection_params, exchange,
queue_name, schema, queue_durable, exclusive, auto_delete,
)
def _create_queue_sync(self, exchange, routing_key, queue_name, durable):
"""Blocking queue creation — run via asyncio.to_thread."""
def _create_topic_sync(self, exchange_name):
"""Blocking exchange creation — run via asyncio.to_thread."""
connection = None
try:
connection = pika.BlockingConnection(self._connection_params)
channel = connection.channel()
channel.exchange_declare(
exchange=exchange,
exchange_type='topic',
exchange=exchange_name,
exchange_type='fanout',
durable=True,
)
channel.queue_declare(
queue=queue_name,
durable=durable,
exclusive=False,
auto_delete=False,
)
channel.queue_bind(
queue=queue_name,
exchange=exchange,
routing_key=routing_key,
)
logger.info(f"Created queue: {queue_name}")
logger.info(f"Created topic (exchange): {exchange_name}")
finally:
if connection and connection.is_open:
try:
@ -449,34 +431,30 @@ class RabbitMQBackend:
except Exception:
pass
async def create_queue(self, topic: str, subscription: str) -> None:
"""Pre-create a named queue bound to the topic exchange.
async def create_topic(self, topic: str) -> None:
"""Create the fanout exchange for a logical topic.
Only applies to shared queues (flow/request class). Response and
notify queues are anonymous/auto-delete and created by consumers.
Only applies to flow and request class topics. Response and
notify exchanges are created on demand by consumers.
"""
exchange, routing_key, cls, durable = self._parse_queue_id(topic)
exchange, cls, durable = self._parse_topic(topic)
if cls in ('response', 'notify'):
return
queue_name = f"{exchange}.{routing_key}.{subscription}"
await asyncio.to_thread(
self._create_queue_sync, exchange, routing_key,
queue_name, durable,
)
await asyncio.to_thread(self._create_topic_sync, exchange)
def _delete_queue_sync(self, queue_name):
"""Blocking queue deletion — run via asyncio.to_thread."""
def _delete_topic_sync(self, exchange_name):
"""Blocking exchange deletion — run via asyncio.to_thread."""
connection = None
try:
connection = pika.BlockingConnection(self._connection_params)
channel = connection.channel()
channel.queue_delete(queue=queue_name)
logger.info(f"Deleted queue: {queue_name}")
channel.exchange_delete(exchange=exchange_name)
logger.info(f"Deleted topic (exchange): {exchange_name}")
except Exception as e:
# Idempotent — queue may already be gone
logger.debug(f"Queue delete for {queue_name}: {e}")
# Idempotent — exchange may already be gone
logger.debug(f"Exchange delete for {exchange_name}: {e}")
finally:
if connection and connection.is_open:
try:
@ -484,31 +462,27 @@ class RabbitMQBackend:
except Exception:
pass
async def delete_queue(self, topic: str, subscription: str) -> None:
"""Delete a named queue and any messages it contains.
async def delete_topic(self, topic: str) -> None:
"""Delete a topic's fanout exchange.
Only applies to shared queues (flow/request class). Response and
notify queues are anonymous/auto-delete and managed by the broker.
Consumer queues lose their binding and drain naturally.
"""
exchange, routing_key, cls, durable = self._parse_queue_id(topic)
exchange, cls, durable = self._parse_topic(topic)
await asyncio.to_thread(self._delete_topic_sync, exchange)
if cls in ('response', 'notify'):
return
queue_name = f"{exchange}.{routing_key}.{subscription}"
await asyncio.to_thread(self._delete_queue_sync, queue_name)
def _queue_exists_sync(self, queue_name):
"""Blocking queue existence check — run via asyncio.to_thread.
def _topic_exists_sync(self, exchange_name):
"""Blocking exchange existence check — run via asyncio.to_thread.
Uses passive=True which checks without creating."""
connection = None
try:
connection = pika.BlockingConnection(self._connection_params)
channel = connection.channel()
channel.queue_declare(queue=queue_name, passive=True)
channel.exchange_declare(
exchange=exchange_name, passive=True,
)
return True
except pika.exceptions.ChannelClosedByBroker:
# 404 NOT_FOUND — queue does not exist
# 404 NOT_FOUND — exchange does not exist
return False
finally:
if connection and connection.is_open:
@ -517,26 +491,25 @@ class RabbitMQBackend:
except Exception:
pass
async def queue_exists(self, topic: str, subscription: str) -> bool:
"""Check whether a named queue exists.
async def topic_exists(self, topic: str) -> bool:
"""Check whether a topic's exchange exists.
Only applies to shared queues (flow/request class). Response and
notify queues are anonymous/ephemeral always returns False.
Only applies to flow and request class topics. Response and
notify topics are ephemeral always returns False.
"""
exchange, routing_key, cls, durable = self._parse_queue_id(topic)
exchange, cls, durable = self._parse_topic(topic)
if cls in ('response', 'notify'):
return False
queue_name = f"{exchange}.{routing_key}.{subscription}"
return await asyncio.to_thread(
self._queue_exists_sync, queue_name
self._topic_exists_sync, exchange
)
async def ensure_queue(self, topic: str, subscription: str) -> None:
"""Ensure a queue exists, creating it if necessary."""
if not await self.queue_exists(topic, subscription):
await self.create_queue(topic, subscription)
async def ensure_topic(self, topic: str) -> None:
"""Ensure a topic exists, creating it if necessary."""
if not await self.topic_exists(topic):
await self.create_topic(topic)
def close(self) -> None:
pass