fix: prevent duplicate dispatcher creation race condition in invoke_global_service (#715)

* fix: prevent duplicate dispatcher creation race condition in invoke_global_service

Concurrent coroutines could all pass the `if key in self.dispatchers` check
before any of them wrote the result back, because `await dispatcher.start()`
yields to the event loop. This caused multiple Pulsar consumers to be created
on the same shared subscription, distributing responses round-robin and
dropping ~2/3 of them — manifesting as a permanent spinner in the Workbench UI.

Apply a double-checked asyncio.Lock in both `invoke_global_service` and
`invoke_flow_service` so only one dispatcher is ever created per service key.

* test: add concurrent-dispatch tests for race condition fix

Add asyncio.gather-based tests that verify invoke_global_service and
invoke_flow_service create exactly one dispatcher under concurrent calls,
preventing the duplicate Pulsar consumer bug.

trustgraph-flow/trustgraph/gateway/dispatch/manager.py

@@ -118,6 +118,7 @@ class DispatcherManager:
 
         self.flows = {}
         self.dispatchers = {}
+        self.dispatcher_lock = asyncio.Lock()
 
     async def start_flow(self, id, flow):
         logger.info(f"Starting flow {id}")
@@ -165,30 +166,28 @@ class DispatcherManager:
 
         key = (None, kind)
 
-        if key in self.dispatchers:
-            return await self.dispatchers[key].process(data, responder)
+        if key not in self.dispatchers:
+            async with self.dispatcher_lock:
+                if key not in self.dispatchers:
+                    request_queue = None
+                    response_queue = None
+                    if kind in self.queue_overrides:
+                        request_queue = self.queue_overrides[kind].get("request")
+                        response_queue = self.queue_overrides[kind].get("response")
 
-        # Get queue overrides if specified for this service
-        request_queue = None
-        response_queue = None
-        if kind in self.queue_overrides:
-            request_queue = self.queue_overrides[kind].get("request")
-            response_queue = self.queue_overrides[kind].get("response")
+                    dispatcher = global_dispatchers[kind](
+                        backend = self.backend,
+                        timeout = 120,
+                        consumer = f"{self.prefix}-{kind}-request",
+                        subscriber = f"{self.prefix}-{kind}-request",
+                        request_queue = request_queue,
+                        response_queue = response_queue,
+                    )
 
-        dispatcher = global_dispatchers[kind](
-            backend = self.backend,
-            timeout = 120,
-            consumer = f"{self.prefix}-{kind}-request",
-            subscriber = f"{self.prefix}-{kind}-request",
-            request_queue = request_queue,
-            response_queue = response_queue,
-        )
+                    await dispatcher.start()
+                    self.dispatchers[key] = dispatcher
 
-        await dispatcher.start()
-
-        self.dispatchers[key] = dispatcher
-
-        return await dispatcher.process(data, responder)
+        return await self.dispatchers[key].process(data, responder)
 
     def dispatch_flow_import(self):
         return self.process_flow_import
@@ -299,36 +298,35 @@ class DispatcherManager:
 
         key = (flow, kind)
 
-        if key in self.dispatchers:
-            return await self.dispatchers[key].process(data, responder)
+        if key not in self.dispatchers:
+            async with self.dispatcher_lock:
+                if key not in self.dispatchers:
+                    intf_defs = self.flows[flow]["interfaces"]
 
-        intf_defs = self.flows[flow]["interfaces"]
+                    if kind not in intf_defs:
+                        raise RuntimeError("This kind not supported by flow")
 
-        if kind not in intf_defs:
-            raise RuntimeError("This kind not supported by flow")
+                    qconfig = intf_defs[kind]
 
-        qconfig = intf_defs[kind]
+                    if kind in request_response_dispatchers:
+                        dispatcher = request_response_dispatchers[kind](
+                            backend = self.backend,
+                            request_queue = qconfig["request"],
+                            response_queue = qconfig["response"],
+                            timeout = 120,
+                            consumer = f"{self.prefix}-{flow}-{kind}-request",
+                            subscriber = f"{self.prefix}-{flow}-{kind}-request",
+                        )
+                    elif kind in sender_dispatchers:
+                        dispatcher = sender_dispatchers[kind](
+                            backend = self.backend,
+                            queue = qconfig,
+                        )
+                    else:
+                        raise RuntimeError("Invalid kind")
 
-        if kind in request_response_dispatchers:
-            dispatcher = request_response_dispatchers[kind](
-                backend = self.backend,
-                request_queue = qconfig["request"],
-                response_queue = qconfig["response"],
-                timeout = 120,
-                consumer = f"{self.prefix}-{flow}-{kind}-request",
-                subscriber = f"{self.prefix}-{flow}-{kind}-request",
-            )
-        elif kind in sender_dispatchers:
-            dispatcher = sender_dispatchers[kind](
-                backend = self.backend,
-                queue = qconfig,
-            )
-        else:
-            raise RuntimeError("Invalid kind")
-            
-        await dispatcher.start()
+                    await dispatcher.start()
+                    self.dispatchers[key] = dispatcher
 
-        self.dispatchers[key] = dispatcher
-
-        return await dispatcher.process(data, responder)
+        return await self.dispatchers[key].process(data, responder)