nomyo-ai/nomyo-router
3
2
Fork 0
Code Issues 2 Pull requests 8 Projects Releases 10 Packages 1 Wiki Activity Actions

improving queue logic for high load scenarios

Browse source
This commit is contained in:
Alpha Nerd 2025-09-19 16:38:48 +02:00
parent 8fe3880af7
commit f0e181d6b8
1 changed files with 6 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

8
router.py
View file

@ -327,12 +327,16 @@ class rechunk:
# SSE Helpser # SSE Helpser
# ------------------------------------------------------------------ # ------------------------------------------------------------------
async def publish_snapshot(): async def publish_snapshot():
snapshot = json.dumps({"usage_counts": usage_counts}) async with usage_lock:
snapshot = json.dumps({"usage_counts": usage_counts})
async with _subscribers_lock: async with _subscribers_lock:
for q in _subscribers: for q in _subscribers:
# If the queue is full, drop the message to avoid back‑pressure. # If the queue is full, drop the message to avoid back‑pressure.
if q.full(): if q.full():
continue try:
await q.get()
except asyncio.QueueEmpty:
pass
await q.put(snapshot) await q.put(snapshot)
async def close_all_sse_queues(): async def close_all_sse_queues():