mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-07-13 07:12:11 +02:00
Better proc group logging and concurrency (#810)
- Silence pika, cassandra etc. logging at INFO (too much chatter) - Add per processor log tags so that logs can be understood in processor group. - Deal with RabbitMQ lag weirdness - Added more processor group examples
This commit is contained in:
parent
ce3c8b421b
commit
2bf4af294e
20 changed files with 1021 additions and 647 deletions
|
|
@ -4,6 +4,7 @@ Embeddings service, applies an embeddings model using fastembed
|
|||
Input is text, output is embeddings vector.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from ... base import EmbeddingsService
|
||||
|
|
@ -37,7 +38,13 @@ class Processor(EmbeddingsService):
|
|||
self._load_model(model)
|
||||
|
||||
def _load_model(self, model_name):
|
||||
"""Load a model, caching it for reuse"""
|
||||
"""Load a model, caching it for reuse.
|
||||
|
||||
Synchronous — CPU and I/O heavy. Callers that run on the
|
||||
event loop must dispatch via asyncio.to_thread to avoid
|
||||
freezing the loop (which, in processor-group deployments,
|
||||
freezes every sibling processor in the same process).
|
||||
"""
|
||||
if self.cached_model_name != model_name:
|
||||
logger.info(f"Loading FastEmbed model: {model_name}")
|
||||
self.embeddings = TextEmbedding(model_name=model_name)
|
||||
|
|
@ -46,6 +53,11 @@ class Processor(EmbeddingsService):
|
|||
else:
|
||||
logger.debug(f"Using cached model: {model_name}")
|
||||
|
||||
def _run_embed(self, texts):
|
||||
"""Synchronous embed call. Runs in a worker thread via
|
||||
asyncio.to_thread from on_embeddings."""
|
||||
return list(self.embeddings.embed(texts))
|
||||
|
||||
async def on_embeddings(self, texts, model=None):
|
||||
|
||||
if not texts:
|
||||
|
|
@ -53,11 +65,18 @@ class Processor(EmbeddingsService):
|
|||
|
||||
use_model = model or self.default_model
|
||||
|
||||
# Reload model if it has changed
|
||||
self._load_model(use_model)
|
||||
# Reload model if it has changed. Model loading is sync
|
||||
# and can take seconds; push it to a worker thread so the
|
||||
# event loop (and any sibling processors in group mode)
|
||||
# stay responsive.
|
||||
if self.cached_model_name != use_model:
|
||||
await asyncio.to_thread(self._load_model, use_model)
|
||||
|
||||
# FastEmbed processes the full batch efficiently
|
||||
vecs = list(self.embeddings.embed(texts))
|
||||
# FastEmbed inference is synchronous ONNX runtime work.
|
||||
# Dispatch to a worker thread so the event loop stays
|
||||
# responsive for other tasks (important in group mode
|
||||
# where the loop is shared across many processors).
|
||||
vecs = await asyncio.to_thread(self._run_embed, texts)
|
||||
|
||||
# Return list of vectors, one per input text
|
||||
return [v.tolist() for v in vecs]
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue