apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-07-24 12:41:02 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Logging strategy updates

Browse source
This commit is contained in:
Cyber MacGeddon 2025-07-30 22:11:37 +01:00
parent 84598c50f6
commit c28ba58591
2 changed files with 18 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

8
trustgraph-flow/trustgraph/librarian/librarian.py
View file

@ -45,20 +45,20 @@ class Librarian:
# Create object ID for blob # Create object ID for blob
object_id = uuid.uuid4() object_id = uuid.uuid4()
print("Add blob...") logger.debug("Adding blob...")
await self.blob_store.add( await self.blob_store.add(
object_id, base64.b64decode(request.content), object_id, base64.b64decode(request.content),
request.document_metadata.kind request.document_metadata.kind
) )
print("Add table...") logger.debug("Adding to table...")
await self.table_store.add_document( await self.table_store.add_document(
request.document_metadata, object_id request.document_metadata, object_id
) )
print("Add complete", flush=True) logger.debug("Add complete")
return LibrarianResponse( return LibrarianResponse(
error = None, error = None,
@ -206,7 +206,7 @@ class Librarian:
content = content, content = content,
) )
print("Add complete", flush=True) logger.debug("Add complete")
return LibrarianResponse( return LibrarianResponse(
error = None, error = None,

24
trustgraph-flow/trustgraph/librarian/service.py
View file

@ -7,6 +7,7 @@ from functools import partial
import asyncio import asyncio
import base64 import base64
import json import json
import logging
from .. base import AsyncProcessor, Consumer, Producer, Publisher, Subscriber from .. base import AsyncProcessor, Consumer, Producer, Publisher, Subscriber
from .. base import ConsumerMetrics, ProducerMetrics from .. base import ConsumerMetrics, ProducerMetrics
@ -21,6 +22,9 @@ from .. exceptions import RequestError
from . librarian import Librarian from . librarian import Librarian
# Module logger
logger = logging.getLogger(__name__)
default_ident = "librarian" default_ident = "librarian"
default_librarian_request_queue = librarian_request_queue default_librarian_request_queue = librarian_request_queue
@ -119,7 +123,7 @@ class Processor(AsyncProcessor):
self.flows = {} self.flows = {}
print("Initialised.", flush=True) logger.info("Librarian service initialized")
async def start(self): async def start(self):
@ -129,7 +133,7 @@ class Processor(AsyncProcessor):
async def on_librarian_config(self, config, version): async def on_librarian_config(self, config, version):
print("config version", version) logger.info(f"Configuration version: {version}")
if "flows" in config: if "flows" in config:
@ -138,7 +142,7 @@ class Processor(AsyncProcessor):
for k, v in config["flows"].items() for k, v in config["flows"].items()
} }
print(self.flows) logger.debug(f"Flows: {self.flows}")
def __del__(self): def __del__(self):
@ -146,9 +150,9 @@ class Processor(AsyncProcessor):
async def load_document(self, document, processing, content): async def load_document(self, document, processing, content):
print("Ready for processing...") logger.debug("Ready for document processing...")
print(document, processing, len(content)) logger.debug(f"Document: {document}, processing: {processing}, content length: {len(content)}")
if processing.flow not in self.flows: if processing.flow not in self.flows:
raise RuntimeError("Invalid flow ID") raise RuntimeError("Invalid flow ID")
@ -188,7 +192,7 @@ class Processor(AsyncProcessor):
) )
schema = Document schema = Document
print(f"Submit on queue {q}...") logger.debug(f"Submitting to queue {q}...")
pub = Publisher( pub = Publisher(
self.pulsar_client, q, schema=schema self.pulsar_client, q, schema=schema
@ -203,14 +207,14 @@ class Processor(AsyncProcessor):
await pub.stop() await pub.stop()
print("Document submitted") logger.debug("Document submitted")
async def process_request(self, v): async def process_request(self, v):
if v.operation is None: if v.operation is None:
raise RequestError("Null operation") raise RequestError("Null operation")
print("request", v.operation) logger.debug(f"Librarian request: {v.operation}")
impls = { impls = {
"add-document": self.librarian.add_document, "add-document": self.librarian.add_document,
@ -237,7 +241,7 @@ class Processor(AsyncProcessor):
id = msg.properties()["id"] id = msg.properties()["id"]
print(f"Handling input {id}...", flush=True) logger.info(f"Handling librarian input {id}...")
try: try:
@ -276,7 +280,7 @@ class Processor(AsyncProcessor):
return return
print("Done.", flush=True) logger.debug("Librarian input processing complete")
@staticmethod @staticmethod
def add_args(parser): def add_args(parser):