mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-04-26 00:46:22 +02:00
4fb0b4d8e8
Remove Pulsar-specific concepts from application code so that the pub/sub backend is swappable via configuration. Rename translators: - to_pulsar/from_pulsar → decode/encode across all translator classes, dispatch handlers, and tests (55+ files) - from_response_with_completion → encode_with_completion - Remove pulsar.schema.Record from translator base class Queue naming (CLASS:TOPICSPACE:TOPIC): - Replace topic() helper with queue() using new format: flow:tg:name, request:tg:name, response:tg:name, state:tg:name - Queue class implies persistence/TTL (no QoS in names) - Update Pulsar backend map_topic() to parse new format - Librarian queues use flow class (persistent, for chunking) - Config push uses state class (persistent, last-value) - Remove 15 dead topic imports from schema files - Update init_trustgraph.py namespace: config → state Confine Pulsar to pulsar_backend.py: - Delete legacy PulsarClient class from pubsub.py - Move add_args to add_pubsub_args() with standalone flag for CLI tools (defaults to localhost) - PulsarBackendConsumer.receive() catches _pulsar.Timeout, raises standard TimeoutError - Remove Pulsar imports from: async_processor, flow_processor, log_level, all 11 client files, 4 storage writers, gateway service, gateway config receiver - Remove log_level/LoggerLevel from client API - Rewrite tg-monitor-prompts to use backend abstraction - Update tg-dump-queues to use add_pubsub_args Also: pubsub-abstraction.md tech spec covering problem statement, design goals, as-is requirements, candidate broker assessment, approach, and implementation order.
168 lines
4 KiB
Python
168 lines
4 KiB
Python
|
|
import json
|
|
import dataclasses
|
|
|
|
from .. schema import PromptRequest, PromptResponse
|
|
from .. schema import prompt_request_queue
|
|
from .. schema import prompt_response_queue
|
|
from . base import BaseClient
|
|
|
|
# Ugly
|
|
|
|
@dataclasses.dataclass
|
|
class Definition:
|
|
name: str
|
|
definition: str
|
|
|
|
@dataclasses.dataclass
|
|
class Relationship:
|
|
s: str
|
|
p: str
|
|
o: str
|
|
o_entity: str
|
|
|
|
@dataclasses.dataclass
|
|
class Topic:
|
|
name: str
|
|
definition: str
|
|
|
|
class PromptClient(BaseClient):
|
|
|
|
def __init__(
|
|
self,
|
|
subscriber=None,
|
|
input_queue=None,
|
|
output_queue=None,
|
|
pulsar_host="pulsar://pulsar:6650",
|
|
pulsar_api_key=None,
|
|
):
|
|
|
|
if input_queue == None:
|
|
input_queue = prompt_request_queue
|
|
|
|
if output_queue == None:
|
|
output_queue = prompt_response_queue
|
|
|
|
super(PromptClient, self).__init__(
|
|
subscriber=subscriber,
|
|
input_queue=input_queue,
|
|
output_queue=output_queue,
|
|
pulsar_host=pulsar_host,
|
|
pulsar_api_key=pulsar_api_key,
|
|
input_schema=PromptRequest,
|
|
output_schema=PromptResponse,
|
|
)
|
|
|
|
def request(self, id, variables, timeout=300):
|
|
|
|
resp = self.call(
|
|
id=id,
|
|
terms={
|
|
k: json.dumps(v)
|
|
for k, v in variables.items()
|
|
},
|
|
timeout=timeout
|
|
)
|
|
|
|
if resp.text: return resp.text
|
|
|
|
return json.loads(resp.object)
|
|
|
|
def request_definitions(self, chunk, timeout=300):
|
|
|
|
defs = self.request(
|
|
id="extract-definitions",
|
|
variables={
|
|
"text": chunk
|
|
},
|
|
timeout=timeout
|
|
)
|
|
|
|
return [
|
|
Definition(name=d["entity"], definition=d["definition"])
|
|
for d in defs
|
|
]
|
|
|
|
def request_relationships(self, chunk, timeout=300):
|
|
|
|
rels = self.request(
|
|
id="extract-relationships",
|
|
variables={
|
|
"text": chunk
|
|
},
|
|
timeout=timeout
|
|
)
|
|
|
|
return [
|
|
Relationship(
|
|
s=d["subject"],
|
|
p=d["predicate"],
|
|
o=d["object"],
|
|
o_entity=d["object-entity"]
|
|
)
|
|
for d in rels
|
|
]
|
|
|
|
def request_topics(self, chunk, timeout=300):
|
|
|
|
topics = self.request(
|
|
id="extract-topics",
|
|
variables={
|
|
"text": chunk
|
|
},
|
|
timeout=timeout
|
|
)
|
|
|
|
return [
|
|
Topic(name=d["topic"], definition=d["definition"])
|
|
for d in topics
|
|
]
|
|
|
|
def request_rows(self, schema, chunk, timeout=300):
|
|
|
|
return self.request(
|
|
id="extract-rows",
|
|
variables={
|
|
"chunk": chunk,
|
|
"row-schema": {
|
|
"name": schema.name,
|
|
"description": schema.description,
|
|
"fields": [
|
|
{
|
|
"name": f.name, "type": str(f.type),
|
|
"size": f.size, "primary": f.primary,
|
|
"description": f.description,
|
|
}
|
|
for f in schema.fields
|
|
]
|
|
}
|
|
},
|
|
timeout=timeout
|
|
)
|
|
|
|
def request_kg_prompt(self, query, kg, timeout=300):
|
|
|
|
return self.request(
|
|
id="kg-prompt",
|
|
variables={
|
|
"query": query,
|
|
"knowledge": [
|
|
{ "s": v[0], "p": v[1], "o": v[2] }
|
|
for v in kg
|
|
]
|
|
},
|
|
timeout=timeout
|
|
)
|
|
|
|
def request_document_prompt(self, query, documents, timeout=300):
|
|
|
|
return self.request(
|
|
id="document-prompt",
|
|
variables={
|
|
"query": query,
|
|
"documents": documents,
|
|
},
|
|
timeout=timeout
|
|
)
|
|
|
|
|