mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-07-23 04:01:02 +02:00
More dispatch code translated
This commit is contained in:
Cyber MacGeddon
parent
8bbf1cc6c0
commit
5ba7199895
14 changed files with 548 additions and 176 deletions
|
|
@ -2,6 +2,7 @@
|
|||
from ... schema import ConfigRequest, ConfigResponse, ConfigKey, ConfigValue
|
||||
from ... schema import config_request_queue
|
||||
from ... schema import config_response_queue
|
||||
from .... base.messaging import TranslatorRegistry
|
||||
|
||||
from . requestor import ServiceRequestor
|
||||
|
||||
|
|
@ -19,60 +20,12 @@ class ConfigRequestor(ServiceRequestor):
|
|||
timeout=timeout,
|
||||
)
|
||||
|
||||
self.request_translator = TranslatorRegistry.get_request_translator("config")
|
||||
self.response_translator = TranslatorRegistry.get_response_translator("config")
|
||||
|
||||
def to_request(self, body):
|
||||
|
||||
if "keys" in body:
|
||||
keys = [
|
||||
ConfigKey(
|
||||
type = k["type"],
|
||||
key = k["key"],
|
||||
)
|
||||
for k in body["keys"]
|
||||
]
|
||||
else:
|
||||
keys = None
|
||||
|
||||
if "values" in body:
|
||||
values = [
|
||||
ConfigValue(
|
||||
type = v["type"],
|
||||
key = v["key"],
|
||||
value = v["value"],
|
||||
)
|
||||
for v in body["values"]
|
||||
]
|
||||
else:
|
||||
values = None
|
||||
|
||||
return ConfigRequest(
|
||||
operation = body.get("operation", None),
|
||||
keys = keys,
|
||||
type = body.get("type", None),
|
||||
values = values
|
||||
)
|
||||
return self.request_translator.to_pulsar(body)
|
||||
|
||||
def from_response(self, message):
|
||||
|
||||
response = { }
|
||||
|
||||
if message.version is not None:
|
||||
response["version"] = message.version
|
||||
|
||||
if message.values is not None:
|
||||
response["values"] = [
|
||||
{
|
||||
"type": v.type,
|
||||
"key": v.key,
|
||||
"value": v.value,
|
||||
}
|
||||
for v in message.values
|
||||
]
|
||||
|
||||
if message.directory is not None:
|
||||
response["directory"] = message.directory
|
||||
|
||||
if message.config is not None:
|
||||
response["config"] = message.config
|
||||
|
||||
return response, True
|
||||
return self.response_translator.from_response_with_completion(message)
|
||||
|
||||
|
|
|
|||
|
|
@ -6,8 +6,7 @@ from aiohttp import WSMsgType
|
|||
from ... schema import Metadata
|
||||
from ... schema import DocumentEmbeddings, ChunkEmbeddings
|
||||
from ... base import Publisher
|
||||
|
||||
from . serialize import to_subgraph
|
||||
from .... base.messaging.translators.document_loading import DocumentEmbeddingsTranslator
|
||||
|
||||
class DocumentEmbeddingsImport:
|
||||
|
||||
|
|
@ -17,6 +16,7 @@ class DocumentEmbeddingsImport:
|
|||
|
||||
self.ws = ws
|
||||
self.running = running
|
||||
self.translator = DocumentEmbeddingsTranslator()
|
||||
|
||||
self.publisher = Publisher(
|
||||
pulsar_client, topic = queue, schema = DocumentEmbeddings
|
||||
|
|
@ -36,23 +36,7 @@ class DocumentEmbeddingsImport:
|
|||
async def receive(self, msg):
|
||||
|
||||
data = msg.json()
|
||||
|
||||
elt = DocumentEmbeddings(
|
||||
metadata=Metadata(
|
||||
id=data["metadata"]["id"],
|
||||
metadata=to_subgraph(data["metadata"]["metadata"]),
|
||||
user=data["metadata"]["user"],
|
||||
collection=data["metadata"]["collection"],
|
||||
),
|
||||
chunks=[
|
||||
ChunkEmbeddings(
|
||||
chunk=de["chunk"].encode("utf-8"),
|
||||
vectors=de["vectors"],
|
||||
)
|
||||
for de in data["chunks"]
|
||||
],
|
||||
)
|
||||
|
||||
elt = self.translator.to_pulsar(data)
|
||||
await self.publisher.send(None, elt)
|
||||
|
||||
async def run(self):
|
||||
|
|
|
|||
|
|
@ -2,9 +2,9 @@
|
|||
import base64
|
||||
|
||||
from ... schema import Document, Metadata
|
||||
from .... base.messaging import TranslatorRegistry
|
||||
|
||||
from . sender import ServiceSender
|
||||
from . serialize import to_subgraph
|
||||
|
||||
class DocumentLoad(ServiceSender):
|
||||
def __init__(self, pulsar_client, queue):
|
||||
|
|
@ -15,26 +15,9 @@ class DocumentLoad(ServiceSender):
|
|||
schema = Document,
|
||||
)
|
||||
|
||||
self.translator = TranslatorRegistry.get_request_translator("document")
|
||||
|
||||
def to_request(self, body):
|
||||
|
||||
if "metadata" in body:
|
||||
metadata = to_subgraph(body["metadata"])
|
||||
else:
|
||||
metadata = []
|
||||
|
||||
# Doing a base64 decoe/encode here to make sure the
|
||||
# content is valid base64
|
||||
doc = base64.b64decode(body["data"])
|
||||
|
||||
print("Document received")
|
||||
|
||||
return Document(
|
||||
metadata=Metadata(
|
||||
id=body.get("id"),
|
||||
metadata=metadata,
|
||||
user=body.get("user", "trustgraph"),
|
||||
collection=body.get("collection", "default"),
|
||||
),
|
||||
data=base64.b64encode(doc).decode("utf-8")
|
||||
)
|
||||
return self.translator.to_pulsar(body)
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
from ... schema import FlowRequest, FlowResponse
|
||||
from ... schema import flow_request_queue
|
||||
from ... schema import flow_response_queue
|
||||
from .... base.messaging import TranslatorRegistry
|
||||
|
||||
from . requestor import ServiceRequestor
|
||||
|
||||
|
|
@ -19,34 +20,12 @@ class FlowRequestor(ServiceRequestor):
|
|||
timeout=timeout,
|
||||
)
|
||||
|
||||
def to_request(self, body):
|
||||
self.request_translator = TranslatorRegistry.get_request_translator("flow")
|
||||
self.response_translator = TranslatorRegistry.get_response_translator("flow")
|
||||
|
||||
return FlowRequest(
|
||||
operation = body.get("operation", None),
|
||||
class_name = body.get("class-name", None),
|
||||
class_definition = body.get("class-definition", None),
|
||||
description = body.get("description", None),
|
||||
flow_id = body.get("flow-id", None),
|
||||
)
|
||||
def to_request(self, body):
|
||||
return self.request_translator.to_pulsar(body)
|
||||
|
||||
def from_response(self, message):
|
||||
|
||||
response = { }
|
||||
|
||||
if message.class_names is not None:
|
||||
response["class-names"] = message.class_names
|
||||
|
||||
if message.flow_ids is not None:
|
||||
response["flow-ids"] = message.flow_ids
|
||||
|
||||
if message.class_definition is not None:
|
||||
response["class-definition"] = message.class_definition
|
||||
|
||||
if message.flow is not None:
|
||||
response["flow"] = message.flow
|
||||
|
||||
if message.description is not None:
|
||||
response["description"] = message.description
|
||||
|
||||
return response, True
|
||||
return self.response_translator.from_response_with_completion(message)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
|
||||
from ... schema import GraphEmbeddingsRequest, GraphEmbeddingsResponse
|
||||
from .... base.messaging import TranslatorRegistry
|
||||
|
||||
from . requestor import ServiceRequestor
|
||||
from . serialize import serialize_value
|
||||
|
||||
class GraphEmbeddingsQueryRequestor(ServiceRequestor):
|
||||
def __init__(
|
||||
|
|
@ -21,22 +21,12 @@ class GraphEmbeddingsQueryRequestor(ServiceRequestor):
|
|||
timeout=timeout,
|
||||
)
|
||||
|
||||
self.request_translator = TranslatorRegistry.get_request_translator("graph-embeddings-query")
|
||||
self.response_translator = TranslatorRegistry.get_response_translator("graph-embeddings-query")
|
||||
|
||||
def to_request(self, body):
|
||||
|
||||
limit = int(body.get("limit", 20))
|
||||
|
||||
return GraphEmbeddingsRequest(
|
||||
vectors = body["vectors"],
|
||||
limit = limit,
|
||||
user = body.get("user", "trustgraph"),
|
||||
collection = body.get("collection", "default"),
|
||||
)
|
||||
return self.request_translator.to_pulsar(body)
|
||||
|
||||
def from_response(self, message):
|
||||
|
||||
return {
|
||||
"entities": [
|
||||
serialize_value(ent) for ent in message.entities
|
||||
]
|
||||
}, True
|
||||
return self.response_translator.from_response_with_completion(message)
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
import json
|
||||
|
||||
from ... schema import PromptRequest, PromptResponse
|
||||
from .... base.messaging import TranslatorRegistry
|
||||
|
||||
from . requestor import ServiceRequestor
|
||||
|
||||
|
|
@ -22,22 +23,12 @@ class PromptRequestor(ServiceRequestor):
|
|||
timeout=timeout,
|
||||
)
|
||||
|
||||
self.request_translator = TranslatorRegistry.get_request_translator("prompt")
|
||||
self.response_translator = TranslatorRegistry.get_response_translator("prompt")
|
||||
|
||||
def to_request(self, body):
|
||||
return PromptRequest(
|
||||
id=body["id"],
|
||||
terms={
|
||||
k: json.dumps(v)
|
||||
for k, v in body["variables"].items()
|
||||
}
|
||||
)
|
||||
return self.request_translator.to_pulsar(body)
|
||||
|
||||
def from_response(self, message):
|
||||
if message.object:
|
||||
return {
|
||||
"object": message.object
|
||||
}, True
|
||||
else:
|
||||
return {
|
||||
"text": message.text
|
||||
}, True
|
||||
return self.response_translator.from_response_with_completion(message)
|
||||
|
||||
|
|
|
|||
|
|
@ -3,6 +3,13 @@ import base64
|
|||
|
||||
from ... schema import Value, Triple, DocumentMetadata, ProcessingMetadata
|
||||
|
||||
# DEPRECATED: These functions have been moved to trustgraph.base.messaging.translators
|
||||
# Use the new messaging translation system instead for consistency and reusability.
|
||||
# Examples:
|
||||
# from trustgraph.base.messaging.translators.primitives import ValueTranslator
|
||||
# value_translator = ValueTranslator()
|
||||
# pulsar_value = value_translator.to_pulsar({"v": "example", "e": True})
|
||||
|
||||
def to_value(x):
|
||||
return Value(value=x["v"], is_uri=x["e"])
|
||||
|
||||
|
|
|
|||
|
|
@ -2,9 +2,9 @@
|
|||
import base64
|
||||
|
||||
from ... schema import TextDocument, Metadata
|
||||
from .... base.messaging import TranslatorRegistry
|
||||
|
||||
from . sender import ServiceSender
|
||||
from . serialize import to_subgraph
|
||||
|
||||
class TextLoad(ServiceSender):
|
||||
def __init__(self, pulsar_client, queue):
|
||||
|
|
@ -15,30 +15,9 @@ class TextLoad(ServiceSender):
|
|||
schema = TextDocument,
|
||||
)
|
||||
|
||||
self.translator = TranslatorRegistry.get_request_translator("text-document")
|
||||
|
||||
def to_request(self, body):
|
||||
|
||||
if "metadata" in body:
|
||||
metadata = to_subgraph(body["metadata"])
|
||||
else:
|
||||
metadata = []
|
||||
|
||||
if "charset" in body:
|
||||
charset = body["charset"]
|
||||
else:
|
||||
charset = "utf-8"
|
||||
|
||||
# Text is base64 encoded
|
||||
text = base64.b64decode(body["text"]).decode(charset)
|
||||
|
||||
print("Text document received")
|
||||
|
||||
return TextDocument(
|
||||
metadata=Metadata(
|
||||
id=body.get("id"),
|
||||
metadata=metadata,
|
||||
user=body.get("user", "trustgraph"),
|
||||
collection=body.get("collection", "default"),
|
||||
),
|
||||
text=text,
|
||||
)
|
||||
return self.translator.to_pulsar(body)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue