mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-04-25 00:16:23 +02:00
Feature/config service (#332)
Configuration service provides an API to change configuration. Complete configuration is pushed down a config queue so that users have a complete copy of config object.
This commit is contained in:
parent
21bda863a7
commit
fa09dc319e
17 changed files with 1002 additions and 5 deletions
248
test-api/test-config-api
Executable file
248
test-api/test-config-api
Executable file
|
|
@ -0,0 +1,248 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
import requests
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
|
||||||
|
url = "http://localhost:8088/api/v1/"
|
||||||
|
|
||||||
|
############################################################################
|
||||||
|
|
||||||
|
input = {
|
||||||
|
"operation": "config"
|
||||||
|
}
|
||||||
|
|
||||||
|
resp = requests.post(
|
||||||
|
f"{url}config",
|
||||||
|
json=input,
|
||||||
|
)
|
||||||
|
|
||||||
|
if resp.status_code != 200:
|
||||||
|
raise RuntimeError(f"Status code: {resp.status_code}")
|
||||||
|
|
||||||
|
resp = resp.json()
|
||||||
|
|
||||||
|
if "error" in resp:
|
||||||
|
print(f"Error: {resp['error']}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
print(json.dumps(resp, indent=4))
|
||||||
|
|
||||||
|
############################################################################
|
||||||
|
|
||||||
|
input = {
|
||||||
|
"operation": "put",
|
||||||
|
"values": [
|
||||||
|
{
|
||||||
|
"type": "test",
|
||||||
|
"key": "key1",
|
||||||
|
"value": "value1"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "test",
|
||||||
|
"key": "key2",
|
||||||
|
"value": "value2"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
resp = requests.post(
|
||||||
|
f"{url}config",
|
||||||
|
json=input,
|
||||||
|
)
|
||||||
|
|
||||||
|
if resp.status_code != 200:
|
||||||
|
raise RuntimeError(f"Status code: {resp.status_code}")
|
||||||
|
|
||||||
|
resp = resp.json()
|
||||||
|
|
||||||
|
if "error" in resp:
|
||||||
|
print(f"Error: {resp['error']}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
print(json.dumps(resp, indent=4))
|
||||||
|
|
||||||
|
############################################################################
|
||||||
|
|
||||||
|
input = {
|
||||||
|
"operation": "put",
|
||||||
|
"values": [
|
||||||
|
{
|
||||||
|
"type": "test",
|
||||||
|
"key": "key3",
|
||||||
|
"value": "testing 1 2 3"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
resp = requests.post(
|
||||||
|
f"{url}config",
|
||||||
|
json=input,
|
||||||
|
)
|
||||||
|
|
||||||
|
if resp.status_code != 200:
|
||||||
|
raise RuntimeError(f"Status code: {resp.status_code}")
|
||||||
|
|
||||||
|
resp = resp.json()
|
||||||
|
|
||||||
|
if "error" in resp:
|
||||||
|
print(f"Error: {resp['error']}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
print(json.dumps(resp, indent=4))
|
||||||
|
|
||||||
|
############################################################################
|
||||||
|
|
||||||
|
input = {
|
||||||
|
"operation": "get",
|
||||||
|
"keys": [
|
||||||
|
{
|
||||||
|
"type": "test",
|
||||||
|
"key": "key2"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "test",
|
||||||
|
"key": "key3"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
resp = requests.post(
|
||||||
|
f"{url}config",
|
||||||
|
json=input,
|
||||||
|
)
|
||||||
|
|
||||||
|
if resp.status_code != 200:
|
||||||
|
raise RuntimeError(f"Status code: {resp.status_code}")
|
||||||
|
|
||||||
|
resp = resp.json()
|
||||||
|
|
||||||
|
if "error" in resp:
|
||||||
|
print(f"Error: {resp['error']}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
print(json.dumps(resp, indent=4))
|
||||||
|
|
||||||
|
############################################################################
|
||||||
|
|
||||||
|
input = {
|
||||||
|
"operation": "config"
|
||||||
|
}
|
||||||
|
|
||||||
|
resp = requests.post(
|
||||||
|
f"{url}config",
|
||||||
|
json=input,
|
||||||
|
)
|
||||||
|
|
||||||
|
if resp.status_code != 200:
|
||||||
|
raise RuntimeError(f"Status code: {resp.status_code}")
|
||||||
|
|
||||||
|
resp = resp.json()
|
||||||
|
|
||||||
|
if "error" in resp:
|
||||||
|
print(f"Error: {resp['error']}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
print(json.dumps(resp, indent=4))
|
||||||
|
|
||||||
|
############################################################################
|
||||||
|
|
||||||
|
input = {
|
||||||
|
"operation": "list",
|
||||||
|
"type": "test"
|
||||||
|
}
|
||||||
|
|
||||||
|
resp = requests.post(
|
||||||
|
f"{url}config",
|
||||||
|
json=input,
|
||||||
|
)
|
||||||
|
|
||||||
|
if resp.status_code != 200:
|
||||||
|
raise RuntimeError(f"Status code: {resp.status_code}")
|
||||||
|
|
||||||
|
resp = resp.json()
|
||||||
|
|
||||||
|
if "error" in resp:
|
||||||
|
print(f"Error: {resp['error']}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
print(json.dumps(resp, indent=4))
|
||||||
|
|
||||||
|
############################################################################
|
||||||
|
|
||||||
|
input = {
|
||||||
|
"operation": "getvalues",
|
||||||
|
"type": "test"
|
||||||
|
}
|
||||||
|
|
||||||
|
resp = requests.post(
|
||||||
|
f"{url}config",
|
||||||
|
json=input,
|
||||||
|
)
|
||||||
|
|
||||||
|
if resp.status_code != 200:
|
||||||
|
raise RuntimeError(f"Status code: {resp.status_code}")
|
||||||
|
|
||||||
|
resp = resp.json()
|
||||||
|
|
||||||
|
if "error" in resp:
|
||||||
|
print(f"Error: {resp['error']}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
print(json.dumps(resp, indent=4))
|
||||||
|
|
||||||
|
############################################################################
|
||||||
|
|
||||||
|
input = {
|
||||||
|
"operation": "delete",
|
||||||
|
"keys": [
|
||||||
|
{
|
||||||
|
"type": "test",
|
||||||
|
"key": "key1"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "test",
|
||||||
|
"key": "key3"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
resp = requests.post(
|
||||||
|
f"{url}config",
|
||||||
|
json=input,
|
||||||
|
)
|
||||||
|
|
||||||
|
if resp.status_code != 200:
|
||||||
|
raise RuntimeError(f"Status code: {resp.status_code}")
|
||||||
|
|
||||||
|
resp = resp.json()
|
||||||
|
|
||||||
|
if "error" in resp:
|
||||||
|
print(f"Error: {resp['error']}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
print(json.dumps(resp, indent=4))
|
||||||
|
|
||||||
|
############################################################################
|
||||||
|
|
||||||
|
input = {
|
||||||
|
"operation": "config"
|
||||||
|
}
|
||||||
|
|
||||||
|
resp = requests.post(
|
||||||
|
f"{url}config",
|
||||||
|
json=input,
|
||||||
|
)
|
||||||
|
|
||||||
|
if resp.status_code != 200:
|
||||||
|
raise RuntimeError(f"Status code: {resp.status_code}")
|
||||||
|
|
||||||
|
resp = resp.json()
|
||||||
|
|
||||||
|
if "error" in resp:
|
||||||
|
print(f"Error: {resp['error']}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
print(json.dumps(resp, indent=4))
|
||||||
|
|
||||||
|
############################################################################
|
||||||
11
tests/test-get-config
Executable file
11
tests/test-get-config
Executable file
|
|
@ -0,0 +1,11 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
import pulsar
|
||||||
|
from trustgraph.clients.config_client import ConfigClient
|
||||||
|
|
||||||
|
cli = ConfigClient(pulsar_host="pulsar://localhost:6650")
|
||||||
|
|
||||||
|
resp = cli.request_config()
|
||||||
|
|
||||||
|
print(resp)
|
||||||
|
|
||||||
|
|
@ -382,3 +382,33 @@ class Api:
|
||||||
if resp.status_code != 200:
|
if resp.status_code != 200:
|
||||||
raise ProtocolException(f"Status code {resp.status_code}")
|
raise ProtocolException(f"Status code {resp.status_code}")
|
||||||
|
|
||||||
|
def get_config(self):
|
||||||
|
|
||||||
|
# The input consists of system and prompt strings
|
||||||
|
input = {
|
||||||
|
"operation": "config"
|
||||||
|
}
|
||||||
|
|
||||||
|
url = f"{self.url}config"
|
||||||
|
|
||||||
|
# Invoke the API, input is passed as JSON
|
||||||
|
resp = requests.post(url, json=input)
|
||||||
|
|
||||||
|
# Should be a 200 status code
|
||||||
|
if resp.status_code != 200:
|
||||||
|
raise ProtocolException(f"Status code {resp.status_code}")
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Parse the response as JSON
|
||||||
|
object = resp.json()
|
||||||
|
except:
|
||||||
|
raise ProtocolException(f"Expected JSON response")
|
||||||
|
|
||||||
|
self.check_error(resp)
|
||||||
|
|
||||||
|
try:
|
||||||
|
return object["config"], object["version"]
|
||||||
|
except:
|
||||||
|
raise ProtocolException(f"Response not formatted correctly")
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -28,6 +28,7 @@ class BaseClient:
|
||||||
output_schema=None,
|
output_schema=None,
|
||||||
pulsar_host="pulsar://pulsar:6650",
|
pulsar_host="pulsar://pulsar:6650",
|
||||||
pulsar_api_key=None,
|
pulsar_api_key=None,
|
||||||
|
listener=None,
|
||||||
):
|
):
|
||||||
|
|
||||||
if input_queue == None: raise RuntimeError("Need input_queue")
|
if input_queue == None: raise RuntimeError("Need input_queue")
|
||||||
|
|
@ -41,14 +42,16 @@ class BaseClient:
|
||||||
if pulsar_api_key:
|
if pulsar_api_key:
|
||||||
auth = pulsar.AuthenticationToken(pulsar_api_key)
|
auth = pulsar.AuthenticationToken(pulsar_api_key)
|
||||||
self.client = pulsar.Client(
|
self.client = pulsar.Client(
|
||||||
pulsar_host,
|
pulsar_host,
|
||||||
logger=pulsar.ConsoleLogger(log_level),
|
logger=pulsar.ConsoleLogger(log_level),
|
||||||
authentication=auth,
|
authentication=auth,
|
||||||
|
listener=listener,
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
self.client = pulsar.Client(
|
self.client = pulsar.Client(
|
||||||
pulsar_host,
|
pulsar_host,
|
||||||
logger=pulsar.ConsoleLogger(log_level)
|
logger=pulsar.ConsoleLogger(log_level),
|
||||||
|
listener_name=listener,
|
||||||
)
|
)
|
||||||
|
|
||||||
self.producer = self.client.create_producer(
|
self.producer = self.client.create_producer(
|
||||||
|
|
|
||||||
161
trustgraph-base/trustgraph/clients/config_client.py
Normal file
161
trustgraph-base/trustgraph/clients/config_client.py
Normal file
|
|
@ -0,0 +1,161 @@
|
||||||
|
|
||||||
|
import _pulsar
|
||||||
|
import json
|
||||||
|
import dataclasses
|
||||||
|
|
||||||
|
from .. schema import ConfigRequest, ConfigResponse, ConfigKey, ConfigValue
|
||||||
|
from .. schema import config_request_queue
|
||||||
|
from .. schema import config_response_queue
|
||||||
|
from . base import BaseClient
|
||||||
|
|
||||||
|
# Ugly
|
||||||
|
ERROR=_pulsar.LoggerLevel.Error
|
||||||
|
WARN=_pulsar.LoggerLevel.Warn
|
||||||
|
INFO=_pulsar.LoggerLevel.Info
|
||||||
|
DEBUG=_pulsar.LoggerLevel.Debug
|
||||||
|
|
||||||
|
@dataclasses.dataclass
|
||||||
|
class Definition:
|
||||||
|
name: str
|
||||||
|
definition: str
|
||||||
|
|
||||||
|
@dataclasses.dataclass
|
||||||
|
class Relationship:
|
||||||
|
s: str
|
||||||
|
p: str
|
||||||
|
o: str
|
||||||
|
o_entity: str
|
||||||
|
|
||||||
|
@dataclasses.dataclass
|
||||||
|
class Topic:
|
||||||
|
name: str
|
||||||
|
definition: str
|
||||||
|
|
||||||
|
class ConfigClient(BaseClient):
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self, log_level=ERROR,
|
||||||
|
subscriber=None,
|
||||||
|
input_queue=None,
|
||||||
|
output_queue=None,
|
||||||
|
pulsar_host="pulsar://pulsar:6650",
|
||||||
|
listener=None,
|
||||||
|
pulsar_api_key=None,
|
||||||
|
):
|
||||||
|
|
||||||
|
if input_queue == None:
|
||||||
|
input_queue = config_request_queue
|
||||||
|
|
||||||
|
if output_queue == None:
|
||||||
|
output_queue = config_response_queue
|
||||||
|
|
||||||
|
super(ConfigClient, self).__init__(
|
||||||
|
log_level=log_level,
|
||||||
|
subscriber=subscriber,
|
||||||
|
input_queue=input_queue,
|
||||||
|
output_queue=output_queue,
|
||||||
|
pulsar_host=pulsar_host,
|
||||||
|
pulsar_api_key=pulsar_api_key,
|
||||||
|
input_schema=ConfigRequest,
|
||||||
|
output_schema=ConfigResponse,
|
||||||
|
listener=listener,
|
||||||
|
)
|
||||||
|
|
||||||
|
def request_get(self, keys, timeout=300):
|
||||||
|
|
||||||
|
resp = self.call(
|
||||||
|
id=id,
|
||||||
|
operation="get",
|
||||||
|
keys=[
|
||||||
|
ConfigKey(
|
||||||
|
type = k["type"],
|
||||||
|
key = k["key"]
|
||||||
|
)
|
||||||
|
for k in keys
|
||||||
|
],
|
||||||
|
timeout=timeout
|
||||||
|
)
|
||||||
|
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
"type": v.type,
|
||||||
|
"key": v.key,
|
||||||
|
"value": v.value
|
||||||
|
}
|
||||||
|
for v in resp.values
|
||||||
|
]
|
||||||
|
|
||||||
|
def request_list(self, type, timeout=300):
|
||||||
|
|
||||||
|
resp = self.call(
|
||||||
|
id=id,
|
||||||
|
operation="list",
|
||||||
|
type=type,
|
||||||
|
timeout=timeout
|
||||||
|
)
|
||||||
|
|
||||||
|
return resp.directory
|
||||||
|
|
||||||
|
def request_getvalues(self, type, timeout=300):
|
||||||
|
|
||||||
|
resp = self.call(
|
||||||
|
id=id,
|
||||||
|
operation="getvalues",
|
||||||
|
type=type,
|
||||||
|
timeout=timeout
|
||||||
|
)
|
||||||
|
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
"type": v.type,
|
||||||
|
"key": v.key,
|
||||||
|
"value": v.value
|
||||||
|
}
|
||||||
|
for v in resp.values
|
||||||
|
]
|
||||||
|
|
||||||
|
def request_delete(self, keys, timeout=300):
|
||||||
|
|
||||||
|
resp = self.call(
|
||||||
|
id=id,
|
||||||
|
operation="delete",
|
||||||
|
keys=[
|
||||||
|
ConfigKey(
|
||||||
|
type = k["type"],
|
||||||
|
key = k["key"]
|
||||||
|
)
|
||||||
|
for k in keys
|
||||||
|
],
|
||||||
|
timeout=timeout
|
||||||
|
)
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
def request_put(self, value, timeout=300):
|
||||||
|
|
||||||
|
resp = self.call(
|
||||||
|
id=id,
|
||||||
|
operation="put",
|
||||||
|
values=[
|
||||||
|
ConfigValue(
|
||||||
|
type = k["type"],
|
||||||
|
key = k["key"],
|
||||||
|
value = k["value"]
|
||||||
|
)
|
||||||
|
for k in keys
|
||||||
|
],
|
||||||
|
timeout=timeout
|
||||||
|
)
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
def request_config(self, timeout=300):
|
||||||
|
|
||||||
|
resp = self.call(
|
||||||
|
id=id,
|
||||||
|
operation="config",
|
||||||
|
timeout=timeout
|
||||||
|
)
|
||||||
|
|
||||||
|
return resp.config
|
||||||
|
|
||||||
|
|
@ -11,5 +11,6 @@ from . metadata import *
|
||||||
from . agent import *
|
from . agent import *
|
||||||
from . lookup import *
|
from . lookup import *
|
||||||
from . library import *
|
from . library import *
|
||||||
|
from . config import *
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
71
trustgraph-base/trustgraph/schema/config.py
Normal file
71
trustgraph-base/trustgraph/schema/config.py
Normal file
|
|
@ -0,0 +1,71 @@
|
||||||
|
|
||||||
|
from pulsar.schema import Record, Bytes, String, Boolean, Array, Map, Integer
|
||||||
|
|
||||||
|
from . topic import topic
|
||||||
|
from . types import Error, RowSchema
|
||||||
|
|
||||||
|
############################################################################
|
||||||
|
|
||||||
|
# Config service:
|
||||||
|
# get(keys) -> (version, values)
|
||||||
|
# list(type) -> (version, values)
|
||||||
|
# getvalues(type) -> (version, values)
|
||||||
|
# put(values) -> ()
|
||||||
|
# delete(keys) -> ()
|
||||||
|
# config() -> (version, config)
|
||||||
|
class ConfigKey(Record):
|
||||||
|
type = String()
|
||||||
|
key = String()
|
||||||
|
|
||||||
|
class ConfigValue(Record):
|
||||||
|
type = String()
|
||||||
|
key = String()
|
||||||
|
value = String()
|
||||||
|
|
||||||
|
# Prompt services, abstract the prompt generation
|
||||||
|
class ConfigRequest(Record):
|
||||||
|
|
||||||
|
operation = String() # get, list, getvalues, delete, put, config
|
||||||
|
|
||||||
|
# get, delete
|
||||||
|
keys = Array(ConfigKey())
|
||||||
|
|
||||||
|
# list, getvalues
|
||||||
|
type = String()
|
||||||
|
|
||||||
|
# put
|
||||||
|
values = Array(ConfigValue())
|
||||||
|
|
||||||
|
class ConfigResponse(Record):
|
||||||
|
|
||||||
|
# get, list, getvalues, config
|
||||||
|
version = Integer()
|
||||||
|
|
||||||
|
# get, getvalues
|
||||||
|
values = Array(ConfigValue())
|
||||||
|
|
||||||
|
# list
|
||||||
|
directory = Array(String())
|
||||||
|
|
||||||
|
# config
|
||||||
|
config = Map(Map(String()))
|
||||||
|
|
||||||
|
# Everything
|
||||||
|
error = Error()
|
||||||
|
|
||||||
|
class ConfigPush(Record):
|
||||||
|
version = Integer()
|
||||||
|
config = Map(Map(String()))
|
||||||
|
|
||||||
|
config_request_queue = topic(
|
||||||
|
'config', kind='non-persistent', namespace='request'
|
||||||
|
)
|
||||||
|
config_response_queue = topic(
|
||||||
|
'config', kind='non-persistent', namespace='response'
|
||||||
|
)
|
||||||
|
config_push_queue = topic(
|
||||||
|
'config', kind='persistent', namespace='config'
|
||||||
|
)
|
||||||
|
|
||||||
|
############################################################################
|
||||||
|
|
||||||
|
|
@ -79,6 +79,13 @@ def init(url, tenant="tg"):
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
ensure_namespace(url, tenant, "config", {
|
||||||
|
"retention_policies": {
|
||||||
|
"retentionSizeInMB": 50,
|
||||||
|
"retentionTimeInMinutes": -1,
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
|
|
||||||
parser = argparse.ArgumentParser(
|
parser = argparse.ArgumentParser(
|
||||||
|
|
|
||||||
49
trustgraph-cli/scripts/tg-show-config
Executable file
49
trustgraph-cli/scripts/tg-show-config
Executable file
|
|
@ -0,0 +1,49 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
"""
|
||||||
|
Dumps out the current configuration
|
||||||
|
"""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import os
|
||||||
|
from trustgraph.api import Api
|
||||||
|
import json
|
||||||
|
|
||||||
|
default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
|
||||||
|
|
||||||
|
def show_config(url):
|
||||||
|
|
||||||
|
api = Api(url)
|
||||||
|
|
||||||
|
config, version = api.get_config()
|
||||||
|
|
||||||
|
print("Version:", version)
|
||||||
|
print(json.dumps(config, indent=4))
|
||||||
|
|
||||||
|
def main():
|
||||||
|
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
prog='tg-show-config',
|
||||||
|
description=__doc__,
|
||||||
|
)
|
||||||
|
|
||||||
|
parser.add_argument(
|
||||||
|
'-u', '--api-url',
|
||||||
|
default=default_url,
|
||||||
|
help=f'API URL (default: {default_url})',
|
||||||
|
)
|
||||||
|
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
try:
|
||||||
|
|
||||||
|
show_config(
|
||||||
|
url=args.api_url,
|
||||||
|
)
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
|
||||||
|
print("Exception:", e, flush=True)
|
||||||
|
|
||||||
|
main()
|
||||||
|
|
||||||
|
|
@ -62,5 +62,6 @@ setuptools.setup(
|
||||||
"scripts/tg-processor-state",
|
"scripts/tg-processor-state",
|
||||||
"scripts/tg-save-kg-core",
|
"scripts/tg-save-kg-core",
|
||||||
"scripts/tg-save-doc-embeds",
|
"scripts/tg-save-doc-embeds",
|
||||||
|
"scripts/tg-show-config",
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
|
||||||
6
trustgraph-flow/scripts/config-svc
Executable file
6
trustgraph-flow/scripts/config-svc
Executable file
|
|
@ -0,0 +1,6 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
from trustgraph.config.service import run
|
||||||
|
|
||||||
|
run()
|
||||||
|
|
||||||
|
|
@ -73,6 +73,7 @@ setuptools.setup(
|
||||||
"scripts/api-gateway",
|
"scripts/api-gateway",
|
||||||
"scripts/chunker-recursive",
|
"scripts/chunker-recursive",
|
||||||
"scripts/chunker-token",
|
"scripts/chunker-token",
|
||||||
|
"scripts/config-svc",
|
||||||
"scripts/de-query-milvus",
|
"scripts/de-query-milvus",
|
||||||
"scripts/de-query-pinecone",
|
"scripts/de-query-pinecone",
|
||||||
"scripts/de-query-qdrant",
|
"scripts/de-query-qdrant",
|
||||||
|
|
|
||||||
3
trustgraph-flow/trustgraph/config/service/__init__.py
Normal file
3
trustgraph-flow/trustgraph/config/service/__init__.py
Normal file
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
from . service import *
|
||||||
|
|
||||||
7
trustgraph-flow/trustgraph/config/service/__main__.py
Normal file
7
trustgraph-flow/trustgraph/config/service/__main__.py
Normal file
|
|
@ -0,0 +1,7 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
from . service import run
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
run()
|
||||||
|
|
||||||
312
trustgraph-flow/trustgraph/config/service/service.py
Normal file
312
trustgraph-flow/trustgraph/config/service/service.py
Normal file
|
|
@ -0,0 +1,312 @@
|
||||||
|
|
||||||
|
"""
|
||||||
|
Config service. Fetchs an extract from the Wikipedia page
|
||||||
|
using the API.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from pulsar.schema import JsonSchema
|
||||||
|
|
||||||
|
from trustgraph.schema import ConfigRequest, ConfigResponse, ConfigPush
|
||||||
|
from trustgraph.schema import ConfigValue, Error
|
||||||
|
from trustgraph.schema import config_request_queue, config_response_queue
|
||||||
|
from trustgraph.schema import config_push_queue
|
||||||
|
from trustgraph.log_level import LogLevel
|
||||||
|
from trustgraph.base import ConsumerProducer
|
||||||
|
|
||||||
|
module = ".".join(__name__.split(".")[1:-1])
|
||||||
|
|
||||||
|
default_input_queue = config_request_queue
|
||||||
|
default_output_queue = config_response_queue
|
||||||
|
default_push_queue = config_push_queue
|
||||||
|
default_subscriber = module
|
||||||
|
|
||||||
|
# This behaves just like a dict, should be easier to add persistent storage
|
||||||
|
# later
|
||||||
|
|
||||||
|
class ConfigurationItems(dict):
|
||||||
|
pass
|
||||||
|
|
||||||
|
class Configuration(dict):
|
||||||
|
|
||||||
|
def __getitem__(self, key):
|
||||||
|
if key not in self:
|
||||||
|
self[key] = ConfigurationItems()
|
||||||
|
return dict.__getitem__(self, key)
|
||||||
|
|
||||||
|
class Processor(ConsumerProducer):
|
||||||
|
|
||||||
|
def __init__(self, **params):
|
||||||
|
|
||||||
|
input_queue = params.get("input_queue", default_input_queue)
|
||||||
|
output_queue = params.get("output_queue", default_output_queue)
|
||||||
|
push_queue = params.get("push_queue", default_push_queue)
|
||||||
|
subscriber = params.get("subscriber", default_subscriber)
|
||||||
|
|
||||||
|
super(Processor, self).__init__(
|
||||||
|
**params | {
|
||||||
|
"input_queue": input_queue,
|
||||||
|
"output_queue": output_queue,
|
||||||
|
"push_queue": output_queue,
|
||||||
|
"subscriber": subscriber,
|
||||||
|
"input_schema": ConfigRequest,
|
||||||
|
"output_schema": ConfigResponse,
|
||||||
|
"push_schema": ConfigPush,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
self.push_prod = self.client.create_producer(
|
||||||
|
topic=push_queue,
|
||||||
|
schema=JsonSchema(ConfigPush),
|
||||||
|
)
|
||||||
|
|
||||||
|
# FIXME: The state is held internally. This only works if there's
|
||||||
|
# one config service. Should be more than one, and use a
|
||||||
|
# back-end state store.
|
||||||
|
self.config = Configuration()
|
||||||
|
|
||||||
|
# Version counter
|
||||||
|
self.version = 0
|
||||||
|
|
||||||
|
async def start(self):
|
||||||
|
await self.push()
|
||||||
|
|
||||||
|
async def handle_get(self, v, id):
|
||||||
|
|
||||||
|
for k in v.keys:
|
||||||
|
if k.type not in self.config or k.key not in self.config[k.type]:
|
||||||
|
return ConfigResponse(
|
||||||
|
version = None,
|
||||||
|
values = None,
|
||||||
|
directory = None,
|
||||||
|
config = None,
|
||||||
|
error = Error(
|
||||||
|
code = "key-error",
|
||||||
|
message = f"Key error"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
values = [
|
||||||
|
ConfigValue(
|
||||||
|
type = k.type,
|
||||||
|
key = k.key,
|
||||||
|
value = self.config[k.type][k.key]
|
||||||
|
)
|
||||||
|
for k in v.keys
|
||||||
|
]
|
||||||
|
|
||||||
|
return ConfigResponse(
|
||||||
|
version = self.version,
|
||||||
|
values = values,
|
||||||
|
directory = None,
|
||||||
|
config = None,
|
||||||
|
error = None,
|
||||||
|
)
|
||||||
|
|
||||||
|
async def handle_list(self, v, id):
|
||||||
|
|
||||||
|
if v.type not in self.config:
|
||||||
|
|
||||||
|
return ConfigResponse(
|
||||||
|
version = None,
|
||||||
|
values = None,
|
||||||
|
directory = None,
|
||||||
|
config = None,
|
||||||
|
error = Error(
|
||||||
|
code="key-error",
|
||||||
|
message="No such type",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
return ConfigResponse(
|
||||||
|
version = self.version,
|
||||||
|
values = None,
|
||||||
|
directory = list(self.config[v.type].keys()),
|
||||||
|
config = None,
|
||||||
|
error = None,
|
||||||
|
)
|
||||||
|
|
||||||
|
async def handle_getvalues(self, v, id):
|
||||||
|
|
||||||
|
if v.type not in self.config:
|
||||||
|
|
||||||
|
return ConfigResponse(
|
||||||
|
version = None,
|
||||||
|
values = None,
|
||||||
|
directory = None,
|
||||||
|
config = None,
|
||||||
|
error = Error(
|
||||||
|
code = "key-error",
|
||||||
|
message = f"Key error"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
values = [
|
||||||
|
ConfigValue(
|
||||||
|
type = v.type,
|
||||||
|
key = k,
|
||||||
|
value = self.config[v.type][k],
|
||||||
|
)
|
||||||
|
for k in self.config[v.type]
|
||||||
|
]
|
||||||
|
|
||||||
|
return ConfigResponse(
|
||||||
|
version = self.version,
|
||||||
|
values = values,
|
||||||
|
directory = None,
|
||||||
|
config = None,
|
||||||
|
error = None,
|
||||||
|
)
|
||||||
|
|
||||||
|
async def handle_delete(self, v, id):
|
||||||
|
|
||||||
|
for k in v.keys:
|
||||||
|
if k.type not in self.config or k.key not in self.config[k.type]:
|
||||||
|
return ConfigResponse(
|
||||||
|
version = None,
|
||||||
|
values = None,
|
||||||
|
directory = None,
|
||||||
|
config = None,
|
||||||
|
error = Error(
|
||||||
|
code = "key-error",
|
||||||
|
message = f"Key error"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
for k in v.keys:
|
||||||
|
del self.config[k.type][k.key]
|
||||||
|
|
||||||
|
self.version += 1
|
||||||
|
|
||||||
|
await self.push()
|
||||||
|
|
||||||
|
return ConfigResponse(
|
||||||
|
version = None,
|
||||||
|
value = None,
|
||||||
|
directory = None,
|
||||||
|
values = None,
|
||||||
|
config = None,
|
||||||
|
error = None,
|
||||||
|
)
|
||||||
|
|
||||||
|
async def handle_put(self, v, id):
|
||||||
|
|
||||||
|
for k in v.values:
|
||||||
|
self.config[k.type][k.key] = k.value
|
||||||
|
|
||||||
|
self.version += 1
|
||||||
|
|
||||||
|
await self.push()
|
||||||
|
|
||||||
|
return ConfigResponse(
|
||||||
|
version = None,
|
||||||
|
value = None,
|
||||||
|
directory = None,
|
||||||
|
values = None,
|
||||||
|
error = None,
|
||||||
|
)
|
||||||
|
|
||||||
|
async def handle_config(self, v, id):
|
||||||
|
|
||||||
|
return ConfigResponse(
|
||||||
|
version = self.version,
|
||||||
|
value = None,
|
||||||
|
directory = None,
|
||||||
|
values = None,
|
||||||
|
config = self.config,
|
||||||
|
error = None,
|
||||||
|
)
|
||||||
|
|
||||||
|
async def push(self):
|
||||||
|
|
||||||
|
resp = ConfigPush(
|
||||||
|
version = self.version,
|
||||||
|
value = None,
|
||||||
|
directory = None,
|
||||||
|
values = None,
|
||||||
|
config = self.config,
|
||||||
|
error = None,
|
||||||
|
)
|
||||||
|
self.push_prod.send(resp)
|
||||||
|
print("Pushed.")
|
||||||
|
|
||||||
|
async def handle(self, msg):
|
||||||
|
|
||||||
|
v = msg.value()
|
||||||
|
|
||||||
|
# Sender-produced ID
|
||||||
|
id = msg.properties()["id"]
|
||||||
|
|
||||||
|
print(f"Handling {id}...", flush=True)
|
||||||
|
|
||||||
|
try:
|
||||||
|
|
||||||
|
if v.operation == "get":
|
||||||
|
|
||||||
|
resp = await self.handle_get(v, id)
|
||||||
|
|
||||||
|
elif v.operation == "list":
|
||||||
|
|
||||||
|
resp = await self.handle_list(v, id)
|
||||||
|
|
||||||
|
elif v.operation == "getvalues":
|
||||||
|
|
||||||
|
resp = await self.handle_getvalues(v, id)
|
||||||
|
|
||||||
|
elif v.operation == "delete":
|
||||||
|
|
||||||
|
resp = await self.handle_delete(v, id)
|
||||||
|
|
||||||
|
elif v.operation == "put":
|
||||||
|
|
||||||
|
resp = await self.handle_put(v, id)
|
||||||
|
|
||||||
|
elif v.operation == "config":
|
||||||
|
|
||||||
|
resp = await self.handle_config(v, id)
|
||||||
|
|
||||||
|
else:
|
||||||
|
|
||||||
|
resp = ConfigResponse(
|
||||||
|
value=None,
|
||||||
|
directory=None,
|
||||||
|
values=None,
|
||||||
|
error=Error(
|
||||||
|
code="bad-operation",
|
||||||
|
message="Bad operation"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
await self.send(resp, properties={"id": id})
|
||||||
|
|
||||||
|
self.consumer.acknowledge(msg)
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
|
||||||
|
resp = ConfigResponse(
|
||||||
|
error=Error(
|
||||||
|
type = "unexpected-error",
|
||||||
|
message = str(e),
|
||||||
|
),
|
||||||
|
text=None,
|
||||||
|
)
|
||||||
|
await self.send(resp, properties={"id": id})
|
||||||
|
self.consumer.acknowledge(msg)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def add_args(parser):
|
||||||
|
|
||||||
|
ConsumerProducer.add_args(
|
||||||
|
parser, default_input_queue, default_subscriber,
|
||||||
|
default_output_queue,
|
||||||
|
)
|
||||||
|
|
||||||
|
parser.add_argument(
|
||||||
|
'-q', '--push-queue',
|
||||||
|
default=default_push_queue,
|
||||||
|
help=f'Config push queue (default: {default_push_queue})'
|
||||||
|
)
|
||||||
|
|
||||||
|
def run():
|
||||||
|
|
||||||
|
Processor.launch(module, __doc__)
|
||||||
|
|
||||||
77
trustgraph-flow/trustgraph/gateway/config.py
Normal file
77
trustgraph-flow/trustgraph/gateway/config.py
Normal file
|
|
@ -0,0 +1,77 @@
|
||||||
|
|
||||||
|
from .. schema import ConfigRequest, ConfigResponse, ConfigKey, ConfigValue
|
||||||
|
from .. schema import config_request_queue
|
||||||
|
from .. schema import config_response_queue
|
||||||
|
|
||||||
|
from . endpoint import ServiceEndpoint
|
||||||
|
from . requestor import ServiceRequestor
|
||||||
|
|
||||||
|
class ConfigRequestor(ServiceRequestor):
|
||||||
|
def __init__(self, pulsar_client, timeout, auth):
|
||||||
|
|
||||||
|
super(ConfigRequestor, self).__init__(
|
||||||
|
pulsar_client=pulsar_client,
|
||||||
|
request_queue=config_request_queue,
|
||||||
|
response_queue=config_response_queue,
|
||||||
|
request_schema=ConfigRequest,
|
||||||
|
response_schema=ConfigResponse,
|
||||||
|
timeout=timeout,
|
||||||
|
)
|
||||||
|
|
||||||
|
def to_request(self, body):
|
||||||
|
|
||||||
|
if "keys" in body:
|
||||||
|
keys = [
|
||||||
|
ConfigKey(
|
||||||
|
type = k["type"],
|
||||||
|
key = k["key"],
|
||||||
|
)
|
||||||
|
for k in body["keys"]
|
||||||
|
]
|
||||||
|
else:
|
||||||
|
keys = None
|
||||||
|
|
||||||
|
if "values" in body:
|
||||||
|
values = [
|
||||||
|
ConfigValue(
|
||||||
|
type = v["type"],
|
||||||
|
key = v["key"],
|
||||||
|
value = v["value"],
|
||||||
|
)
|
||||||
|
for v in body["values"]
|
||||||
|
]
|
||||||
|
else:
|
||||||
|
values = None
|
||||||
|
|
||||||
|
return ConfigRequest(
|
||||||
|
operation = body.get("operation", None),
|
||||||
|
keys = keys,
|
||||||
|
type = body.get("type", None),
|
||||||
|
values = values
|
||||||
|
)
|
||||||
|
|
||||||
|
def from_response(self, message):
|
||||||
|
|
||||||
|
response = { }
|
||||||
|
|
||||||
|
if message.version:
|
||||||
|
response["version"] = message.version
|
||||||
|
|
||||||
|
if message.values:
|
||||||
|
response["values"] = [
|
||||||
|
{
|
||||||
|
"type": v.type,
|
||||||
|
"key": v.key,
|
||||||
|
"value": v.value,
|
||||||
|
}
|
||||||
|
for v in message.values
|
||||||
|
]
|
||||||
|
|
||||||
|
if message.directory:
|
||||||
|
response["directory"] = message.directory
|
||||||
|
|
||||||
|
if message.config:
|
||||||
|
response["config"] = message.config
|
||||||
|
|
||||||
|
return response, True
|
||||||
|
|
||||||
|
|
@ -38,6 +38,7 @@ from . agent import AgentRequestor
|
||||||
from . dbpedia import DbpediaRequestor
|
from . dbpedia import DbpediaRequestor
|
||||||
from . internet_search import InternetSearchRequestor
|
from . internet_search import InternetSearchRequestor
|
||||||
from . librarian import LibrarianRequestor
|
from . librarian import LibrarianRequestor
|
||||||
|
from . config import ConfigRequestor
|
||||||
from . triples_stream import TriplesStreamEndpoint
|
from . triples_stream import TriplesStreamEndpoint
|
||||||
from . graph_embeddings_stream import GraphEmbeddingsStreamEndpoint
|
from . graph_embeddings_stream import GraphEmbeddingsStreamEndpoint
|
||||||
from . document_embeddings_stream import DocumentEmbeddingsStreamEndpoint
|
from . document_embeddings_stream import DocumentEmbeddingsStreamEndpoint
|
||||||
|
|
@ -141,6 +142,10 @@ class Api:
|
||||||
pulsar_client=self.pulsar_client, timeout=self.timeout,
|
pulsar_client=self.pulsar_client, timeout=self.timeout,
|
||||||
auth = self.auth,
|
auth = self.auth,
|
||||||
),
|
),
|
||||||
|
"config": ConfigRequestor(
|
||||||
|
pulsar_client=self.pulsar_client, timeout=self.timeout,
|
||||||
|
auth = self.auth,
|
||||||
|
),
|
||||||
"encyclopedia": EncyclopediaRequestor(
|
"encyclopedia": EncyclopediaRequestor(
|
||||||
pulsar_client=self.pulsar_client, timeout=self.timeout,
|
pulsar_client=self.pulsar_client, timeout=self.timeout,
|
||||||
auth = self.auth,
|
auth = self.auth,
|
||||||
|
|
@ -199,6 +204,10 @@ class Api:
|
||||||
endpoint_path = "/api/v1/librarian", auth=self.auth,
|
endpoint_path = "/api/v1/librarian", auth=self.auth,
|
||||||
requestor = self.services["librarian"],
|
requestor = self.services["librarian"],
|
||||||
),
|
),
|
||||||
|
ServiceEndpoint(
|
||||||
|
endpoint_path = "/api/v1/config", auth=self.auth,
|
||||||
|
requestor = self.services["config"],
|
||||||
|
),
|
||||||
ServiceEndpoint(
|
ServiceEndpoint(
|
||||||
endpoint_path = "/api/v1/encyclopedia", auth=self.auth,
|
endpoint_path = "/api/v1/encyclopedia", auth=self.auth,
|
||||||
requestor = self.services["encyclopedia"],
|
requestor = self.services["encyclopedia"],
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue