Feature/config service (#332)

Configuration service provides an API to change configuration. Complete configuration is pushed down a config queue so that users have a complete copy of config object.

trustgraph-base/trustgraph/api/api.py

@@ -382,3 +382,33 @@ class Api:
         if resp.status_code != 200:
             raise ProtocolException(f"Status code {resp.status_code}")
 
+    def get_config(self):
+
+        # The input consists of system and prompt strings
+        input = {
+            "operation": "config"
+        }
+
+        url = f"{self.url}config"
+
+        # Invoke the API, input is passed as JSON
+        resp = requests.post(url, json=input)
+
+        # Should be a 200 status code
+        if resp.status_code != 200:
+            raise ProtocolException(f"Status code {resp.status_code}")
+
+        try:
+            # Parse the response as JSON
+            object = resp.json()
+        except:
+            raise ProtocolException(f"Expected JSON response")
+
+        self.check_error(resp)
+
+        try:
+            return object["config"], object["version"]
+        except:
+            raise ProtocolException(f"Response not formatted correctly")
+
+        

trustgraph-base/trustgraph/clients/base.py

@@ -28,6 +28,7 @@ class BaseClient:
             output_schema=None,
             pulsar_host="pulsar://pulsar:6650",
             pulsar_api_key=None,
+            listener=None,
     ):
 
         if input_queue == None: raise RuntimeError("Need input_queue")
@@ -41,14 +42,16 @@ class BaseClient:
         if pulsar_api_key:
             auth = pulsar.AuthenticationToken(pulsar_api_key)
             self.client = pulsar.Client(
-            pulsar_host,
-            logger=pulsar.ConsoleLogger(log_level),
-            authentication=auth,
+                pulsar_host,
+                logger=pulsar.ConsoleLogger(log_level),
+                authentication=auth,
+                listener=listener,
             )
         else:
             self.client = pulsar.Client(
-            pulsar_host,
-            logger=pulsar.ConsoleLogger(log_level)
+                pulsar_host,
+                logger=pulsar.ConsoleLogger(log_level),
+                listener_name=listener,
             )
 
         self.producer = self.client.create_producer(

trustgraph-base/trustgraph/clients/config_client.py

@@ -0,0 +1,161 @@
+
+import _pulsar
+import json
+import dataclasses
+
+from .. schema import ConfigRequest, ConfigResponse, ConfigKey, ConfigValue
+from .. schema import config_request_queue
+from .. schema import config_response_queue
+from . base import BaseClient
+
+# Ugly
+ERROR=_pulsar.LoggerLevel.Error
+WARN=_pulsar.LoggerLevel.Warn
+INFO=_pulsar.LoggerLevel.Info
+DEBUG=_pulsar.LoggerLevel.Debug
+
+@dataclasses.dataclass
+class Definition:
+    name: str
+    definition: str
+
+@dataclasses.dataclass
+class Relationship:
+    s: str
+    p: str
+    o: str
+    o_entity: str
+
+@dataclasses.dataclass
+class Topic:
+    name: str
+    definition: str
+
+class ConfigClient(BaseClient):
+
+    def __init__(
+            self, log_level=ERROR,
+            subscriber=None,
+            input_queue=None,
+            output_queue=None,
+            pulsar_host="pulsar://pulsar:6650",
+            listener=None,
+            pulsar_api_key=None,
+    ):
+
+        if input_queue == None:
+            input_queue = config_request_queue
+
+        if output_queue == None:
+            output_queue = config_response_queue
+
+        super(ConfigClient, self).__init__(
+            log_level=log_level,
+            subscriber=subscriber,
+            input_queue=input_queue,
+            output_queue=output_queue,
+            pulsar_host=pulsar_host,
+            pulsar_api_key=pulsar_api_key,
+            input_schema=ConfigRequest,
+            output_schema=ConfigResponse,
+            listener=listener,
+        )
+
+    def request_get(self, keys, timeout=300):
+
+        resp = self.call(
+            id=id,
+            operation="get",
+            keys=[
+                ConfigKey(
+                    type = k["type"],
+                    key = k["key"]
+                )
+                for k in keys
+            ],
+            timeout=timeout
+        )
+
+        return [
+            {
+                "type": v.type,
+                "key": v.key,
+                "value": v.value
+            }
+            for v in resp.values
+        ]
+
+    def request_list(self, type, timeout=300):
+
+        resp = self.call(
+            id=id,
+            operation="list",
+            type=type,
+            timeout=timeout
+        )
+
+        return resp.directory
+
+    def request_getvalues(self, type, timeout=300):
+
+        resp = self.call(
+            id=id,
+            operation="getvalues",
+            type=type,
+            timeout=timeout
+        )
+
+        return [
+            {
+                "type": v.type,
+                "key": v.key,
+                "value": v.value
+            }
+            for v in resp.values
+        ]
+
+    def request_delete(self, keys, timeout=300):
+
+        resp = self.call(
+            id=id,
+            operation="delete",
+            keys=[
+                ConfigKey(
+                    type = k["type"],
+                    key = k["key"]
+                )
+                for k in keys
+            ],
+            timeout=timeout
+        )
+
+        return None
+
+    def request_put(self, value, timeout=300):
+
+        resp = self.call(
+            id=id,
+            operation="put",
+            values=[
+                ConfigValue(
+                    type = k["type"],
+                    key = k["key"],
+                    value = k["value"]
+                )
+                for k in keys
+            ],
+            timeout=timeout
+        )
+
+        return None
+
+    def request_config(self, timeout=300):
+
+        resp = self.call(
+            id=id,
+            operation="config",
+            timeout=timeout
+        )
+
+        return resp.config
+

trustgraph-base/trustgraph/schema/__init__.py

@@ -11,5 +11,6 @@ from . metadata import *
 from . agent import *
 from . lookup import *
 from . library import *
+from . config import *
 
 

trustgraph-base/trustgraph/schema/config.py

@@ -0,0 +1,71 @@
+
+from pulsar.schema import Record, Bytes, String, Boolean, Array, Map, Integer
+
+from . topic import topic
+from . types import Error, RowSchema
+
+############################################################################
+
+# Config service:
+#   get(keys) -> (version, values)
+#   list(type) -> (version, values)
+#   getvalues(type) -> (version, values)
+#   put(values) -> ()
+#   delete(keys) -> ()
+#   config() -> (version, config)
+class ConfigKey(Record):
+    type = String()
+    key = String()
+
+class ConfigValue(Record):
+    type = String()
+    key = String()
+    value = String()
+
+# Prompt services, abstract the prompt generation
+class ConfigRequest(Record):
+
+    operation = String() # get, list, getvalues, delete, put, config
+
+    # get, delete
+    keys = Array(ConfigKey())
+
+    # list, getvalues
+    type = String()
+
+    # put
+    values = Array(ConfigValue())
+
+class ConfigResponse(Record):
+
+    # get, list, getvalues, config
+    version = Integer()
+
+    # get, getvalues
+    values = Array(ConfigValue())
+
+    # list
+    directory = Array(String())
+
+    # config
+    config = Map(Map(String()))
+
+    # Everything
+    error = Error()
+
+class ConfigPush(Record):
+    version = Integer()
+    config = Map(Map(String()))
+
+config_request_queue = topic(
+    'config', kind='non-persistent', namespace='request'
+)
+config_response_queue = topic(
+    'config', kind='non-persistent', namespace='response'
+)
+config_push_queue = topic(
+    'config', kind='persistent', namespace='config'
+)
+
+############################################################################
+