mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-04-26 00:46:22 +02:00
Feature/persist config (#370)
* Cassandra tables for config * Config is backed by Cassandra
This commit is contained in:
cybermaggedon
GitHub
parent
f7123ac57f
commit
4461d7b289
5 changed files with 516 additions and 103 deletions
|
|
@ -2,56 +2,93 @@
|
|||
from trustgraph.schema import ConfigResponse
|
||||
from trustgraph.schema import ConfigValue, Error
|
||||
|
||||
# This behaves just like a dict, should be easier to add persistent storage
|
||||
# later
|
||||
class ConfigurationItems(dict):
|
||||
pass
|
||||
from ... tables.config import ConfigTableStore
|
||||
|
||||
class Configuration(dict):
|
||||
class ConfigurationClass:
|
||||
|
||||
async def keys(self):
|
||||
return await self.table_store.get_keys(self.type)
|
||||
|
||||
async def values(self):
|
||||
vals = await self.table_store.get_values(self.type)
|
||||
return {
|
||||
v[0]: v[1]
|
||||
for v in vals
|
||||
}
|
||||
|
||||
async def get(self, key):
|
||||
return await self.table_store.get_value(self.type, key)
|
||||
|
||||
async def put(self, key, value):
|
||||
return await self.table_store.put_config(self.type, key, value)
|
||||
|
||||
async def delete(self, key):
|
||||
return await self.table_store.delete_key(self.type, key)
|
||||
|
||||
async def has(self, key):
|
||||
val = await self.table_store.get_value(self.type, key)
|
||||
return val is not None
|
||||
|
||||
class Configuration:
|
||||
|
||||
# FIXME: The state is held internally. This only works if there's
|
||||
# one config service. Should be more than one, and use a
|
||||
# back-end state store.
|
||||
|
||||
def __init__(self, push):
|
||||
# FIXME: This has state now, but does it address all of the above?
|
||||
# REVIEW: Above
|
||||
|
||||
# Version counter
|
||||
self.version = 0
|
||||
# FIXME: Some version vs config race conditions
|
||||
|
||||
def __init__(self, push, host, user, password, keyspace):
|
||||
|
||||
# External function to respond to update
|
||||
self.push = push
|
||||
|
||||
def __getitem__(self, key):
|
||||
if key not in self:
|
||||
self[key] = ConfigurationItems()
|
||||
return dict.__getitem__(self, key)
|
||||
self.table_store = ConfigTableStore(
|
||||
host, user, password, keyspace
|
||||
)
|
||||
|
||||
async def inc_version(self):
|
||||
await self.table_store.inc_version()
|
||||
|
||||
async def get_version(self):
|
||||
return await self.table_store.get_version()
|
||||
|
||||
def get(self, type):
|
||||
|
||||
c = ConfigurationClass()
|
||||
c.table_store = self.table_store
|
||||
c.type = type
|
||||
|
||||
return c
|
||||
|
||||
async def handle_get(self, v):
|
||||
|
||||
for k in v.keys:
|
||||
if k.type not in self or k.key not in self[k.type]:
|
||||
return ConfigResponse(
|
||||
version = None,
|
||||
values = None,
|
||||
directory = None,
|
||||
config = None,
|
||||
error = Error(
|
||||
type = "key-error",
|
||||
message = f"Key error"
|
||||
)
|
||||
)
|
||||
# for k in v.keys:
|
||||
# if k.type not in self or k.key not in self[k.type]:
|
||||
# return ConfigResponse(
|
||||
# version = None,
|
||||
# values = None,
|
||||
# directory = None,
|
||||
# config = None,
|
||||
# error = Error(
|
||||
# type = "key-error",
|
||||
# message = f"Key error"
|
||||
# )
|
||||
# )
|
||||
|
||||
values = [
|
||||
ConfigValue(
|
||||
type = k.type,
|
||||
key = k.key,
|
||||
value = self[k.type][k.key]
|
||||
value = await self.table_store.get_value(k.type, k.key)
|
||||
)
|
||||
for k in v.keys
|
||||
]
|
||||
|
||||
return ConfigResponse(
|
||||
version = self.version,
|
||||
version = await self.get_version(),
|
||||
values = values,
|
||||
directory = None,
|
||||
config = None,
|
||||
|
|
@ -60,23 +97,23 @@ class Configuration(dict):
|
|||
|
||||
async def handle_list(self, v):
|
||||
|
||||
if v.type not in self:
|
||||
# if v.type not in self:
|
||||
|
||||
return ConfigResponse(
|
||||
version = None,
|
||||
values = None,
|
||||
directory = None,
|
||||
config = None,
|
||||
error = Error(
|
||||
type = "key-error",
|
||||
message = "No such type",
|
||||
),
|
||||
)
|
||||
# return ConfigResponse(
|
||||
# version = None,
|
||||
# values = None,
|
||||
# directory = None,
|
||||
# config = None,
|
||||
# error = Error(
|
||||
# type = "key-error",
|
||||
# message = "No such type",
|
||||
# ),
|
||||
# )
|
||||
|
||||
return ConfigResponse(
|
||||
version = self.version,
|
||||
version = await self.get_version(),
|
||||
values = None,
|
||||
directory = list(self[v.type].keys()),
|
||||
directory = await self.table_store.get_keys(v.type),
|
||||
config = None,
|
||||
error = None,
|
||||
)
|
||||
|
|
@ -96,17 +133,17 @@ class Configuration(dict):
|
|||
)
|
||||
)
|
||||
|
||||
values = [
|
||||
ConfigValue(
|
||||
type = v.type,
|
||||
key = k,
|
||||
value = self[v.type][k],
|
||||
)
|
||||
for k in self[v.type]
|
||||
]
|
||||
v = await self.table_store.get_values(v.type)
|
||||
|
||||
values = map(
|
||||
lambda x: ConfigValue(
|
||||
type = v.type, key = x[0], value = x[1]
|
||||
),
|
||||
v
|
||||
)
|
||||
|
||||
return ConfigResponse(
|
||||
version = self.version,
|
||||
version = await self.get_version(),
|
||||
values = values,
|
||||
directory = None,
|
||||
config = None,
|
||||
|
|
@ -115,23 +152,24 @@ class Configuration(dict):
|
|||
|
||||
async def handle_delete(self, v):
|
||||
|
||||
for k in v.keys:
|
||||
if k.type not in self or k.key not in self[k.type]:
|
||||
return ConfigResponse(
|
||||
version = None,
|
||||
values = None,
|
||||
directory = None,
|
||||
config = None,
|
||||
error = Error(
|
||||
type = "key-error",
|
||||
message = f"Key error"
|
||||
)
|
||||
)
|
||||
# for k in v.keys:
|
||||
# if k.type not in self or k.key not in self[k.type]:
|
||||
# return ConfigResponse(
|
||||
# version = None,
|
||||
# values = None,
|
||||
# directory = None,
|
||||
# config = None,
|
||||
# error = Error(
|
||||
# type = "key-error",
|
||||
# message = f"Key error"
|
||||
# )
|
||||
# )
|
||||
|
||||
for k in v.keys:
|
||||
del self[k.type][k.key]
|
||||
|
||||
self.version += 1
|
||||
await self.table_store.delete_key(k.type, k.key)
|
||||
|
||||
await self.inc_version()
|
||||
|
||||
await self.push()
|
||||
|
||||
|
|
@ -147,9 +185,10 @@ class Configuration(dict):
|
|||
async def handle_put(self, v):
|
||||
|
||||
for k in v.values:
|
||||
self[k.type][k.key] = k.value
|
||||
|
||||
self.version += 1
|
||||
await self.table_store.put_config(k.type, k.key, k.value)
|
||||
|
||||
await self.inc_version()
|
||||
|
||||
await self.push()
|
||||
|
||||
|
|
@ -161,14 +200,29 @@ class Configuration(dict):
|
|||
error = None,
|
||||
)
|
||||
|
||||
async def get_config(self):
|
||||
|
||||
table = await self.table_store.get_all()
|
||||
|
||||
config = {}
|
||||
|
||||
for row in table:
|
||||
if row[0] not in config:
|
||||
config[row[0]] = {}
|
||||
config[row[0]][row[1]] = row[2]
|
||||
|
||||
return config
|
||||
|
||||
async def handle_config(self, v):
|
||||
|
||||
config = await self.get_config()
|
||||
|
||||
return ConfigResponse(
|
||||
version = self.version,
|
||||
version = await self.get_version(),
|
||||
value = None,
|
||||
directory = None,
|
||||
values = None,
|
||||
config = self,
|
||||
config = config,
|
||||
error = None,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ class FlowConfig:
|
|||
|
||||
async def handle_list_classes(self, msg):
|
||||
|
||||
names = list(self.config["flow-classes"].keys())
|
||||
names = list(await self.config.get("flow-classes").keys())
|
||||
|
||||
return FlowResponse(
|
||||
error = None,
|
||||
|
|
@ -20,14 +20,18 @@ class FlowConfig:
|
|||
|
||||
return FlowResponse(
|
||||
error = None,
|
||||
class_definition = self.config["flow-classes"][msg.class_name],
|
||||
class_definition = await self.config.get(
|
||||
"flow-classes"
|
||||
).get(msg.class_name),
|
||||
)
|
||||
|
||||
async def handle_put_class(self, msg):
|
||||
|
||||
self.config["flow-classes"][msg.class_name] = msg.class_definition
|
||||
await self.config.get("flow-classes").put(
|
||||
msg.class_name, msg.class_definition
|
||||
)
|
||||
|
||||
self.config.version += 1
|
||||
await self.config.inc_version()
|
||||
|
||||
await self.config.push()
|
||||
|
||||
|
|
@ -39,9 +43,9 @@ class FlowConfig:
|
|||
|
||||
print(msg)
|
||||
|
||||
del self.config["flow-classes"][msg.class_name]
|
||||
await self.config.get("flow-classes").delete(msg.class_name)
|
||||
|
||||
self.config.version += 1
|
||||
await self.config.inc_version()
|
||||
|
||||
await self.config.push()
|
||||
|
||||
|
|
@ -51,7 +55,7 @@ class FlowConfig:
|
|||
|
||||
async def handle_list_flows(self, msg):
|
||||
|
||||
names = list(self.config["flows"].keys())
|
||||
names = list(await self.config.get("flows").keys())
|
||||
|
||||
return FlowResponse(
|
||||
error = None,
|
||||
|
|
@ -60,7 +64,7 @@ class FlowConfig:
|
|||
|
||||
async def handle_get_flow(self, msg):
|
||||
|
||||
flow = self.config["flows"][msg.flow_id]
|
||||
flow = await self.config.get("flows").get(msg.flow_id)
|
||||
|
||||
return FlowResponse(
|
||||
error = None,
|
||||
|
|
@ -75,13 +79,13 @@ class FlowConfig:
|
|||
if msg.flow_id is None:
|
||||
raise RuntimeError("No flow ID")
|
||||
|
||||
if msg.flow_id in self.config["flows"]:
|
||||
if msg.flow_id in await self.config.get("flows").values():
|
||||
raise RuntimeError("Flow already exists")
|
||||
|
||||
if msg.description is None:
|
||||
raise RuntimeError("No description")
|
||||
|
||||
if msg.class_name not in self.config["flow-classes"]:
|
||||
if msg.class_name not in await self.config.get("flow-classes").values():
|
||||
raise RuntimeError("Class does not exist")
|
||||
|
||||
def repl_template(tmp):
|
||||
|
|
@ -91,7 +95,9 @@ class FlowConfig:
|
|||
"{id}", msg.flow_id
|
||||
)
|
||||
|
||||
cls = json.loads(self.config["flow-classes"][msg.class_name])
|
||||
cls = json.loads(
|
||||
await self.config.get("flow-classes").get(msg.class_name)
|
||||
)
|
||||
|
||||
for kind in ("class", "flow"):
|
||||
|
||||
|
|
@ -106,15 +112,18 @@ class FlowConfig:
|
|||
for k2, v2 in v.items()
|
||||
}
|
||||
|
||||
if processor in self.config["flows-active"]:
|
||||
target = json.loads(self.config["flows-active"][processor])
|
||||
flac = await self.config.get("flows-active").values()
|
||||
if processor in flac:
|
||||
target = json.loads(flac[processor])
|
||||
else:
|
||||
target = {}
|
||||
|
||||
if variant not in target:
|
||||
target[variant] = v
|
||||
|
||||
self.config["flows-active"][processor] = json.dumps(target)
|
||||
await self.config.get("flows-active").put(
|
||||
processor, json.dumps(target)
|
||||
)
|
||||
|
||||
def repl_interface(i):
|
||||
if isinstance(i, str):
|
||||
|
|
@ -133,13 +142,16 @@ class FlowConfig:
|
|||
else:
|
||||
interfaces = {}
|
||||
|
||||
self.config["flows"][msg.flow_id] = json.dumps({
|
||||
"description": msg.description,
|
||||
"class-name": msg.class_name,
|
||||
"interfaces": interfaces,
|
||||
})
|
||||
await self.config.get("flows").put(
|
||||
msg.flow_id,
|
||||
json.dumps({
|
||||
"description": msg.description,
|
||||
"class-name": msg.class_name,
|
||||
"interfaces": interfaces,
|
||||
})
|
||||
)
|
||||
|
||||
self.config.version += 1
|
||||
await self.config.inc_version()
|
||||
|
||||
await self.config.push()
|
||||
|
||||
|
|
@ -152,17 +164,17 @@ class FlowConfig:
|
|||
if msg.flow_id is None:
|
||||
raise RuntimeError("No flow ID")
|
||||
|
||||
if msg.flow_id not in self.config["flows"]:
|
||||
if msg.flow_id not in await self.config.get("flows").keys():
|
||||
raise RuntimeError("Flow ID invalid")
|
||||
|
||||
flow = json.loads(self.config["flows"][msg.flow_id])
|
||||
flow = json.loads(await self.config.get("flows").get(msg.flow_id))
|
||||
|
||||
if "class-name" not in flow:
|
||||
raise RuntimeError("Internal error: flow has no flow class")
|
||||
|
||||
class_name = flow["class-name"]
|
||||
|
||||
cls = json.loads(self.config["flow-classes"][class_name])
|
||||
cls = json.loads(await self.config.get("flow-classes").get(class_name))
|
||||
|
||||
def repl_template(tmp):
|
||||
return tmp.replace(
|
||||
|
|
@ -179,20 +191,24 @@ class FlowConfig:
|
|||
|
||||
variant = repl_template(variant)
|
||||
|
||||
if processor in self.config["flows-active"]:
|
||||
target = json.loads(self.config["flows-active"][processor])
|
||||
flac = await self.config.get("flows-active").values()
|
||||
|
||||
if processor in flac:
|
||||
target = json.loads(flac[processor])
|
||||
else:
|
||||
target = {}
|
||||
|
||||
if variant in target:
|
||||
del target[variant]
|
||||
|
||||
self.config["flows-active"][processor] = json.dumps(target)
|
||||
await self.config.get("flows-active").put(
|
||||
processor, json.dumps(target)
|
||||
)
|
||||
|
||||
if msg.flow_id in self.config["flows"]:
|
||||
del self.config["flows"][msg.flow_id]
|
||||
if msg.flow_id in await self.config.get("flows").values():
|
||||
await self.config.get("flows").delete(msg.flow_id)
|
||||
|
||||
self.config.version += 1
|
||||
await self.config.inc_version()
|
||||
|
||||
await self.config.push()
|
||||
|
||||
|
|
|
|||
|
|
@ -20,6 +20,9 @@ from . flow import FlowConfig
|
|||
from ... base import ProcessorMetrics, ConsumerMetrics, ProducerMetrics
|
||||
from ... base import Consumer, Producer
|
||||
|
||||
# FIXME: How to ensure this doesn't conflict with other usage?
|
||||
keyspace = "config"
|
||||
|
||||
default_ident = "config-svc"
|
||||
|
||||
default_config_request_queue = config_request_queue
|
||||
|
|
@ -29,6 +32,8 @@ default_config_push_queue = config_push_queue
|
|||
default_flow_request_queue = flow_request_queue
|
||||
default_flow_response_queue = flow_response_queue
|
||||
|
||||
default_cassandra_host = "cassandra"
|
||||
|
||||
class Processor(AsyncProcessor):
|
||||
|
||||
def __init__(self, **params):
|
||||
|
|
@ -50,6 +55,10 @@ class Processor(AsyncProcessor):
|
|||
"flow_response_queue", default_flow_response_queue
|
||||
)
|
||||
|
||||
cassandra_host = params.get("cassandra_host", default_cassandra_host)
|
||||
cassandra_user = params.get("cassandra_user")
|
||||
cassandra_password = params.get("cassandra_password")
|
||||
|
||||
id = params.get("id")
|
||||
|
||||
flow_request_schema = FlowRequest
|
||||
|
|
@ -62,6 +71,8 @@ class Processor(AsyncProcessor):
|
|||
"config_push_schema": ConfigPush.__name__,
|
||||
"flow_request_schema": FlowRequest.__name__,
|
||||
"flow_response_schema": FlowResponse.__name__,
|
||||
"cassandra_host": cassandra_host,
|
||||
"cassandra_user": cassandra_user,
|
||||
}
|
||||
)
|
||||
|
||||
|
|
@ -125,7 +136,14 @@ class Processor(AsyncProcessor):
|
|||
metrics = flow_response_metrics,
|
||||
)
|
||||
|
||||
self.config = Configuration(self.push)
|
||||
self.config = Configuration(
|
||||
host = cassandra_host.split(","),
|
||||
user = cassandra_user,
|
||||
password = cassandra_password,
|
||||
keyspace = keyspace,
|
||||
push = self.push
|
||||
)
|
||||
|
||||
self.flow = FlowConfig(self.config)
|
||||
|
||||
print("Service initialised.")
|
||||
|
|
@ -138,18 +156,23 @@ class Processor(AsyncProcessor):
|
|||
|
||||
async def push(self):
|
||||
|
||||
config = await self.config.get_config()
|
||||
version = await self.config.get_version()
|
||||
|
||||
resp = ConfigPush(
|
||||
version = self.config.version,
|
||||
version = version,
|
||||
value = None,
|
||||
directory = None,
|
||||
values = None,
|
||||
config = self.config,
|
||||
config = config,
|
||||
error = None,
|
||||
)
|
||||
|
||||
await self.config_push_producer.send(resp)
|
||||
|
||||
print("Pushed version ", self.config.version)
|
||||
# Race condition, should make sure version & config sync
|
||||
|
||||
print("Pushed version ", await self.config.get_version())
|
||||
|
||||
async def on_config_request(self, msg, consumer, flow):
|
||||
|
||||
|
|
@ -248,6 +271,24 @@ class Processor(AsyncProcessor):
|
|||
help=f'Flow response queue {default_flow_response_queue}',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--cassandra-host',
|
||||
default="cassandra",
|
||||
help=f'Graph host (default: cassandra)'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--cassandra-user',
|
||||
default=None,
|
||||
help=f'Cassandra user'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--cassandra-password',
|
||||
default=None,
|
||||
help=f'Cassandra password'
|
||||
)
|
||||
|
||||
def run():
|
||||
|
||||
Processor.launch(default_ident, __doc__)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue