Refactor names (#4)

- Downsize embeddings model to mini-lm in docker-compose files
- Rename for structure
- Default queues defined in schema file
- Standardize naming: graph embeddings, chunk embeddings, triples
This commit is contained in:
cybermaggedon 2024-07-23 21:34:03 +01:00 committed by GitHub
parent cbddf197ad
commit 3947920ee8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
71 changed files with 764 additions and 585 deletions

View file

View file

@ -0,0 +1,3 @@
from . write import *

View file

@ -0,0 +1,7 @@
#!/usr/bin/env python3
from . write import run
if __name__ == '__main__':
run()

View file

@ -0,0 +1,68 @@
"""
Graph writer. Input is graph edge. Writes edges to Cassandra graph.
"""
import pulsar
import base64
import os
import argparse
import time
from .... trustgraph import TrustGraph
from .... schema import Triple
from .... schema import triples_store_queue
from .... log_level import LogLevel
from .... base import Consumer
module = ".".join(__name__.split(".")[1:-1])
default_input_queue = triples_store_queue
default_subscriber = module
default_graph_host='localhost'
class Processor(Consumer):
def __init__(self, **params):
input_queue = params.get("input_queue", default_input_queue)
subscriber = params.get("subscriber", default_subscriber)
graph_host = params.get("graph_host", default_graph_host)
super(Processor, self).__init__(
**params | {
"input_queue": input_queue,
"subscriber": subscriber,
"input_schema": Triple,
}
)
self.tg = TrustGraph([graph_host])
def handle(self, msg):
v = msg.value()
self.tg.insert(
v.s.value,
v.p.value,
v.o.value
)
@staticmethod
def add_args(parser):
Consumer.add_args(
parser, default_input_queue, default_subscriber,
)
parser.add_argument(
'-g', '--graph-host',
default="localhost",
help=f'Graph host (default: localhost)'
)
def run():
Processor.start(module, __doc__)