apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-07-17 01:01:03 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Iron out collection problems

Browse source
This commit is contained in:
Cyber MacGeddon 2024-10-02 16:41:19 +01:00
parent feac569cd1
commit 66190689ee
4 changed files with 40 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

26
trustgraph-cli/scripts/tg-graph-show
View file

@ -9,12 +9,17 @@ import os
from trustgraph.clients.triples_query_client import TriplesQueryClient
default_pulsar_host = os.getenv("PULSAR_HOST", 'pulsar://localhost:6650')
default_user = 'trustgraph'
default_collection = 'default'
def show_graph(pulsar):
def show_graph(pulsar, user, collection):
tq = TriplesQueryClient(pulsar_host=pulsar)
rows = tq.request(None, None, None, limit=10_000_000)
rows = tq.request(
user=user, collection=collection,
s=None, p=None, o=None, limit=10_000_000
)
for row in rows:
print(row.s.value, row.p.value, row.o.value)
@ -32,11 +37,26 @@ def main():
help=f'Pulsar host (default: {default_pulsar_host})',
)
parser.add_argument(
'-u', '--user',
default=default_user,
help=f'User ID (default: {default_user})'
)
parser.add_argument(
'-c', '--collection',
default=default_collection,
help=f'Collection ID (default: {default_collection})'
)
args = parser.parse_args()
try:
show_graph(args.pulsar_host)
show_graph(
pulsar=args.pulsar_host, user=args.user,
collection=args.collection,
)
except Exception as e:

4
trustgraph-flow/trustgraph/storage/doc_embeddings/qdrant/write.py
View file

@ -37,7 +37,6 @@ class Processor(Consumer):
)
self.last_collection = None
self.last_dim = None
self.client = QdrantClient(url=store_uri)
@ -57,7 +56,7 @@ class Processor(Consumer):
str(dim)
)
if dim != self.last_dim:
if collection != self.last_collection:
if not self.client.collection_exists(collection):
@ -73,7 +72,6 @@ class Processor(Consumer):
raise e
self.last_collection = collection
self.last_dim = dim
self.client.upsert(
collection_name=collection,

4
trustgraph-flow/trustgraph/storage/graph_embeddings/qdrant/write.py
View file

@ -37,7 +37,6 @@ class Processor(Consumer):
)
self.last_collection = None
self.last_dim = None
self.client = QdrantClient(url=store_uri)
@ -55,7 +54,7 @@ class Processor(Consumer):
str(dim)
)
if dim != self.last_dim:
if collection != self.last_collection:
if not self.client.collection_exists(collection):
@ -71,7 +70,6 @@ class Processor(Consumer):
raise e
self.last_collection = collection
self.last_dim = dim
self.client.upsert(
collection_name=collection,

21
trustgraph-flow/trustgraph/storage/triples/cassandra/write.py
View file

@ -47,13 +47,22 @@ class Processor(Consumer):
table = (v.metadata.user, v.metadata.collection)
if self.table is None or self.table != self.table:
self.tg = TrustGraph(
hosts=self.graph_host,
keyspace=v.metadata.user, table=v.metadata.collection,
)
self.table = table
if self.table is None or self.table != table:
self.tg = None
try:
self.tg = TrustGraph(
hosts=self.graph_host,
keyspace=v.metadata.user, table=v.metadata.collection,
)
except Exception as e:
print("Exception", e, flush=True)
time.sleep(1)
raise e
self.table = table
self.tg.insert(
v.s.value,
v.p.value,