Changed schema for Value -> Term, majorly breaking change (#622)

* Changed schema for Value -> Term, majorly breaking change

* Following the schema change, Value -> Term into all processing

* Updated Cassandra for g, p, s, o index patterns (7 indexes)

* Reviewed and updated all tests

* Neo4j, Memgraph and FalkorDB remain broken, will look at once settled down
This commit is contained in:
cybermaggedon 2026-01-27 13:48:08 +00:00 committed by GitHub
parent e061f2c633
commit cf0daedefa
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
86 changed files with 2458 additions and 1764 deletions

View file

@ -6,7 +6,7 @@ import pytest
from unittest.mock import MagicMock, patch
from trustgraph.storage.graph_embeddings.milvus.write import Processor
from trustgraph.schema import Value, EntityEmbeddings
from trustgraph.schema import Term, EntityEmbeddings, IRI, LITERAL
class TestMilvusGraphEmbeddingsStorageProcessor:
@ -22,11 +22,11 @@ class TestMilvusGraphEmbeddingsStorageProcessor:
# Create test entities with embeddings
entity1 = EntityEmbeddings(
entity=Value(value='http://example.com/entity1', is_uri=True),
entity=Term(type=IRI, iri='http://example.com/entity1'),
vectors=[[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]]
)
entity2 = EntityEmbeddings(
entity=Value(value='literal entity', is_uri=False),
entity=Term(type=LITERAL, value='literal entity'),
vectors=[[0.7, 0.8, 0.9]]
)
message.entities = [entity1, entity2]
@ -84,7 +84,7 @@ class TestMilvusGraphEmbeddingsStorageProcessor:
message.metadata.collection = 'test_collection'
entity = EntityEmbeddings(
entity=Value(value='http://example.com/entity', is_uri=True),
entity=Term(type=IRI, iri='http://example.com/entity'),
vectors=[[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]]
)
message.entities = [entity]
@ -136,7 +136,7 @@ class TestMilvusGraphEmbeddingsStorageProcessor:
message.metadata.collection = 'test_collection'
entity = EntityEmbeddings(
entity=Value(value='', is_uri=False),
entity=Term(type=LITERAL, value=''),
vectors=[[0.1, 0.2, 0.3]]
)
message.entities = [entity]
@ -155,7 +155,7 @@ class TestMilvusGraphEmbeddingsStorageProcessor:
message.metadata.collection = 'test_collection'
entity = EntityEmbeddings(
entity=Value(value=None, is_uri=False),
entity=Term(type=LITERAL, value=None),
vectors=[[0.1, 0.2, 0.3]]
)
message.entities = [entity]
@ -174,15 +174,15 @@ class TestMilvusGraphEmbeddingsStorageProcessor:
message.metadata.collection = 'test_collection'
valid_entity = EntityEmbeddings(
entity=Value(value='http://example.com/valid', is_uri=True),
entity=Term(type=IRI, iri='http://example.com/valid'),
vectors=[[0.1, 0.2, 0.3]]
)
empty_entity = EntityEmbeddings(
entity=Value(value='', is_uri=False),
entity=Term(type=LITERAL, value=''),
vectors=[[0.4, 0.5, 0.6]]
)
none_entity = EntityEmbeddings(
entity=Value(value=None, is_uri=False),
entity=Term(type=LITERAL, value=None),
vectors=[[0.7, 0.8, 0.9]]
)
message.entities = [valid_entity, empty_entity, none_entity]
@ -217,7 +217,7 @@ class TestMilvusGraphEmbeddingsStorageProcessor:
message.metadata.collection = 'test_collection'
entity = EntityEmbeddings(
entity=Value(value='http://example.com/entity', is_uri=True),
entity=Term(type=IRI, iri='http://example.com/entity'),
vectors=[]
)
message.entities = [entity]
@ -236,7 +236,7 @@ class TestMilvusGraphEmbeddingsStorageProcessor:
message.metadata.collection = 'test_collection'
entity = EntityEmbeddings(
entity=Value(value='http://example.com/entity', is_uri=True),
entity=Term(type=IRI, iri='http://example.com/entity'),
vectors=[
[0.1, 0.2], # 2D vector
[0.3, 0.4, 0.5, 0.6], # 4D vector
@ -269,11 +269,11 @@ class TestMilvusGraphEmbeddingsStorageProcessor:
message.metadata.collection = 'test_collection'
uri_entity = EntityEmbeddings(
entity=Value(value='http://example.com/uri_entity', is_uri=True),
entity=Term(type=IRI, iri='http://example.com/uri_entity'),
vectors=[[0.1, 0.2, 0.3]]
)
literal_entity = EntityEmbeddings(
entity=Value(value='literal entity text', is_uri=False),
entity=Term(type=LITERAL, value='literal entity text'),
vectors=[[0.4, 0.5, 0.6]]
)
message.entities = [uri_entity, literal_entity]

View file

@ -9,6 +9,7 @@ from unittest import IsolatedAsyncioTestCase
# Import the service under test
from trustgraph.storage.graph_embeddings.qdrant.write import Processor
from trustgraph.schema import IRI, LITERAL
class TestQdrantGraphEmbeddingsStorage(IsolatedAsyncioTestCase):
@ -67,7 +68,8 @@ class TestQdrantGraphEmbeddingsStorage(IsolatedAsyncioTestCase):
mock_message.metadata.collection = 'test_collection'
mock_entity = MagicMock()
mock_entity.entity.value = 'test_entity'
mock_entity.entity.type = IRI
mock_entity.entity.iri = 'test_entity'
mock_entity.vectors = [[0.1, 0.2, 0.3]] # Single vector with 3 dimensions
mock_message.entities = [mock_entity]
@ -120,11 +122,13 @@ class TestQdrantGraphEmbeddingsStorage(IsolatedAsyncioTestCase):
mock_message.metadata.collection = 'multi_collection'
mock_entity1 = MagicMock()
mock_entity1.entity.value = 'entity_one'
mock_entity1.entity.type = IRI
mock_entity1.entity.iri = 'entity_one'
mock_entity1.vectors = [[0.1, 0.2]]
mock_entity2 = MagicMock()
mock_entity2.entity.value = 'entity_two'
mock_entity2.entity.type = IRI
mock_entity2.entity.iri = 'entity_two'
mock_entity2.vectors = [[0.3, 0.4]]
mock_message.entities = [mock_entity1, mock_entity2]
@ -179,7 +183,8 @@ class TestQdrantGraphEmbeddingsStorage(IsolatedAsyncioTestCase):
mock_message.metadata.collection = 'vector_collection'
mock_entity = MagicMock()
mock_entity.entity.value = 'multi_vector_entity'
mock_entity.entity.type = IRI
mock_entity.entity.iri = 'multi_vector_entity'
mock_entity.vectors = [
[0.1, 0.2, 0.3],
[0.4, 0.5, 0.6],
@ -231,11 +236,12 @@ class TestQdrantGraphEmbeddingsStorage(IsolatedAsyncioTestCase):
mock_message.metadata.collection = 'empty_collection'
mock_entity_empty = MagicMock()
mock_entity_empty.entity.type = LITERAL
mock_entity_empty.entity.value = "" # Empty string
mock_entity_empty.vectors = [[0.1, 0.2]]
mock_entity_none = MagicMock()
mock_entity_none.entity.value = None # None value
mock_entity_none.entity = None # None entity
mock_entity_none.vectors = [[0.3, 0.4]]
mock_message.entities = [mock_entity_empty, mock_entity_none]

View file

@ -7,7 +7,7 @@ from unittest.mock import MagicMock, patch, call
from trustgraph.storage.triples.neo4j.write import Processor as StorageProcessor
from trustgraph.query.triples.neo4j.service import Processor as QueryProcessor
from trustgraph.schema import Triples, Triple, Value, Metadata
from trustgraph.schema import Triples, Triple, Term, Metadata, IRI, LITERAL
from trustgraph.schema import TriplesQueryRequest
@ -60,9 +60,9 @@ class TestNeo4jUserCollectionIsolation:
)
triple = Triple(
s=Value(value="http://example.com/subject", is_uri=True),
p=Value(value="http://example.com/predicate", is_uri=True),
o=Value(value="literal_value", is_uri=False)
s=Term(type=IRI, iri="http://example.com/subject"),
p=Term(type=IRI, iri="http://example.com/predicate"),
o=Term(type=LITERAL, value="literal_value")
)
message = Triples(
@ -128,9 +128,9 @@ class TestNeo4jUserCollectionIsolation:
metadata = Metadata(id="test-id")
triple = Triple(
s=Value(value="http://example.com/subject", is_uri=True),
p=Value(value="http://example.com/predicate", is_uri=True),
o=Value(value="http://example.com/object", is_uri=True)
s=Term(type=IRI, iri="http://example.com/subject"),
p=Term(type=IRI, iri="http://example.com/predicate"),
o=Term(type=IRI, iri="http://example.com/object")
)
message = Triples(
@ -170,8 +170,8 @@ class TestNeo4jUserCollectionIsolation:
query = TriplesQueryRequest(
user="test_user",
collection="test_collection",
s=Value(value="http://example.com/subject", is_uri=True),
p=Value(value="http://example.com/predicate", is_uri=True),
s=Term(type=IRI, iri="http://example.com/subject"),
p=Term(type=IRI, iri="http://example.com/predicate"),
o=None
)
@ -254,9 +254,9 @@ class TestNeo4jUserCollectionIsolation:
metadata=Metadata(user="user1", collection="coll1"),
triples=[
Triple(
s=Value(value="http://example.com/user1/subject", is_uri=True),
p=Value(value="http://example.com/predicate", is_uri=True),
o=Value(value="user1_data", is_uri=False)
s=Term(type=IRI, iri="http://example.com/user1/subject"),
p=Term(type=IRI, iri="http://example.com/predicate"),
o=Term(type=LITERAL, value="user1_data")
)
]
)
@ -265,9 +265,9 @@ class TestNeo4jUserCollectionIsolation:
metadata=Metadata(user="user2", collection="coll2"),
triples=[
Triple(
s=Value(value="http://example.com/user2/subject", is_uri=True),
p=Value(value="http://example.com/predicate", is_uri=True),
o=Value(value="user2_data", is_uri=False)
s=Term(type=IRI, iri="http://example.com/user2/subject"),
p=Term(type=IRI, iri="http://example.com/predicate"),
o=Term(type=LITERAL, value="user2_data")
)
]
)
@ -429,9 +429,9 @@ class TestNeo4jUserCollectionRegression:
metadata=Metadata(user="user1", collection="coll1"),
triples=[
Triple(
s=Value(value=shared_uri, is_uri=True),
p=Value(value="http://example.com/p", is_uri=True),
o=Value(value="user1_value", is_uri=False)
s=Term(type=IRI, iri=shared_uri),
p=Term(type=IRI, iri="http://example.com/p"),
o=Term(type=LITERAL, value="user1_value")
)
]
)
@ -440,9 +440,9 @@ class TestNeo4jUserCollectionRegression:
metadata=Metadata(user="user2", collection="coll2"),
triples=[
Triple(
s=Value(value=shared_uri, is_uri=True),
p=Value(value="http://example.com/p", is_uri=True),
o=Value(value="user2_value", is_uri=False)
s=Term(type=IRI, iri=shared_uri),
p=Term(type=IRI, iri="http://example.com/p"),
o=Term(type=LITERAL, value="user2_value")
)
]
)

View file

@ -6,7 +6,8 @@ import pytest
from unittest.mock import MagicMock, patch, AsyncMock
from trustgraph.storage.triples.cassandra.write import Processor
from trustgraph.schema import Value, Triple
from trustgraph.schema import Triple, LITERAL
from trustgraph.direct.cassandra_kg import DEFAULT_GRAPH
class TestCassandraStorageProcessor:
@ -175,29 +176,37 @@ class TestCassandraStorageProcessor:
processor = Processor(taskgroup=taskgroup_mock)
# Create mock triples
# Create mock triples with proper Term structure
triple1 = MagicMock()
triple1.s.type = LITERAL
triple1.s.value = 'subject1'
triple1.p.type = LITERAL
triple1.p.value = 'predicate1'
triple1.o.type = LITERAL
triple1.o.value = 'object1'
triple1.g = None
triple2 = MagicMock()
triple2.s.type = LITERAL
triple2.s.value = 'subject2'
triple2.p.type = LITERAL
triple2.p.value = 'predicate2'
triple2.o.type = LITERAL
triple2.o.value = 'object2'
triple2.g = None
# Create mock message
mock_message = MagicMock()
mock_message.metadata.user = 'user1'
mock_message.metadata.collection = 'collection1'
mock_message.triples = [triple1, triple2]
await processor.store_triples(mock_message)
# Verify both triples were inserted
# Verify both triples were inserted (with g= parameter)
assert mock_tg_instance.insert.call_count == 2
mock_tg_instance.insert.assert_any_call('collection1', 'subject1', 'predicate1', 'object1')
mock_tg_instance.insert.assert_any_call('collection1', 'subject2', 'predicate2', 'object2')
mock_tg_instance.insert.assert_any_call('collection1', 'subject1', 'predicate1', 'object1', g=DEFAULT_GRAPH)
mock_tg_instance.insert.assert_any_call('collection1', 'subject2', 'predicate2', 'object2', g=DEFAULT_GRAPH)
@pytest.mark.asyncio
@patch('trustgraph.storage.triples.cassandra.write.KnowledgeGraph')
@ -369,25 +378,30 @@ class TestCassandraStorageProcessor:
processor = Processor(taskgroup=taskgroup_mock)
# Create triple with special characters
# Create triple with special characters and proper Term structure
triple = MagicMock()
triple.s.type = LITERAL
triple.s.value = 'subject with spaces & symbols'
triple.p.type = LITERAL
triple.p.value = 'predicate:with/colons'
triple.o.type = LITERAL
triple.o.value = 'object with "quotes" and unicode: ñáéíóú'
triple.g = None
mock_message = MagicMock()
mock_message.metadata.user = 'test_user'
mock_message.metadata.collection = 'test_collection'
mock_message.triples = [triple]
await processor.store_triples(mock_message)
# Verify the triple was inserted with special characters preserved
mock_tg_instance.insert.assert_called_once_with(
'test_collection',
'subject with spaces & symbols',
'predicate:with/colons',
'object with "quotes" and unicode: ñáéíóú'
'object with "quotes" and unicode: ñáéíóú',
g=DEFAULT_GRAPH
)
@pytest.mark.asyncio
@ -475,11 +489,15 @@ class TestCassandraPerformanceOptimizations:
processor = Processor(taskgroup=taskgroup_mock)
# Create test triple
# Create test triple with proper Term structure
triple = MagicMock()
triple.s.type = LITERAL
triple.s.value = 'test_subject'
triple.p.type = LITERAL
triple.p.value = 'test_predicate'
triple.o.type = LITERAL
triple.o.value = 'test_object'
triple.g = None
mock_message = MagicMock()
mock_message.metadata.user = 'user1'
@ -490,7 +508,8 @@ class TestCassandraPerformanceOptimizations:
# Verify insert was called for the triple (implementation details tested in KnowledgeGraph)
mock_tg_instance.insert.assert_called_once_with(
'collection1', 'test_subject', 'test_predicate', 'test_object'
'collection1', 'test_subject', 'test_predicate', 'test_object',
g=DEFAULT_GRAPH
)
def test_environment_variable_controls_mode(self):

View file

@ -6,7 +6,7 @@ import pytest
from unittest.mock import MagicMock, patch
from trustgraph.storage.triples.falkordb.write import Processor
from trustgraph.schema import Value, Triple
from trustgraph.schema import Term, Triple, IRI, LITERAL
class TestFalkorDBStorageProcessor:
@ -22,9 +22,9 @@ class TestFalkorDBStorageProcessor:
# Create a test triple
triple = Triple(
s=Value(value='http://example.com/subject', is_uri=True),
p=Value(value='http://example.com/predicate', is_uri=True),
o=Value(value='literal object', is_uri=False)
s=Term(type=IRI, iri='http://example.com/subject'),
p=Term(type=IRI, iri='http://example.com/predicate'),
o=Term(type=LITERAL, value='literal object')
)
message.triples = [triple]
@ -183,9 +183,9 @@ class TestFalkorDBStorageProcessor:
message.metadata.collection = 'test_collection'
triple = Triple(
s=Value(value='http://example.com/subject', is_uri=True),
p=Value(value='http://example.com/predicate', is_uri=True),
o=Value(value='http://example.com/object', is_uri=True)
s=Term(type=IRI, iri='http://example.com/subject'),
p=Term(type=IRI, iri='http://example.com/predicate'),
o=Term(type=IRI, iri='http://example.com/object')
)
message.triples = [triple]
@ -269,14 +269,14 @@ class TestFalkorDBStorageProcessor:
message.metadata.collection = 'test_collection'
triple1 = Triple(
s=Value(value='http://example.com/subject1', is_uri=True),
p=Value(value='http://example.com/predicate1', is_uri=True),
o=Value(value='literal object1', is_uri=False)
s=Term(type=IRI, iri='http://example.com/subject1'),
p=Term(type=IRI, iri='http://example.com/predicate1'),
o=Term(type=LITERAL, value='literal object1')
)
triple2 = Triple(
s=Value(value='http://example.com/subject2', is_uri=True),
p=Value(value='http://example.com/predicate2', is_uri=True),
o=Value(value='http://example.com/object2', is_uri=True)
s=Term(type=IRI, iri='http://example.com/subject2'),
p=Term(type=IRI, iri='http://example.com/predicate2'),
o=Term(type=IRI, iri='http://example.com/object2')
)
message.triples = [triple1, triple2]
@ -337,14 +337,14 @@ class TestFalkorDBStorageProcessor:
message.metadata.collection = 'test_collection'
triple1 = Triple(
s=Value(value='http://example.com/subject1', is_uri=True),
p=Value(value='http://example.com/predicate1', is_uri=True),
o=Value(value='literal object', is_uri=False)
s=Term(type=IRI, iri='http://example.com/subject1'),
p=Term(type=IRI, iri='http://example.com/predicate1'),
o=Term(type=LITERAL, value='literal object')
)
triple2 = Triple(
s=Value(value='http://example.com/subject2', is_uri=True),
p=Value(value='http://example.com/predicate2', is_uri=True),
o=Value(value='http://example.com/object2', is_uri=True)
s=Term(type=IRI, iri='http://example.com/subject2'),
p=Term(type=IRI, iri='http://example.com/predicate2'),
o=Term(type=IRI, iri='http://example.com/object2')
)
message.triples = [triple1, triple2]

View file

@ -6,7 +6,7 @@ import pytest
from unittest.mock import MagicMock, patch
from trustgraph.storage.triples.memgraph.write import Processor
from trustgraph.schema import Value, Triple
from trustgraph.schema import Term, Triple, IRI, LITERAL
class TestMemgraphStorageProcessor:
@ -22,9 +22,9 @@ class TestMemgraphStorageProcessor:
# Create a test triple
triple = Triple(
s=Value(value='http://example.com/subject', is_uri=True),
p=Value(value='http://example.com/predicate', is_uri=True),
o=Value(value='literal object', is_uri=False)
s=Term(type=IRI, iri='http://example.com/subject'),
p=Term(type=IRI, iri='http://example.com/predicate'),
o=Term(type=LITERAL, value='literal object')
)
message.triples = [triple]
@ -231,9 +231,9 @@ class TestMemgraphStorageProcessor:
mock_tx = MagicMock()
triple = Triple(
s=Value(value='http://example.com/subject', is_uri=True),
p=Value(value='http://example.com/predicate', is_uri=True),
o=Value(value='http://example.com/object', is_uri=True)
s=Term(type=IRI, iri='http://example.com/subject'),
p=Term(type=IRI, iri='http://example.com/predicate'),
o=Term(type=IRI, iri='http://example.com/object')
)
processor.create_triple(mock_tx, triple, "test_user", "test_collection")
@ -265,9 +265,9 @@ class TestMemgraphStorageProcessor:
mock_tx = MagicMock()
triple = Triple(
s=Value(value='http://example.com/subject', is_uri=True),
p=Value(value='http://example.com/predicate', is_uri=True),
o=Value(value='literal object', is_uri=False)
s=Term(type=IRI, iri='http://example.com/subject'),
p=Term(type=IRI, iri='http://example.com/predicate'),
o=Term(type=LITERAL, value='literal object')
)
processor.create_triple(mock_tx, triple, "test_user", "test_collection")
@ -347,14 +347,14 @@ class TestMemgraphStorageProcessor:
message.metadata.collection = 'test_collection'
triple1 = Triple(
s=Value(value='http://example.com/subject1', is_uri=True),
p=Value(value='http://example.com/predicate1', is_uri=True),
o=Value(value='literal object1', is_uri=False)
s=Term(type=IRI, iri='http://example.com/subject1'),
p=Term(type=IRI, iri='http://example.com/predicate1'),
o=Term(type=LITERAL, value='literal object1')
)
triple2 = Triple(
s=Value(value='http://example.com/subject2', is_uri=True),
p=Value(value='http://example.com/predicate2', is_uri=True),
o=Value(value='http://example.com/object2', is_uri=True)
s=Term(type=IRI, iri='http://example.com/subject2'),
p=Term(type=IRI, iri='http://example.com/predicate2'),
o=Term(type=IRI, iri='http://example.com/object2')
)
message.triples = [triple1, triple2]

View file

@ -6,6 +6,7 @@ import pytest
from unittest.mock import MagicMock, patch, AsyncMock
from trustgraph.storage.triples.neo4j.write import Processor
from trustgraph.schema import IRI, LITERAL
class TestNeo4jStorageProcessor:
@ -257,10 +258,12 @@ class TestNeo4jStorageProcessor:
# Create mock triple with URI object
triple = MagicMock()
triple.s.value = "http://example.com/subject"
triple.p.value = "http://example.com/predicate"
triple.o.value = "http://example.com/object"
triple.o.is_uri = True
triple.s.type = IRI
triple.s.iri = "http://example.com/subject"
triple.p.type = IRI
triple.p.iri = "http://example.com/predicate"
triple.o.type = IRI
triple.o.iri = "http://example.com/object"
# Create mock message with metadata
mock_message = MagicMock()
@ -327,10 +330,12 @@ class TestNeo4jStorageProcessor:
# Create mock triple with literal object
triple = MagicMock()
triple.s.value = "http://example.com/subject"
triple.p.value = "http://example.com/predicate"
triple.s.type = IRI
triple.s.iri = "http://example.com/subject"
triple.p.type = IRI
triple.p.iri = "http://example.com/predicate"
triple.o.type = LITERAL
triple.o.value = "literal value"
triple.o.is_uri = False
# Create mock message with metadata
mock_message = MagicMock()
@ -398,16 +403,20 @@ class TestNeo4jStorageProcessor:
# Create mock triples
triple1 = MagicMock()
triple1.s.value = "http://example.com/subject1"
triple1.p.value = "http://example.com/predicate1"
triple1.o.value = "http://example.com/object1"
triple1.o.is_uri = True
triple1.s.type = IRI
triple1.s.iri = "http://example.com/subject1"
triple1.p.type = IRI
triple1.p.iri = "http://example.com/predicate1"
triple1.o.type = IRI
triple1.o.iri = "http://example.com/object1"
triple2 = MagicMock()
triple2.s.value = "http://example.com/subject2"
triple2.p.value = "http://example.com/predicate2"
triple2.s.type = IRI
triple2.s.iri = "http://example.com/subject2"
triple2.p.type = IRI
triple2.p.iri = "http://example.com/predicate2"
triple2.o.type = LITERAL
triple2.o.value = "literal value"
triple2.o.is_uri = False
# Create mock message with metadata
mock_message = MagicMock()
@ -550,10 +559,12 @@ class TestNeo4jStorageProcessor:
# Create triple with special characters
triple = MagicMock()
triple.s.value = "http://example.com/subject with spaces"
triple.p.value = "http://example.com/predicate:with/symbols"
triple.s.type = IRI
triple.s.iri = "http://example.com/subject with spaces"
triple.p.type = IRI
triple.p.iri = "http://example.com/predicate:with/symbols"
triple.o.type = LITERAL
triple.o.value = 'literal with "quotes" and unicode: ñáéíóú'
triple.o.is_uri = False
mock_message = MagicMock()
mock_message.triples = [triple]