mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-07-08 21:02:12 +02:00
The metadata field (list of triples) in the pipeline Metadata class
was redundant. Document metadata triples already flow directly from librarian to triple-store via emit_document_provenance() - they don't need to pass through the extraction pipeline. Additionally, chunker and PDF decoder were overwriting metadata to [] anyway, so any metadata passed through the pipeline was being discarded. Changes: - Remove metadata field from Metadata dataclass (schema/core/metadata.py) - Update all Metadata instantiations to remove metadata=[] parameter - Remove metadata handling from translators (document_loading, knowledge) - Remove metadata consumption from extractors (ontology, agent) - Update gateway serializers and import handlers - Update all unit, integration, and contract tests
This commit is contained in:
parent
1837d73f34
commit
33f031d664
37 changed files with 106 additions and 343 deletions
|
|
@ -314,11 +314,19 @@ Converts row index fields into vector embeddings.
|
||||||
| `document_id` | Librarian reference, provenance linking |
|
| `document_id` | Librarian reference, provenance linking |
|
||||||
| `chunk_id` | Provenance tracking through pipeline |
|
| `chunk_id` | Provenance tracking through pipeline |
|
||||||
|
|
||||||
|
<<<<<<< HEAD
|
||||||
### Potentially Redundant Fields
|
### Potentially Redundant Fields
|
||||||
|
|
||||||
| Field | Status |
|
| Field | Status |
|
||||||
|-------|--------|
|
|-------|--------|
|
||||||
| `metadata.metadata` | Set to `[]` by all extractors; document-level metadata now handled by librarian at submission time |
|
| `metadata.metadata` | Set to `[]` by all extractors; document-level metadata now handled by librarian at submission time |
|
||||||
|
=======
|
||||||
|
### Removed Fields
|
||||||
|
|
||||||
|
| Field | Status |
|
||||||
|
|-------|--------|
|
||||||
|
| `metadata.metadata` | Removed from `Metadata` class. Document-level metadata triples are now emitted directly by librarian to triple store at submission time, not carried through the extraction pipeline. |
|
||||||
|
>>>>>>> e3bcbf73 (The metadata field (list of triples) in the pipeline Metadata class)
|
||||||
|
|
||||||
### Bytes Fields Pattern
|
### Bytes Fields Pattern
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -95,8 +95,7 @@ def sample_message_data():
|
||||||
"Metadata": {
|
"Metadata": {
|
||||||
"id": "test-doc-123",
|
"id": "test-doc-123",
|
||||||
"user": "test_user",
|
"user": "test_user",
|
||||||
"collection": "test_collection",
|
"collection": "test_collection"
|
||||||
"metadata": []
|
|
||||||
},
|
},
|
||||||
"Term": {
|
"Term": {
|
||||||
"type": IRI,
|
"type": IRI,
|
||||||
|
|
|
||||||
|
|
@ -401,25 +401,6 @@ class TestMetadataMessageContracts:
|
||||||
assert metadata.id == "test-doc-123"
|
assert metadata.id == "test-doc-123"
|
||||||
assert metadata.user == "test_user"
|
assert metadata.user == "test_user"
|
||||||
assert metadata.collection == "test_collection"
|
assert metadata.collection == "test_collection"
|
||||||
assert isinstance(metadata.metadata, list)
|
|
||||||
|
|
||||||
def test_metadata_with_triples_contract(self, sample_message_data):
|
|
||||||
"""Test Metadata with embedded triples contract"""
|
|
||||||
# Arrange
|
|
||||||
triple = Triple(**sample_message_data["Triple"])
|
|
||||||
metadata_data = {
|
|
||||||
"id": "doc-with-triples",
|
|
||||||
"user": "test_user",
|
|
||||||
"collection": "test_collection",
|
|
||||||
"metadata": [triple]
|
|
||||||
}
|
|
||||||
|
|
||||||
# Act & Assert
|
|
||||||
assert validate_schema_contract(Metadata, metadata_data)
|
|
||||||
|
|
||||||
metadata = Metadata(**metadata_data)
|
|
||||||
assert len(metadata.metadata) == 1
|
|
||||||
assert metadata.metadata[0].s.iri == "http://example.com/subject"
|
|
||||||
|
|
||||||
def test_error_schema_contract(self):
|
def test_error_schema_contract(self):
|
||||||
"""Test Error schema contract"""
|
"""Test Error schema contract"""
|
||||||
|
|
|
||||||
|
|
@ -24,7 +24,6 @@ class TestRowsCassandraContracts:
|
||||||
id="test-doc-001",
|
id="test-doc-001",
|
||||||
user="test_user",
|
user="test_user",
|
||||||
collection="test_collection",
|
collection="test_collection",
|
||||||
metadata=[]
|
|
||||||
)
|
)
|
||||||
|
|
||||||
test_object = ExtractedObject(
|
test_object = ExtractedObject(
|
||||||
|
|
@ -50,7 +49,6 @@ class TestRowsCassandraContracts:
|
||||||
assert hasattr(test_object.metadata, 'id')
|
assert hasattr(test_object.metadata, 'id')
|
||||||
assert hasattr(test_object.metadata, 'user')
|
assert hasattr(test_object.metadata, 'user')
|
||||||
assert hasattr(test_object.metadata, 'collection')
|
assert hasattr(test_object.metadata, 'collection')
|
||||||
assert hasattr(test_object.metadata, 'metadata')
|
|
||||||
|
|
||||||
# Verify types
|
# Verify types
|
||||||
assert isinstance(test_object.schema_name, str)
|
assert isinstance(test_object.schema_name, str)
|
||||||
|
|
@ -154,7 +152,6 @@ class TestRowsCassandraContracts:
|
||||||
id="serial-001",
|
id="serial-001",
|
||||||
user="test_user",
|
user="test_user",
|
||||||
collection="test_coll",
|
collection="test_coll",
|
||||||
metadata=[]
|
|
||||||
),
|
),
|
||||||
schema_name="test_schema",
|
schema_name="test_schema",
|
||||||
values=[{"field1": "value1", "field2": "123"}],
|
values=[{"field1": "value1", "field2": "123"}],
|
||||||
|
|
@ -234,7 +231,6 @@ class TestRowsCassandraContracts:
|
||||||
id="meta-001",
|
id="meta-001",
|
||||||
user="user123", # -> keyspace
|
user="user123", # -> keyspace
|
||||||
collection="coll456", # -> partition key
|
collection="coll456", # -> partition key
|
||||||
metadata=[{"key": "value"}]
|
|
||||||
),
|
),
|
||||||
schema_name="table789", # -> table name
|
schema_name="table789", # -> table name
|
||||||
values=[{"field": "value"}],
|
values=[{"field": "value"}],
|
||||||
|
|
@ -262,7 +258,6 @@ class TestRowsCassandraContractsBatch:
|
||||||
id="batch-doc-001",
|
id="batch-doc-001",
|
||||||
user="test_user",
|
user="test_user",
|
||||||
collection="test_collection",
|
collection="test_collection",
|
||||||
metadata=[]
|
|
||||||
)
|
)
|
||||||
|
|
||||||
batch_object = ExtractedObject(
|
batch_object = ExtractedObject(
|
||||||
|
|
@ -309,7 +304,6 @@ class TestRowsCassandraContractsBatch:
|
||||||
id="empty-batch-001",
|
id="empty-batch-001",
|
||||||
user="test_user",
|
user="test_user",
|
||||||
collection="test_collection",
|
collection="test_collection",
|
||||||
metadata=[]
|
|
||||||
)
|
)
|
||||||
|
|
||||||
empty_batch_object = ExtractedObject(
|
empty_batch_object = ExtractedObject(
|
||||||
|
|
@ -332,7 +326,6 @@ class TestRowsCassandraContractsBatch:
|
||||||
id="single-batch-001",
|
id="single-batch-001",
|
||||||
user="test_user",
|
user="test_user",
|
||||||
collection="test_collection",
|
collection="test_collection",
|
||||||
metadata=[]
|
|
||||||
)
|
)
|
||||||
|
|
||||||
single_batch_object = ExtractedObject(
|
single_batch_object = ExtractedObject(
|
||||||
|
|
@ -362,7 +355,6 @@ class TestRowsCassandraContractsBatch:
|
||||||
id="batch-serial-001",
|
id="batch-serial-001",
|
||||||
user="test_user",
|
user="test_user",
|
||||||
collection="test_coll",
|
collection="test_coll",
|
||||||
metadata=[]
|
|
||||||
),
|
),
|
||||||
schema_name="test_schema",
|
schema_name="test_schema",
|
||||||
values=[
|
values=[
|
||||||
|
|
@ -436,7 +428,6 @@ class TestRowsCassandraContractsBatch:
|
||||||
id="partition-test-001",
|
id="partition-test-001",
|
||||||
user="consistent_user", # Same keyspace
|
user="consistent_user", # Same keyspace
|
||||||
collection="consistent_collection", # Same partition
|
collection="consistent_collection", # Same partition
|
||||||
metadata=[]
|
|
||||||
)
|
)
|
||||||
|
|
||||||
batch_object = ExtractedObject(
|
batch_object = ExtractedObject(
|
||||||
|
|
|
||||||
|
|
@ -95,7 +95,6 @@ class TestStructuredDataSchemaContracts:
|
||||||
id="structured-data-001",
|
id="structured-data-001",
|
||||||
user="test_user",
|
user="test_user",
|
||||||
collection="test_collection",
|
collection="test_collection",
|
||||||
metadata=[]
|
|
||||||
)
|
)
|
||||||
|
|
||||||
# Act
|
# Act
|
||||||
|
|
@ -121,7 +120,6 @@ class TestStructuredDataSchemaContracts:
|
||||||
id="extracted-obj-001",
|
id="extracted-obj-001",
|
||||||
user="test_user",
|
user="test_user",
|
||||||
collection="test_collection",
|
collection="test_collection",
|
||||||
metadata=[]
|
|
||||||
)
|
)
|
||||||
|
|
||||||
# Act
|
# Act
|
||||||
|
|
@ -147,7 +145,6 @@ class TestStructuredDataSchemaContracts:
|
||||||
id="extracted-batch-001",
|
id="extracted-batch-001",
|
||||||
user="test_user",
|
user="test_user",
|
||||||
collection="test_collection",
|
collection="test_collection",
|
||||||
metadata=[]
|
|
||||||
)
|
)
|
||||||
|
|
||||||
# Act - create object with multiple values
|
# Act - create object with multiple values
|
||||||
|
|
@ -182,7 +179,6 @@ class TestStructuredDataSchemaContracts:
|
||||||
id="extracted-empty-001",
|
id="extracted-empty-001",
|
||||||
user="test_user",
|
user="test_user",
|
||||||
collection="test_collection",
|
collection="test_collection",
|
||||||
metadata=[]
|
|
||||||
)
|
)
|
||||||
|
|
||||||
# Act - create object with empty values array
|
# Act - create object with empty values array
|
||||||
|
|
@ -283,7 +279,6 @@ class TestStructuredEmbeddingsContracts:
|
||||||
id="struct-embed-001",
|
id="struct-embed-001",
|
||||||
user="test_user",
|
user="test_user",
|
||||||
collection="test_collection",
|
collection="test_collection",
|
||||||
metadata=[]
|
|
||||||
)
|
)
|
||||||
|
|
||||||
# Act
|
# Act
|
||||||
|
|
@ -313,7 +308,7 @@ class TestStructuredDataSerializationContracts:
|
||||||
def test_structured_data_submission_serialization(self):
|
def test_structured_data_submission_serialization(self):
|
||||||
"""Test StructuredDataSubmission serialization contract"""
|
"""Test StructuredDataSubmission serialization contract"""
|
||||||
# Arrange
|
# Arrange
|
||||||
metadata = Metadata(id="test", user="user", collection="col", metadata=[])
|
metadata = Metadata(id="test", user="user", collection="col")
|
||||||
submission_data = {
|
submission_data = {
|
||||||
"metadata": metadata,
|
"metadata": metadata,
|
||||||
"format": "json",
|
"format": "json",
|
||||||
|
|
@ -328,7 +323,7 @@ class TestStructuredDataSerializationContracts:
|
||||||
def test_extracted_object_serialization(self):
|
def test_extracted_object_serialization(self):
|
||||||
"""Test ExtractedObject serialization contract"""
|
"""Test ExtractedObject serialization contract"""
|
||||||
# Arrange
|
# Arrange
|
||||||
metadata = Metadata(id="test", user="user", collection="col", metadata=[])
|
metadata = Metadata(id="test", user="user", collection="col")
|
||||||
object_data = {
|
object_data = {
|
||||||
"metadata": metadata,
|
"metadata": metadata,
|
||||||
"schema_name": "test_schema",
|
"schema_name": "test_schema",
|
||||||
|
|
@ -378,7 +373,7 @@ class TestStructuredDataSerializationContracts:
|
||||||
def test_extracted_object_batch_serialization(self):
|
def test_extracted_object_batch_serialization(self):
|
||||||
"""Test ExtractedObject batch serialization contract"""
|
"""Test ExtractedObject batch serialization contract"""
|
||||||
# Arrange
|
# Arrange
|
||||||
metadata = Metadata(id="test", user="user", collection="col", metadata=[])
|
metadata = Metadata(id="test", user="user", collection="col")
|
||||||
batch_object_data = {
|
batch_object_data = {
|
||||||
"metadata": metadata,
|
"metadata": metadata,
|
||||||
"schema_name": "test_schema",
|
"schema_name": "test_schema",
|
||||||
|
|
@ -397,7 +392,7 @@ class TestStructuredDataSerializationContracts:
|
||||||
def test_extracted_object_empty_batch_serialization(self):
|
def test_extracted_object_empty_batch_serialization(self):
|
||||||
"""Test ExtractedObject empty batch serialization contract"""
|
"""Test ExtractedObject empty batch serialization contract"""
|
||||||
# Arrange
|
# Arrange
|
||||||
metadata = Metadata(id="test", user="user", collection="col", metadata=[])
|
metadata = Metadata(id="test", user="user", collection="col")
|
||||||
empty_batch_data = {
|
empty_batch_data = {
|
||||||
"metadata": metadata,
|
"metadata": metadata,
|
||||||
"schema_name": "test_schema",
|
"schema_name": "test_schema",
|
||||||
|
|
|
||||||
|
|
@ -76,13 +76,6 @@ class TestAgentKgExtractionIntegration:
|
||||||
chunk=text.encode('utf-8'),
|
chunk=text.encode('utf-8'),
|
||||||
metadata=Metadata(
|
metadata=Metadata(
|
||||||
id="doc123",
|
id="doc123",
|
||||||
metadata=[
|
|
||||||
Triple(
|
|
||||||
s=Term(type=IRI, iri="doc123"),
|
|
||||||
p=Term(type=IRI, iri="http://example.org/type"),
|
|
||||||
o=Term(type=LITERAL, value="document")
|
|
||||||
)
|
|
||||||
]
|
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -137,10 +130,6 @@ class TestAgentKgExtractionIntegration:
|
||||||
extraction_data = extractor.parse_jsonl(agent_response)
|
extraction_data = extractor.parse_jsonl(agent_response)
|
||||||
triples, entity_contexts = extractor.process_extraction_data(extraction_data, v.metadata)
|
triples, entity_contexts = extractor.process_extraction_data(extraction_data, v.metadata)
|
||||||
|
|
||||||
# Add metadata triples
|
|
||||||
for t in v.metadata.metadata:
|
|
||||||
triples.append(t)
|
|
||||||
|
|
||||||
# Emit outputs
|
# Emit outputs
|
||||||
if triples:
|
if triples:
|
||||||
await extractor.emit_triples(flow("triples"), v.metadata, triples)
|
await extractor.emit_triples(flow("triples"), v.metadata, triples)
|
||||||
|
|
@ -242,9 +231,9 @@ class TestAgentKgExtractionIntegration:
|
||||||
# Act - JSONL parsing is lenient, invalid lines are skipped
|
# Act - JSONL parsing is lenient, invalid lines are skipped
|
||||||
await configured_agent_extractor.on_message(mock_message, mock_consumer, mock_flow_context)
|
await configured_agent_extractor.on_message(mock_message, mock_consumer, mock_flow_context)
|
||||||
|
|
||||||
# Assert - should emit triples (with just metadata) but no entity contexts
|
# Assert - with no valid extraction data, nothing is emitted
|
||||||
triples_publisher = mock_flow_context("triples")
|
triples_publisher = mock_flow_context("triples")
|
||||||
triples_publisher.send.assert_called_once()
|
triples_publisher.send.assert_not_called()
|
||||||
|
|
||||||
entity_contexts_publisher = mock_flow_context("entity-contexts")
|
entity_contexts_publisher = mock_flow_context("entity-contexts")
|
||||||
entity_contexts_publisher.send.assert_not_called()
|
entity_contexts_publisher.send.assert_not_called()
|
||||||
|
|
@ -268,17 +257,12 @@ class TestAgentKgExtractionIntegration:
|
||||||
# Act
|
# Act
|
||||||
await configured_agent_extractor.on_message(mock_message, mock_consumer, mock_flow_context)
|
await configured_agent_extractor.on_message(mock_message, mock_consumer, mock_flow_context)
|
||||||
|
|
||||||
# Assert
|
# Assert - with empty extraction results, nothing is emitted
|
||||||
# Should still emit outputs (even if empty) to maintain flow consistency
|
|
||||||
triples_publisher = mock_flow_context("triples")
|
triples_publisher = mock_flow_context("triples")
|
||||||
entity_contexts_publisher = mock_flow_context("entity-contexts")
|
entity_contexts_publisher = mock_flow_context("entity-contexts")
|
||||||
|
|
||||||
# Triples should include metadata triples at minimum
|
# No triples or entity contexts emitted for empty results
|
||||||
triples_publisher.send.assert_called_once()
|
triples_publisher.send.assert_not_called()
|
||||||
sent_triples = triples_publisher.send.call_args[0][0]
|
|
||||||
assert isinstance(sent_triples, Triples)
|
|
||||||
|
|
||||||
# Entity contexts should not be sent if empty
|
|
||||||
entity_contexts_publisher.send.assert_not_called()
|
entity_contexts_publisher.send.assert_not_called()
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
|
|
@ -308,7 +292,7 @@ class TestAgentKgExtractionIntegration:
|
||||||
test_text = "Test text for prompt rendering"
|
test_text = "Test text for prompt rendering"
|
||||||
chunk = Chunk(
|
chunk = Chunk(
|
||||||
chunk=test_text.encode('utf-8'),
|
chunk=test_text.encode('utf-8'),
|
||||||
metadata=Metadata(id="test-doc", metadata=[])
|
metadata=Metadata(id="test-doc")
|
||||||
)
|
)
|
||||||
|
|
||||||
agent_client = mock_flow_context("agent-request")
|
agent_client = mock_flow_context("agent-request")
|
||||||
|
|
@ -340,7 +324,7 @@ class TestAgentKgExtractionIntegration:
|
||||||
text = f"Test document {i} content"
|
text = f"Test document {i} content"
|
||||||
chunks.append(Chunk(
|
chunks.append(Chunk(
|
||||||
chunk=text.encode('utf-8'),
|
chunk=text.encode('utf-8'),
|
||||||
metadata=Metadata(id=f"doc{i}", metadata=[])
|
metadata=Metadata(id=f"doc{i}")
|
||||||
))
|
))
|
||||||
|
|
||||||
agent_client = mock_flow_context("agent-request")
|
agent_client = mock_flow_context("agent-request")
|
||||||
|
|
@ -375,7 +359,7 @@ class TestAgentKgExtractionIntegration:
|
||||||
unicode_text = "Machine Learning (学习机器) は人工知能の一分野です。"
|
unicode_text = "Machine Learning (学习机器) は人工知能の一分野です。"
|
||||||
chunk = Chunk(
|
chunk = Chunk(
|
||||||
chunk=unicode_text.encode('utf-8'),
|
chunk=unicode_text.encode('utf-8'),
|
||||||
metadata=Metadata(id="unicode-doc", metadata=[])
|
metadata=Metadata(id="unicode-doc")
|
||||||
)
|
)
|
||||||
|
|
||||||
agent_client = mock_flow_context("agent-request")
|
agent_client = mock_flow_context("agent-request")
|
||||||
|
|
@ -411,7 +395,7 @@ class TestAgentKgExtractionIntegration:
|
||||||
large_text = "Machine Learning is important. " * 1000 # Repeat to create large text
|
large_text = "Machine Learning is important. " * 1000 # Repeat to create large text
|
||||||
chunk = Chunk(
|
chunk = Chunk(
|
||||||
chunk=large_text.encode('utf-8'),
|
chunk=large_text.encode('utf-8'),
|
||||||
metadata=Metadata(id="large-doc", metadata=[])
|
metadata=Metadata(id="large-doc")
|
||||||
)
|
)
|
||||||
|
|
||||||
agent_client = mock_flow_context("agent-request")
|
agent_client = mock_flow_context("agent-request")
|
||||||
|
|
|
||||||
|
|
@ -171,7 +171,6 @@ async def test_export_no_message_loss_integration(mock_backend):
|
||||||
triples_obj = Triples(
|
triples_obj = Triples(
|
||||||
metadata=Metadata(
|
metadata=Metadata(
|
||||||
id=f"export-msg-{i}",
|
id=f"export-msg-{i}",
|
||||||
metadata=to_subgraph(msg_data["metadata"]["metadata"]),
|
|
||||||
user=msg_data["metadata"]["user"],
|
user=msg_data["metadata"]["user"],
|
||||||
collection=msg_data["metadata"]["collection"],
|
collection=msg_data["metadata"]["collection"],
|
||||||
),
|
),
|
||||||
|
|
|
||||||
|
|
@ -92,7 +92,6 @@ class TestKnowledgeGraphPipelineIntegration:
|
||||||
id="doc-123",
|
id="doc-123",
|
||||||
user="test_user",
|
user="test_user",
|
||||||
collection="test_collection",
|
collection="test_collection",
|
||||||
metadata=[]
|
|
||||||
),
|
),
|
||||||
chunk=b"Machine Learning is a subset of Artificial Intelligence. Neural Networks are used in Machine Learning to process complex patterns."
|
chunk=b"Machine Learning is a subset of Artificial Intelligence. Neural Networks are used in Machine Learning to process complex patterns."
|
||||||
)
|
)
|
||||||
|
|
@ -243,7 +242,6 @@ class TestKnowledgeGraphPipelineIntegration:
|
||||||
id="test-doc",
|
id="test-doc",
|
||||||
user="test_user",
|
user="test_user",
|
||||||
collection="test_collection",
|
collection="test_collection",
|
||||||
metadata=[]
|
|
||||||
)
|
)
|
||||||
|
|
||||||
# Act
|
# Act
|
||||||
|
|
@ -302,7 +300,6 @@ class TestKnowledgeGraphPipelineIntegration:
|
||||||
id="test-doc",
|
id="test-doc",
|
||||||
user="test_user",
|
user="test_user",
|
||||||
collection="test_collection",
|
collection="test_collection",
|
||||||
metadata=[]
|
|
||||||
)
|
)
|
||||||
|
|
||||||
# Act
|
# Act
|
||||||
|
|
@ -373,7 +370,6 @@ class TestKnowledgeGraphPipelineIntegration:
|
||||||
id="test-doc",
|
id="test-doc",
|
||||||
user="test_user",
|
user="test_user",
|
||||||
collection="test_collection",
|
collection="test_collection",
|
||||||
metadata=[]
|
|
||||||
),
|
),
|
||||||
triples=[
|
triples=[
|
||||||
Triple(
|
Triple(
|
||||||
|
|
@ -406,7 +402,6 @@ class TestKnowledgeGraphPipelineIntegration:
|
||||||
id="test-doc",
|
id="test-doc",
|
||||||
user="test_user",
|
user="test_user",
|
||||||
collection="test_collection",
|
collection="test_collection",
|
||||||
metadata=[]
|
|
||||||
),
|
),
|
||||||
entities=[
|
entities=[
|
||||||
EntityEmbeddings(
|
EntityEmbeddings(
|
||||||
|
|
@ -542,7 +537,7 @@ class TestKnowledgeGraphPipelineIntegration:
|
||||||
]
|
]
|
||||||
|
|
||||||
sample_chunk = Chunk(
|
sample_chunk = Chunk(
|
||||||
metadata=Metadata(id="test", user="user", collection="collection", metadata=[]),
|
metadata=Metadata(id="test", user="user", collection="collection"),
|
||||||
chunk=b"Test chunk"
|
chunk=b"Test chunk"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -569,7 +564,7 @@ class TestKnowledgeGraphPipelineIntegration:
|
||||||
# Arrange
|
# Arrange
|
||||||
large_chunk_batch = [
|
large_chunk_batch = [
|
||||||
Chunk(
|
Chunk(
|
||||||
metadata=Metadata(id=f"doc-{i}", user="user", collection="collection", metadata=[]),
|
metadata=Metadata(id=f"doc-{i}", user="user", collection="collection"),
|
||||||
chunk=f"Document {i} contains machine learning and AI content.".encode("utf-8")
|
chunk=f"Document {i} contains machine learning and AI content.".encode("utf-8")
|
||||||
)
|
)
|
||||||
for i in range(100) # Large batch
|
for i in range(100) # Large batch
|
||||||
|
|
@ -608,13 +603,6 @@ class TestKnowledgeGraphPipelineIntegration:
|
||||||
id="test-doc-123",
|
id="test-doc-123",
|
||||||
user="test_user",
|
user="test_user",
|
||||||
collection="test_collection",
|
collection="test_collection",
|
||||||
metadata=[
|
|
||||||
Triple(
|
|
||||||
s=Term(type=IRI, iri="doc:test"),
|
|
||||||
p=Term(type=IRI, iri="dc:title"),
|
|
||||||
o=Term(type=LITERAL, value="Test Document")
|
|
||||||
)
|
|
||||||
]
|
|
||||||
)
|
)
|
||||||
|
|
||||||
sample_chunk = Chunk(
|
sample_chunk = Chunk(
|
||||||
|
|
|
||||||
|
|
@ -231,7 +231,6 @@ class TestObjectExtractionServiceIntegration:
|
||||||
id="customer-doc-001",
|
id="customer-doc-001",
|
||||||
user="integration_test",
|
user="integration_test",
|
||||||
collection="test_documents",
|
collection="test_documents",
|
||||||
metadata=[]
|
|
||||||
)
|
)
|
||||||
|
|
||||||
chunk_text = """
|
chunk_text = """
|
||||||
|
|
@ -299,7 +298,6 @@ class TestObjectExtractionServiceIntegration:
|
||||||
id="product-doc-001",
|
id="product-doc-001",
|
||||||
user="integration_test",
|
user="integration_test",
|
||||||
collection="test_documents",
|
collection="test_documents",
|
||||||
metadata=[]
|
|
||||||
)
|
)
|
||||||
|
|
||||||
chunk_text = """
|
chunk_text = """
|
||||||
|
|
@ -373,7 +371,6 @@ class TestObjectExtractionServiceIntegration:
|
||||||
id=chunk_id,
|
id=chunk_id,
|
||||||
user="concurrent_test",
|
user="concurrent_test",
|
||||||
collection="test_collection",
|
collection="test_collection",
|
||||||
metadata=[]
|
|
||||||
)
|
)
|
||||||
chunk = Chunk(metadata=metadata, chunk=text.encode('utf-8'))
|
chunk = Chunk(metadata=metadata, chunk=text.encode('utf-8'))
|
||||||
chunks.append(chunk)
|
chunks.append(chunk)
|
||||||
|
|
@ -470,7 +467,7 @@ class TestObjectExtractionServiceIntegration:
|
||||||
await processor.on_schema_config(integration_config, version=1)
|
await processor.on_schema_config(integration_config, version=1)
|
||||||
|
|
||||||
# Create test chunk
|
# Create test chunk
|
||||||
metadata = Metadata(id="error-test", user="test", collection="test", metadata=[])
|
metadata = Metadata(id="error-test", user="test", collection="test")
|
||||||
chunk = Chunk(metadata=metadata, chunk=b"Some text that will fail to process")
|
chunk = Chunk(metadata=metadata, chunk=b"Some text that will fail to process")
|
||||||
|
|
||||||
mock_msg = MagicMock()
|
mock_msg = MagicMock()
|
||||||
|
|
@ -507,7 +504,6 @@ class TestObjectExtractionServiceIntegration:
|
||||||
id="metadata-test-chunk",
|
id="metadata-test-chunk",
|
||||||
user="test_user",
|
user="test_user",
|
||||||
collection="test_collection",
|
collection="test_collection",
|
||||||
metadata=[] # Could include source document metadata
|
|
||||||
)
|
)
|
||||||
|
|
||||||
chunk = Chunk(
|
chunk = Chunk(
|
||||||
|
|
|
||||||
|
|
@ -120,7 +120,6 @@ class TestRowsCassandraIntegration:
|
||||||
id="doc-001",
|
id="doc-001",
|
||||||
user="test_user",
|
user="test_user",
|
||||||
collection="import_2024",
|
collection="import_2024",
|
||||||
metadata=[]
|
|
||||||
),
|
),
|
||||||
schema_name="customer_records",
|
schema_name="customer_records",
|
||||||
values=[{
|
values=[{
|
||||||
|
|
@ -201,7 +200,7 @@ class TestRowsCassandraIntegration:
|
||||||
|
|
||||||
# Process objects for different schemas
|
# Process objects for different schemas
|
||||||
product_obj = ExtractedObject(
|
product_obj = ExtractedObject(
|
||||||
metadata=Metadata(id="p1", user="shop", collection="catalog", metadata=[]),
|
metadata=Metadata(id="p1", user="shop", collection="catalog"),
|
||||||
schema_name="products",
|
schema_name="products",
|
||||||
values=[{"product_id": "P001", "name": "Widget", "price": "19.99"}],
|
values=[{"product_id": "P001", "name": "Widget", "price": "19.99"}],
|
||||||
confidence=0.9,
|
confidence=0.9,
|
||||||
|
|
@ -209,7 +208,7 @@ class TestRowsCassandraIntegration:
|
||||||
)
|
)
|
||||||
|
|
||||||
order_obj = ExtractedObject(
|
order_obj = ExtractedObject(
|
||||||
metadata=Metadata(id="o1", user="shop", collection="sales", metadata=[]),
|
metadata=Metadata(id="o1", user="shop", collection="sales"),
|
||||||
schema_name="orders",
|
schema_name="orders",
|
||||||
values=[{"order_id": "O001", "customer_id": "C001", "total": "59.97"}],
|
values=[{"order_id": "O001", "customer_id": "C001", "total": "59.97"}],
|
||||||
confidence=0.85,
|
confidence=0.85,
|
||||||
|
|
@ -254,7 +253,7 @@ class TestRowsCassandraIntegration:
|
||||||
)
|
)
|
||||||
|
|
||||||
test_obj = ExtractedObject(
|
test_obj = ExtractedObject(
|
||||||
metadata=Metadata(id="t1", user="test", collection="test", metadata=[]),
|
metadata=Metadata(id="t1", user="test", collection="test"),
|
||||||
schema_name="indexed_data",
|
schema_name="indexed_data",
|
||||||
values=[{
|
values=[{
|
||||||
"id": "123",
|
"id": "123",
|
||||||
|
|
@ -337,7 +336,6 @@ class TestRowsCassandraIntegration:
|
||||||
id="batch-001",
|
id="batch-001",
|
||||||
user="test_user",
|
user="test_user",
|
||||||
collection="batch_import",
|
collection="batch_import",
|
||||||
metadata=[]
|
|
||||||
),
|
),
|
||||||
schema_name="batch_customers",
|
schema_name="batch_customers",
|
||||||
values=[
|
values=[
|
||||||
|
|
@ -391,7 +389,7 @@ class TestRowsCassandraIntegration:
|
||||||
|
|
||||||
# Process empty batch object
|
# Process empty batch object
|
||||||
empty_obj = ExtractedObject(
|
empty_obj = ExtractedObject(
|
||||||
metadata=Metadata(id="empty-1", user="test", collection="empty", metadata=[]),
|
metadata=Metadata(id="empty-1", user="test", collection="empty"),
|
||||||
schema_name="empty_test",
|
schema_name="empty_test",
|
||||||
values=[], # Empty batch
|
values=[], # Empty batch
|
||||||
confidence=1.0,
|
confidence=1.0,
|
||||||
|
|
@ -426,7 +424,7 @@ class TestRowsCassandraIntegration:
|
||||||
)
|
)
|
||||||
|
|
||||||
test_obj = ExtractedObject(
|
test_obj = ExtractedObject(
|
||||||
metadata=Metadata(id="t1", user="test", collection="test", metadata=[]),
|
metadata=Metadata(id="t1", user="test", collection="test"),
|
||||||
schema_name="map_test",
|
schema_name="map_test",
|
||||||
values=[{"id": "123", "name": "Test Item", "count": "42"}],
|
values=[{"id": "123", "name": "Test Item", "count": "42"}],
|
||||||
confidence=0.9,
|
confidence=0.9,
|
||||||
|
|
@ -470,7 +468,7 @@ class TestRowsCassandraIntegration:
|
||||||
)
|
)
|
||||||
|
|
||||||
test_obj = ExtractedObject(
|
test_obj = ExtractedObject(
|
||||||
metadata=Metadata(id="t1", user="test", collection="my_collection", metadata=[]),
|
metadata=Metadata(id="t1", user="test", collection="my_collection"),
|
||||||
schema_name="partition_test",
|
schema_name="partition_test",
|
||||||
values=[{"id": "123", "category": "test"}],
|
values=[{"id": "123", "category": "test"}],
|
||||||
confidence=0.9,
|
confidence=0.9,
|
||||||
|
|
|
||||||
|
|
@ -28,7 +28,6 @@ def sample_text_document():
|
||||||
"""Sample document with moderate length text."""
|
"""Sample document with moderate length text."""
|
||||||
metadata = Metadata(
|
metadata = Metadata(
|
||||||
id="test-doc-1",
|
id="test-doc-1",
|
||||||
metadata=[],
|
|
||||||
user="test-user",
|
user="test-user",
|
||||||
collection="test-collection"
|
collection="test-collection"
|
||||||
)
|
)
|
||||||
|
|
@ -44,7 +43,6 @@ def long_text_document():
|
||||||
"""Long document for testing multiple chunks."""
|
"""Long document for testing multiple chunks."""
|
||||||
metadata = Metadata(
|
metadata = Metadata(
|
||||||
id="test-doc-long",
|
id="test-doc-long",
|
||||||
metadata=[],
|
|
||||||
user="test-user",
|
user="test-user",
|
||||||
collection="test-collection"
|
collection="test-collection"
|
||||||
)
|
)
|
||||||
|
|
@ -61,7 +59,6 @@ def unicode_text_document():
|
||||||
"""Document with various unicode characters."""
|
"""Document with various unicode characters."""
|
||||||
metadata = Metadata(
|
metadata = Metadata(
|
||||||
id="test-doc-unicode",
|
id="test-doc-unicode",
|
||||||
metadata=[],
|
|
||||||
user="test-user",
|
user="test-user",
|
||||||
collection="test-collection"
|
collection="test-collection"
|
||||||
)
|
)
|
||||||
|
|
@ -87,7 +84,6 @@ def empty_text_document():
|
||||||
"""Empty document for edge case testing."""
|
"""Empty document for edge case testing."""
|
||||||
metadata = Metadata(
|
metadata = Metadata(
|
||||||
id="test-doc-empty",
|
id="test-doc-empty",
|
||||||
metadata=[],
|
|
||||||
user="test-user",
|
user="test-user",
|
||||||
collection="test-collection"
|
collection="test-collection"
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -184,7 +184,6 @@ class TestRecursiveChunkerSimple(IsolatedAsyncioTestCase):
|
||||||
mock_text_doc = MagicMock()
|
mock_text_doc = MagicMock()
|
||||||
mock_text_doc.metadata = Metadata(
|
mock_text_doc.metadata = Metadata(
|
||||||
id="test-doc-123",
|
id="test-doc-123",
|
||||||
metadata=[],
|
|
||||||
user="test-user",
|
user="test-user",
|
||||||
collection="test-collection"
|
collection="test-collection"
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -181,7 +181,6 @@ class TestTokenChunkerSimple(IsolatedAsyncioTestCase):
|
||||||
mock_text_doc = MagicMock()
|
mock_text_doc = MagicMock()
|
||||||
mock_text_doc.metadata = Metadata(
|
mock_text_doc.metadata = Metadata(
|
||||||
id="test-doc-456",
|
id="test-doc-456",
|
||||||
metadata=[],
|
|
||||||
user="test-user",
|
user="test-user",
|
||||||
collection="test-collection"
|
collection="test-collection"
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -73,7 +73,6 @@ def sample_triples():
|
||||||
id="test-doc-id",
|
id="test-doc-id",
|
||||||
user="test-user",
|
user="test-user",
|
||||||
collection="default", # This should be overridden
|
collection="default", # This should be overridden
|
||||||
metadata=[]
|
|
||||||
),
|
),
|
||||||
triples=[
|
triples=[
|
||||||
Triple(
|
Triple(
|
||||||
|
|
@ -93,7 +92,6 @@ def sample_graph_embeddings():
|
||||||
id="test-doc-id",
|
id="test-doc-id",
|
||||||
user="test-user",
|
user="test-user",
|
||||||
collection="default", # This should be overridden
|
collection="default", # This should be overridden
|
||||||
metadata=[]
|
|
||||||
),
|
),
|
||||||
entities=[
|
entities=[
|
||||||
EntityEmbeddings(
|
EntityEmbeddings(
|
||||||
|
|
|
||||||
|
|
@ -55,13 +55,6 @@ def sample_objects_message():
|
||||||
return {
|
return {
|
||||||
"metadata": {
|
"metadata": {
|
||||||
"id": "obj-123",
|
"id": "obj-123",
|
||||||
"metadata": [
|
|
||||||
{
|
|
||||||
"s": {"v": "obj-123", "e": False},
|
|
||||||
"p": {"v": "source", "e": False},
|
|
||||||
"o": {"v": "test", "e": False}
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"user": "testuser",
|
"user": "testuser",
|
||||||
"collection": "testcollection"
|
"collection": "testcollection"
|
||||||
},
|
},
|
||||||
|
|
@ -244,7 +237,6 @@ class TestRowsImportMessageProcessing:
|
||||||
assert sent_object.metadata.id == "obj-123"
|
assert sent_object.metadata.id == "obj-123"
|
||||||
assert sent_object.metadata.user == "testuser"
|
assert sent_object.metadata.user == "testuser"
|
||||||
assert sent_object.metadata.collection == "testcollection"
|
assert sent_object.metadata.collection == "testcollection"
|
||||||
assert len(sent_object.metadata.metadata) == 1 # One triple in metadata
|
|
||||||
|
|
||||||
@patch('trustgraph.gateway.dispatch.rows_import.Publisher')
|
@patch('trustgraph.gateway.dispatch.rows_import.Publisher')
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
|
|
@ -277,7 +269,6 @@ class TestRowsImportMessageProcessing:
|
||||||
assert sent_object.values[0]["field1"] == "value1"
|
assert sent_object.values[0]["field1"] == "value1"
|
||||||
assert sent_object.confidence == 1.0 # Default value
|
assert sent_object.confidence == 1.0 # Default value
|
||||||
assert sent_object.source_span == "" # Default value
|
assert sent_object.source_span == "" # Default value
|
||||||
assert len(sent_object.metadata.metadata) == 0 # Default empty list
|
|
||||||
|
|
||||||
@patch('trustgraph.gateway.dispatch.rows_import.Publisher')
|
@patch('trustgraph.gateway.dispatch.rows_import.Publisher')
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
|
|
|
||||||
|
|
@ -29,11 +29,10 @@ class Triple:
|
||||||
self.o = o
|
self.o = o
|
||||||
|
|
||||||
class Metadata:
|
class Metadata:
|
||||||
def __init__(self, id, user, collection, metadata):
|
def __init__(self, id, user, collection):
|
||||||
self.id = id
|
self.id = id
|
||||||
self.user = user
|
self.user = user
|
||||||
self.collection = collection
|
self.collection = collection
|
||||||
self.metadata = metadata
|
|
||||||
|
|
||||||
class Triples:
|
class Triples:
|
||||||
def __init__(self, metadata, triples):
|
def __init__(self, metadata, triples):
|
||||||
|
|
@ -110,7 +109,6 @@ def sample_triples(sample_triple):
|
||||||
id="test-doc-123",
|
id="test-doc-123",
|
||||||
user="test_user",
|
user="test_user",
|
||||||
collection="test_collection",
|
collection="test_collection",
|
||||||
metadata=[]
|
|
||||||
)
|
)
|
||||||
|
|
||||||
return Triples(
|
return Triples(
|
||||||
|
|
@ -126,7 +124,6 @@ def sample_chunk():
|
||||||
id="test-chunk-456",
|
id="test-chunk-456",
|
||||||
user="test_user",
|
user="test_user",
|
||||||
collection="test_collection",
|
collection="test_collection",
|
||||||
metadata=[]
|
|
||||||
)
|
)
|
||||||
|
|
||||||
return Chunk(
|
return Chunk(
|
||||||
|
|
|
||||||
|
|
@ -51,13 +51,6 @@ class TestAgentKgExtractor:
|
||||||
"""Sample metadata for testing"""
|
"""Sample metadata for testing"""
|
||||||
return Metadata(
|
return Metadata(
|
||||||
id="doc123",
|
id="doc123",
|
||||||
metadata=[
|
|
||||||
Triple(
|
|
||||||
s=Term(type=IRI, iri="doc123"),
|
|
||||||
p=Term(type=IRI, iri="http://example.org/type"),
|
|
||||||
o=Term(type=LITERAL, value="document")
|
|
||||||
)
|
|
||||||
]
|
|
||||||
)
|
)
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
|
|
@ -274,7 +267,7 @@ This is not JSON at all
|
||||||
|
|
||||||
def test_process_extraction_data_no_metadata_id(self, agent_extractor):
|
def test_process_extraction_data_no_metadata_id(self, agent_extractor):
|
||||||
"""Test processing when metadata has no ID"""
|
"""Test processing when metadata has no ID"""
|
||||||
metadata = Metadata(id=None, metadata=[])
|
metadata = Metadata(id=None)
|
||||||
data = [
|
data = [
|
||||||
{"type": "definition", "entity": "Test Entity", "definition": "Test definition"}
|
{"type": "definition", "entity": "Test Entity", "definition": "Test definition"}
|
||||||
]
|
]
|
||||||
|
|
@ -345,8 +338,6 @@ This is not JSON at all
|
||||||
assert sent_triples.metadata.id == sample_metadata.id
|
assert sent_triples.metadata.id == sample_metadata.id
|
||||||
assert sent_triples.metadata.user == sample_metadata.user
|
assert sent_triples.metadata.user == sample_metadata.user
|
||||||
assert sent_triples.metadata.collection == sample_metadata.collection
|
assert sent_triples.metadata.collection == sample_metadata.collection
|
||||||
# Note: metadata.metadata is now empty array in the new implementation
|
|
||||||
assert sent_triples.metadata.metadata == []
|
|
||||||
assert len(sent_triples.triples) == 1
|
assert len(sent_triples.triples) == 1
|
||||||
assert sent_triples.triples[0].s.iri == "test:subject"
|
assert sent_triples.triples[0].s.iri == "test:subject"
|
||||||
|
|
||||||
|
|
@ -371,8 +362,6 @@ This is not JSON at all
|
||||||
assert sent_contexts.metadata.id == sample_metadata.id
|
assert sent_contexts.metadata.id == sample_metadata.id
|
||||||
assert sent_contexts.metadata.user == sample_metadata.user
|
assert sent_contexts.metadata.user == sample_metadata.user
|
||||||
assert sent_contexts.metadata.collection == sample_metadata.collection
|
assert sent_contexts.metadata.collection == sample_metadata.collection
|
||||||
# Note: metadata.metadata is now empty array in the new implementation
|
|
||||||
assert sent_contexts.metadata.metadata == []
|
|
||||||
assert len(sent_contexts.entities) == 1
|
assert len(sent_contexts.entities) == 1
|
||||||
assert sent_contexts.entities[0].entity.iri == "test:entity"
|
assert sent_contexts.entities[0].entity.iri == "test:entity"
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -177,13 +177,13 @@ class TestAgentKgExtractionEdgeCases:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
# Test with metadata without ID
|
# Test with metadata without ID
|
||||||
metadata = Metadata(id=None, metadata=[])
|
metadata = Metadata(id=None)
|
||||||
triples, contexts = agent_extractor.process_extraction_data([], metadata)
|
triples, contexts = agent_extractor.process_extraction_data([], metadata)
|
||||||
assert len(triples) == 0
|
assert len(triples) == 0
|
||||||
assert len(contexts) == 0
|
assert len(contexts) == 0
|
||||||
|
|
||||||
# Test with metadata with empty string ID
|
# Test with metadata with empty string ID
|
||||||
metadata = Metadata(id="", metadata=[])
|
metadata = Metadata(id="")
|
||||||
data = [{"type": "definition", "entity": "Test", "definition": "Test def"}]
|
data = [{"type": "definition", "entity": "Test", "definition": "Test def"}]
|
||||||
triples, contexts = agent_extractor.process_extraction_data(data, metadata)
|
triples, contexts = agent_extractor.process_extraction_data(data, metadata)
|
||||||
|
|
||||||
|
|
@ -193,7 +193,7 @@ class TestAgentKgExtractionEdgeCases:
|
||||||
|
|
||||||
def test_process_extraction_data_special_entity_names(self, agent_extractor):
|
def test_process_extraction_data_special_entity_names(self, agent_extractor):
|
||||||
"""Test processing with special characters in entity names"""
|
"""Test processing with special characters in entity names"""
|
||||||
metadata = Metadata(id="doc123", metadata=[])
|
metadata = Metadata(id="doc123")
|
||||||
|
|
||||||
special_entities = [
|
special_entities = [
|
||||||
"Entity with spaces",
|
"Entity with spaces",
|
||||||
|
|
@ -225,7 +225,7 @@ class TestAgentKgExtractionEdgeCases:
|
||||||
|
|
||||||
def test_process_extraction_data_very_long_definitions(self, agent_extractor):
|
def test_process_extraction_data_very_long_definitions(self, agent_extractor):
|
||||||
"""Test processing with very long entity definitions"""
|
"""Test processing with very long entity definitions"""
|
||||||
metadata = Metadata(id="doc123", metadata=[])
|
metadata = Metadata(id="doc123")
|
||||||
|
|
||||||
# Create very long definition
|
# Create very long definition
|
||||||
long_definition = "This is a very long definition. " * 1000
|
long_definition = "This is a very long definition. " * 1000
|
||||||
|
|
@ -247,7 +247,7 @@ class TestAgentKgExtractionEdgeCases:
|
||||||
|
|
||||||
def test_process_extraction_data_duplicate_entities(self, agent_extractor):
|
def test_process_extraction_data_duplicate_entities(self, agent_extractor):
|
||||||
"""Test processing with duplicate entity names"""
|
"""Test processing with duplicate entity names"""
|
||||||
metadata = Metadata(id="doc123", metadata=[])
|
metadata = Metadata(id="doc123")
|
||||||
|
|
||||||
data = [
|
data = [
|
||||||
{"type": "definition", "entity": "Machine Learning", "definition": "First definition"},
|
{"type": "definition", "entity": "Machine Learning", "definition": "First definition"},
|
||||||
|
|
@ -269,7 +269,7 @@ class TestAgentKgExtractionEdgeCases:
|
||||||
|
|
||||||
def test_process_extraction_data_empty_strings(self, agent_extractor):
|
def test_process_extraction_data_empty_strings(self, agent_extractor):
|
||||||
"""Test processing with empty strings in data"""
|
"""Test processing with empty strings in data"""
|
||||||
metadata = Metadata(id="doc123", metadata=[])
|
metadata = Metadata(id="doc123")
|
||||||
|
|
||||||
data = [
|
data = [
|
||||||
{"type": "definition", "entity": "", "definition": "Definition for empty entity"},
|
{"type": "definition", "entity": "", "definition": "Definition for empty entity"},
|
||||||
|
|
@ -291,7 +291,7 @@ class TestAgentKgExtractionEdgeCases:
|
||||||
|
|
||||||
def test_process_extraction_data_nested_json_in_strings(self, agent_extractor):
|
def test_process_extraction_data_nested_json_in_strings(self, agent_extractor):
|
||||||
"""Test processing when definitions contain JSON-like strings"""
|
"""Test processing when definitions contain JSON-like strings"""
|
||||||
metadata = Metadata(id="doc123", metadata=[])
|
metadata = Metadata(id="doc123")
|
||||||
|
|
||||||
data = [
|
data = [
|
||||||
{
|
{
|
||||||
|
|
@ -315,7 +315,7 @@ class TestAgentKgExtractionEdgeCases:
|
||||||
|
|
||||||
def test_process_extraction_data_boolean_object_entity_variations(self, agent_extractor):
|
def test_process_extraction_data_boolean_object_entity_variations(self, agent_extractor):
|
||||||
"""Test processing with various boolean values for object-entity"""
|
"""Test processing with various boolean values for object-entity"""
|
||||||
metadata = Metadata(id="doc123", metadata=[])
|
metadata = Metadata(id="doc123")
|
||||||
|
|
||||||
data = [
|
data = [
|
||||||
# Explicit True
|
# Explicit True
|
||||||
|
|
@ -343,7 +343,7 @@ class TestAgentKgExtractionEdgeCases:
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_emit_empty_collections(self, agent_extractor):
|
async def test_emit_empty_collections(self, agent_extractor):
|
||||||
"""Test emitting empty triples and entity contexts"""
|
"""Test emitting empty triples and entity contexts"""
|
||||||
metadata = Metadata(id="test", metadata=[])
|
metadata = Metadata(id="test")
|
||||||
|
|
||||||
# Test emitting empty triples
|
# Test emitting empty triples
|
||||||
mock_publisher = AsyncMock()
|
mock_publisher = AsyncMock()
|
||||||
|
|
@ -389,7 +389,7 @@ class TestAgentKgExtractionEdgeCases:
|
||||||
|
|
||||||
def test_process_extraction_data_performance_large_dataset(self, agent_extractor):
|
def test_process_extraction_data_performance_large_dataset(self, agent_extractor):
|
||||||
"""Test performance with large extraction datasets"""
|
"""Test performance with large extraction datasets"""
|
||||||
metadata = Metadata(id="large-doc", metadata=[])
|
metadata = Metadata(id="large-doc")
|
||||||
|
|
||||||
# Create large dataset in JSONL format
|
# Create large dataset in JSONL format
|
||||||
num_definitions = 1000
|
num_definitions = 1000
|
||||||
|
|
|
||||||
|
|
@ -314,7 +314,6 @@ class TestObjectExtractionBusinessLogic:
|
||||||
id="test-extraction-001",
|
id="test-extraction-001",
|
||||||
user="test_user",
|
user="test_user",
|
||||||
collection="test_collection",
|
collection="test_collection",
|
||||||
metadata=[]
|
|
||||||
)
|
)
|
||||||
|
|
||||||
values = [{
|
values = [{
|
||||||
|
|
|
||||||
|
|
@ -373,7 +373,6 @@ class TestTripleConstructionLogic:
|
||||||
id="test-doc-123",
|
id="test-doc-123",
|
||||||
user="test_user",
|
user="test_user",
|
||||||
collection="test_collection",
|
collection="test_collection",
|
||||||
metadata=[]
|
|
||||||
)
|
)
|
||||||
|
|
||||||
# Act
|
# Act
|
||||||
|
|
|
||||||
|
|
@ -190,7 +190,6 @@ class TestRowsCassandraStorageLogic:
|
||||||
id="test-001",
|
id="test-001",
|
||||||
user="test_user",
|
user="test_user",
|
||||||
collection="test_collection",
|
collection="test_collection",
|
||||||
metadata=[]
|
|
||||||
),
|
),
|
||||||
schema_name="test_schema",
|
schema_name="test_schema",
|
||||||
values=[{"id": "123", "value": "test_data"}],
|
values=[{"id": "123", "value": "test_data"}],
|
||||||
|
|
@ -252,7 +251,6 @@ class TestRowsCassandraStorageLogic:
|
||||||
id="test-001",
|
id="test-001",
|
||||||
user="test_user",
|
user="test_user",
|
||||||
collection="test_collection",
|
collection="test_collection",
|
||||||
metadata=[]
|
|
||||||
),
|
),
|
||||||
schema_name="multi_index_schema",
|
schema_name="multi_index_schema",
|
||||||
values=[{"id": "123", "category": "electronics", "status": "active"}],
|
values=[{"id": "123", "category": "electronics", "status": "active"}],
|
||||||
|
|
@ -310,7 +308,6 @@ class TestRowsCassandraStorageBatchLogic:
|
||||||
id="batch-001",
|
id="batch-001",
|
||||||
user="test_user",
|
user="test_user",
|
||||||
collection="batch_collection",
|
collection="batch_collection",
|
||||||
metadata=[]
|
|
||||||
),
|
),
|
||||||
schema_name="batch_schema",
|
schema_name="batch_schema",
|
||||||
values=[
|
values=[
|
||||||
|
|
@ -365,7 +362,6 @@ class TestRowsCassandraStorageBatchLogic:
|
||||||
id="empty-001",
|
id="empty-001",
|
||||||
user="test_user",
|
user="test_user",
|
||||||
collection="empty_collection",
|
collection="empty_collection",
|
||||||
metadata=[]
|
|
||||||
),
|
),
|
||||||
schema_name="empty_schema",
|
schema_name="empty_schema",
|
||||||
values=[], # Empty batch
|
values=[], # Empty batch
|
||||||
|
|
|
||||||
|
|
@ -2,19 +2,12 @@ import base64
|
||||||
from typing import Dict, Any
|
from typing import Dict, Any
|
||||||
from ...schema import Document, TextDocument, Chunk, DocumentEmbeddings, ChunkEmbeddings
|
from ...schema import Document, TextDocument, Chunk, DocumentEmbeddings, ChunkEmbeddings
|
||||||
from .base import SendTranslator
|
from .base import SendTranslator
|
||||||
from .metadata import DocumentMetadataTranslator
|
|
||||||
from .primitives import SubgraphTranslator
|
|
||||||
|
|
||||||
|
|
||||||
class DocumentTranslator(SendTranslator):
|
class DocumentTranslator(SendTranslator):
|
||||||
"""Translator for Document schema objects (PDF docs etc.)"""
|
"""Translator for Document schema objects (PDF docs etc.)"""
|
||||||
|
|
||||||
def __init__(self):
|
|
||||||
self.subgraph_translator = SubgraphTranslator()
|
|
||||||
|
|
||||||
def to_pulsar(self, data: Dict[str, Any]) -> Document:
|
def to_pulsar(self, data: Dict[str, Any]) -> Document:
|
||||||
metadata = data.get("metadata", [])
|
|
||||||
|
|
||||||
# Handle base64 content validation
|
# Handle base64 content validation
|
||||||
doc = base64.b64decode(data["data"])
|
doc = base64.b64decode(data["data"])
|
||||||
|
|
||||||
|
|
@ -22,7 +15,6 @@ class DocumentTranslator(SendTranslator):
|
||||||
return Document(
|
return Document(
|
||||||
metadata=Metadata(
|
metadata=Metadata(
|
||||||
id=data.get("id"),
|
id=data.get("id"),
|
||||||
metadata=self.subgraph_translator.to_pulsar(metadata) if metadata else [],
|
|
||||||
user=data.get("user", "trustgraph"),
|
user=data.get("user", "trustgraph"),
|
||||||
collection=data.get("collection", "default"),
|
collection=data.get("collection", "default"),
|
||||||
),
|
),
|
||||||
|
|
@ -42,8 +34,6 @@ class DocumentTranslator(SendTranslator):
|
||||||
metadata_dict["user"] = obj.metadata.user
|
metadata_dict["user"] = obj.metadata.user
|
||||||
if obj.metadata.collection:
|
if obj.metadata.collection:
|
||||||
metadata_dict["collection"] = obj.metadata.collection
|
metadata_dict["collection"] = obj.metadata.collection
|
||||||
if obj.metadata.metadata:
|
|
||||||
metadata_dict["metadata"] = self.subgraph_translator.from_pulsar(obj.metadata.metadata)
|
|
||||||
|
|
||||||
result["metadata"] = metadata_dict
|
result["metadata"] = metadata_dict
|
||||||
|
|
||||||
|
|
@ -53,11 +43,7 @@ class DocumentTranslator(SendTranslator):
|
||||||
class TextDocumentTranslator(SendTranslator):
|
class TextDocumentTranslator(SendTranslator):
|
||||||
"""Translator for TextDocument schema objects"""
|
"""Translator for TextDocument schema objects"""
|
||||||
|
|
||||||
def __init__(self):
|
|
||||||
self.subgraph_translator = SubgraphTranslator()
|
|
||||||
|
|
||||||
def to_pulsar(self, data: Dict[str, Any]) -> TextDocument:
|
def to_pulsar(self, data: Dict[str, Any]) -> TextDocument:
|
||||||
metadata = data.get("metadata", [])
|
|
||||||
charset = data.get("charset", "utf-8")
|
charset = data.get("charset", "utf-8")
|
||||||
|
|
||||||
# Text is base64 encoded in input
|
# Text is base64 encoded in input
|
||||||
|
|
@ -67,7 +53,6 @@ class TextDocumentTranslator(SendTranslator):
|
||||||
return TextDocument(
|
return TextDocument(
|
||||||
metadata=Metadata(
|
metadata=Metadata(
|
||||||
id=data.get("id"),
|
id=data.get("id"),
|
||||||
metadata=self.subgraph_translator.to_pulsar(metadata) if metadata else [],
|
|
||||||
user=data.get("user", "trustgraph"),
|
user=data.get("user", "trustgraph"),
|
||||||
collection=data.get("collection", "default"),
|
collection=data.get("collection", "default"),
|
||||||
),
|
),
|
||||||
|
|
@ -87,8 +72,6 @@ class TextDocumentTranslator(SendTranslator):
|
||||||
metadata_dict["user"] = obj.metadata.user
|
metadata_dict["user"] = obj.metadata.user
|
||||||
if obj.metadata.collection:
|
if obj.metadata.collection:
|
||||||
metadata_dict["collection"] = obj.metadata.collection
|
metadata_dict["collection"] = obj.metadata.collection
|
||||||
if obj.metadata.metadata:
|
|
||||||
metadata_dict["metadata"] = self.subgraph_translator.from_pulsar(obj.metadata.metadata)
|
|
||||||
|
|
||||||
result["metadata"] = metadata_dict
|
result["metadata"] = metadata_dict
|
||||||
|
|
||||||
|
|
@ -98,17 +81,11 @@ class TextDocumentTranslator(SendTranslator):
|
||||||
class ChunkTranslator(SendTranslator):
|
class ChunkTranslator(SendTranslator):
|
||||||
"""Translator for Chunk schema objects"""
|
"""Translator for Chunk schema objects"""
|
||||||
|
|
||||||
def __init__(self):
|
|
||||||
self.subgraph_translator = SubgraphTranslator()
|
|
||||||
|
|
||||||
def to_pulsar(self, data: Dict[str, Any]) -> Chunk:
|
def to_pulsar(self, data: Dict[str, Any]) -> Chunk:
|
||||||
metadata = data.get("metadata", [])
|
|
||||||
|
|
||||||
from ...schema import Metadata
|
from ...schema import Metadata
|
||||||
return Chunk(
|
return Chunk(
|
||||||
metadata=Metadata(
|
metadata=Metadata(
|
||||||
id=data.get("id"),
|
id=data.get("id"),
|
||||||
metadata=self.subgraph_translator.to_pulsar(metadata) if metadata else [],
|
|
||||||
user=data.get("user", "trustgraph"),
|
user=data.get("user", "trustgraph"),
|
||||||
collection=data.get("collection", "default"),
|
collection=data.get("collection", "default"),
|
||||||
),
|
),
|
||||||
|
|
@ -128,8 +105,6 @@ class ChunkTranslator(SendTranslator):
|
||||||
metadata_dict["user"] = obj.metadata.user
|
metadata_dict["user"] = obj.metadata.user
|
||||||
if obj.metadata.collection:
|
if obj.metadata.collection:
|
||||||
metadata_dict["collection"] = obj.metadata.collection
|
metadata_dict["collection"] = obj.metadata.collection
|
||||||
if obj.metadata.metadata:
|
|
||||||
metadata_dict["metadata"] = self.subgraph_translator.from_pulsar(obj.metadata.metadata)
|
|
||||||
|
|
||||||
result["metadata"] = metadata_dict
|
result["metadata"] = metadata_dict
|
||||||
|
|
||||||
|
|
@ -139,9 +114,6 @@ class ChunkTranslator(SendTranslator):
|
||||||
class DocumentEmbeddingsTranslator(SendTranslator):
|
class DocumentEmbeddingsTranslator(SendTranslator):
|
||||||
"""Translator for DocumentEmbeddings schema objects"""
|
"""Translator for DocumentEmbeddings schema objects"""
|
||||||
|
|
||||||
def __init__(self):
|
|
||||||
self.subgraph_translator = SubgraphTranslator()
|
|
||||||
|
|
||||||
def to_pulsar(self, data: Dict[str, Any]) -> DocumentEmbeddings:
|
def to_pulsar(self, data: Dict[str, Any]) -> DocumentEmbeddings:
|
||||||
metadata = data.get("metadata", {})
|
metadata = data.get("metadata", {})
|
||||||
|
|
||||||
|
|
@ -157,7 +129,6 @@ class DocumentEmbeddingsTranslator(SendTranslator):
|
||||||
return DocumentEmbeddings(
|
return DocumentEmbeddings(
|
||||||
metadata=Metadata(
|
metadata=Metadata(
|
||||||
id=metadata.get("id"),
|
id=metadata.get("id"),
|
||||||
metadata=self.subgraph_translator.to_pulsar(metadata.get("metadata", [])),
|
|
||||||
user=metadata.get("user", "trustgraph"),
|
user=metadata.get("user", "trustgraph"),
|
||||||
collection=metadata.get("collection", "default"),
|
collection=metadata.get("collection", "default"),
|
||||||
),
|
),
|
||||||
|
|
@ -183,8 +154,6 @@ class DocumentEmbeddingsTranslator(SendTranslator):
|
||||||
metadata_dict["user"] = obj.metadata.user
|
metadata_dict["user"] = obj.metadata.user
|
||||||
if obj.metadata.collection:
|
if obj.metadata.collection:
|
||||||
metadata_dict["collection"] = obj.metadata.collection
|
metadata_dict["collection"] = obj.metadata.collection
|
||||||
if obj.metadata.metadata:
|
|
||||||
metadata_dict["metadata"] = self.subgraph_translator.from_pulsar(obj.metadata.metadata)
|
|
||||||
|
|
||||||
result["metadata"] = metadata_dict
|
result["metadata"] = metadata_dict
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,6 @@ from ...schema import (
|
||||||
)
|
)
|
||||||
from .base import MessageTranslator
|
from .base import MessageTranslator
|
||||||
from .primitives import ValueTranslator, SubgraphTranslator
|
from .primitives import ValueTranslator, SubgraphTranslator
|
||||||
from .metadata import DocumentMetadataTranslator
|
|
||||||
|
|
||||||
|
|
||||||
class KnowledgeRequestTranslator(MessageTranslator):
|
class KnowledgeRequestTranslator(MessageTranslator):
|
||||||
|
|
@ -21,9 +20,6 @@ class KnowledgeRequestTranslator(MessageTranslator):
|
||||||
triples = Triples(
|
triples = Triples(
|
||||||
metadata=Metadata(
|
metadata=Metadata(
|
||||||
id=data["triples"]["metadata"]["id"],
|
id=data["triples"]["metadata"]["id"],
|
||||||
metadata=self.subgraph_translator.to_pulsar(
|
|
||||||
data["triples"]["metadata"]["metadata"]
|
|
||||||
),
|
|
||||||
user=data["triples"]["metadata"]["user"],
|
user=data["triples"]["metadata"]["user"],
|
||||||
collection=data["triples"]["metadata"]["collection"]
|
collection=data["triples"]["metadata"]["collection"]
|
||||||
),
|
),
|
||||||
|
|
@ -35,9 +31,6 @@ class KnowledgeRequestTranslator(MessageTranslator):
|
||||||
graph_embeddings = GraphEmbeddings(
|
graph_embeddings = GraphEmbeddings(
|
||||||
metadata=Metadata(
|
metadata=Metadata(
|
||||||
id=data["graph-embeddings"]["metadata"]["id"],
|
id=data["graph-embeddings"]["metadata"]["id"],
|
||||||
metadata=self.subgraph_translator.to_pulsar(
|
|
||||||
data["graph-embeddings"]["metadata"]["metadata"]
|
|
||||||
),
|
|
||||||
user=data["graph-embeddings"]["metadata"]["user"],
|
user=data["graph-embeddings"]["metadata"]["user"],
|
||||||
collection=data["graph-embeddings"]["metadata"]["collection"]
|
collection=data["graph-embeddings"]["metadata"]["collection"]
|
||||||
),
|
),
|
||||||
|
|
@ -78,9 +71,6 @@ class KnowledgeRequestTranslator(MessageTranslator):
|
||||||
result["triples"] = {
|
result["triples"] = {
|
||||||
"metadata": {
|
"metadata": {
|
||||||
"id": obj.triples.metadata.id,
|
"id": obj.triples.metadata.id,
|
||||||
"metadata": self.subgraph_translator.from_pulsar(
|
|
||||||
obj.triples.metadata.metadata
|
|
||||||
),
|
|
||||||
"user": obj.triples.metadata.user,
|
"user": obj.triples.metadata.user,
|
||||||
"collection": obj.triples.metadata.collection,
|
"collection": obj.triples.metadata.collection,
|
||||||
},
|
},
|
||||||
|
|
@ -91,9 +81,6 @@ class KnowledgeRequestTranslator(MessageTranslator):
|
||||||
result["graph-embeddings"] = {
|
result["graph-embeddings"] = {
|
||||||
"metadata": {
|
"metadata": {
|
||||||
"id": obj.graph_embeddings.metadata.id,
|
"id": obj.graph_embeddings.metadata.id,
|
||||||
"metadata": self.subgraph_translator.from_pulsar(
|
|
||||||
obj.graph_embeddings.metadata.metadata
|
|
||||||
),
|
|
||||||
"user": obj.graph_embeddings.metadata.user,
|
"user": obj.graph_embeddings.metadata.user,
|
||||||
"collection": obj.graph_embeddings.metadata.collection,
|
"collection": obj.graph_embeddings.metadata.collection,
|
||||||
},
|
},
|
||||||
|
|
@ -130,9 +117,6 @@ class KnowledgeResponseTranslator(MessageTranslator):
|
||||||
"triples": {
|
"triples": {
|
||||||
"metadata": {
|
"metadata": {
|
||||||
"id": obj.triples.metadata.id,
|
"id": obj.triples.metadata.id,
|
||||||
"metadata": self.subgraph_translator.from_pulsar(
|
|
||||||
obj.triples.metadata.metadata
|
|
||||||
),
|
|
||||||
"user": obj.triples.metadata.user,
|
"user": obj.triples.metadata.user,
|
||||||
"collection": obj.triples.metadata.collection,
|
"collection": obj.triples.metadata.collection,
|
||||||
},
|
},
|
||||||
|
|
@ -146,9 +130,6 @@ class KnowledgeResponseTranslator(MessageTranslator):
|
||||||
"graph-embeddings": {
|
"graph-embeddings": {
|
||||||
"metadata": {
|
"metadata": {
|
||||||
"id": obj.graph_embeddings.metadata.id,
|
"id": obj.graph_embeddings.metadata.id,
|
||||||
"metadata": self.subgraph_translator.from_pulsar(
|
|
||||||
obj.graph_embeddings.metadata.metadata
|
|
||||||
),
|
|
||||||
"user": obj.graph_embeddings.metadata.user,
|
"user": obj.graph_embeddings.metadata.user,
|
||||||
"collection": obj.graph_embeddings.metadata.collection,
|
"collection": obj.graph_embeddings.metadata.collection,
|
||||||
},
|
},
|
||||||
|
|
|
||||||
|
|
@ -1,14 +1,10 @@
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass
|
||||||
from .primitives import Triple
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class Metadata:
|
class Metadata:
|
||||||
# Source identifier
|
# Source identifier
|
||||||
id: str = ""
|
id: str = ""
|
||||||
|
|
||||||
# Subgraph
|
|
||||||
metadata: list[Triple] = field(default_factory=list)
|
|
||||||
|
|
||||||
# Collection management
|
# Collection management
|
||||||
user: str = ""
|
user: str = ""
|
||||||
collection: str = ""
|
collection: str = ""
|
||||||
|
|
|
||||||
|
|
@ -178,7 +178,6 @@ class Processor(ChunkingService):
|
||||||
await flow("triples").send(Triples(
|
await flow("triples").send(Triples(
|
||||||
metadata=Metadata(
|
metadata=Metadata(
|
||||||
id=chunk_uri,
|
id=chunk_uri,
|
||||||
metadata=[],
|
|
||||||
user=v.metadata.user,
|
user=v.metadata.user,
|
||||||
collection=v.metadata.collection,
|
collection=v.metadata.collection,
|
||||||
),
|
),
|
||||||
|
|
@ -189,7 +188,6 @@ class Processor(ChunkingService):
|
||||||
r = Chunk(
|
r = Chunk(
|
||||||
metadata=Metadata(
|
metadata=Metadata(
|
||||||
id=chunk_uri,
|
id=chunk_uri,
|
||||||
metadata=[],
|
|
||||||
user=v.metadata.user,
|
user=v.metadata.user,
|
||||||
collection=v.metadata.collection,
|
collection=v.metadata.collection,
|
||||||
),
|
),
|
||||||
|
|
|
||||||
|
|
@ -302,7 +302,6 @@ class Processor(FlowProcessor):
|
||||||
await flow("triples").send(Triples(
|
await flow("triples").send(Triples(
|
||||||
metadata=Metadata(
|
metadata=Metadata(
|
||||||
id=pg_uri,
|
id=pg_uri,
|
||||||
metadata=[],
|
|
||||||
user=v.metadata.user,
|
user=v.metadata.user,
|
||||||
collection=v.metadata.collection,
|
collection=v.metadata.collection,
|
||||||
),
|
),
|
||||||
|
|
@ -314,7 +313,6 @@ class Processor(FlowProcessor):
|
||||||
r = TextDocument(
|
r = TextDocument(
|
||||||
metadata=Metadata(
|
metadata=Metadata(
|
||||||
id=pg_uri,
|
id=pg_uri,
|
||||||
metadata=[],
|
|
||||||
user=v.metadata.user,
|
user=v.metadata.user,
|
||||||
collection=v.metadata.collection,
|
collection=v.metadata.collection,
|
||||||
),
|
),
|
||||||
|
|
|
||||||
|
|
@ -104,7 +104,6 @@ class Processor(FlowProcessor):
|
||||||
tpls = Triples(
|
tpls = Triples(
|
||||||
metadata = Metadata(
|
metadata = Metadata(
|
||||||
id = metadata.id,
|
id = metadata.id,
|
||||||
metadata = [],
|
|
||||||
user = metadata.user,
|
user = metadata.user,
|
||||||
collection = metadata.collection,
|
collection = metadata.collection,
|
||||||
),
|
),
|
||||||
|
|
@ -117,7 +116,6 @@ class Processor(FlowProcessor):
|
||||||
ecs = EntityContexts(
|
ecs = EntityContexts(
|
||||||
metadata = Metadata(
|
metadata = Metadata(
|
||||||
id = metadata.id,
|
id = metadata.id,
|
||||||
metadata = [],
|
|
||||||
user = metadata.user,
|
user = metadata.user,
|
||||||
collection = metadata.collection,
|
collection = metadata.collection,
|
||||||
),
|
),
|
||||||
|
|
@ -216,10 +214,6 @@ class Processor(FlowProcessor):
|
||||||
extraction_data, v.metadata
|
extraction_data, v.metadata
|
||||||
)
|
)
|
||||||
|
|
||||||
# Put document metadata into triples
|
|
||||||
for t in v.metadata.metadata:
|
|
||||||
triples.append(t)
|
|
||||||
|
|
||||||
# Emit outputs
|
# Emit outputs
|
||||||
if triples:
|
if triples:
|
||||||
await self.emit_triples(flow("triples"), v.metadata, triples)
|
await self.emit_triples(flow("triples"), v.metadata, triples)
|
||||||
|
|
|
||||||
|
|
@ -218,7 +218,6 @@ class Processor(FlowProcessor):
|
||||||
flow("triples"),
|
flow("triples"),
|
||||||
Metadata(
|
Metadata(
|
||||||
id=v.metadata.id,
|
id=v.metadata.id,
|
||||||
metadata=[],
|
|
||||||
user=v.metadata.user,
|
user=v.metadata.user,
|
||||||
collection=v.metadata.collection,
|
collection=v.metadata.collection,
|
||||||
),
|
),
|
||||||
|
|
@ -232,7 +231,6 @@ class Processor(FlowProcessor):
|
||||||
flow("entity-contexts"),
|
flow("entity-contexts"),
|
||||||
Metadata(
|
Metadata(
|
||||||
id=v.metadata.id,
|
id=v.metadata.id,
|
||||||
metadata=[],
|
|
||||||
user=v.metadata.user,
|
user=v.metadata.user,
|
||||||
collection=v.metadata.collection,
|
collection=v.metadata.collection,
|
||||||
),
|
),
|
||||||
|
|
|
||||||
|
|
@ -306,10 +306,6 @@ class Processor(FlowProcessor):
|
||||||
flow, chunk, ontology_subset, prompt_variables
|
flow, chunk, ontology_subset, prompt_variables
|
||||||
)
|
)
|
||||||
|
|
||||||
# Add metadata triples
|
|
||||||
for t in v.metadata.metadata:
|
|
||||||
triples.append(t)
|
|
||||||
|
|
||||||
# Generate ontology definition triples
|
# Generate ontology definition triples
|
||||||
ontology_triples = self.build_ontology_triples(ontology_subset)
|
ontology_triples = self.build_ontology_triples(ontology_subset)
|
||||||
|
|
||||||
|
|
@ -558,7 +554,6 @@ class Processor(FlowProcessor):
|
||||||
t = Triples(
|
t = Triples(
|
||||||
metadata=Metadata(
|
metadata=Metadata(
|
||||||
id=metadata.id,
|
id=metadata.id,
|
||||||
metadata=[],
|
|
||||||
user=metadata.user,
|
user=metadata.user,
|
||||||
collection=metadata.collection,
|
collection=metadata.collection,
|
||||||
),
|
),
|
||||||
|
|
@ -571,7 +566,6 @@ class Processor(FlowProcessor):
|
||||||
ec = EntityContexts(
|
ec = EntityContexts(
|
||||||
metadata=Metadata(
|
metadata=Metadata(
|
||||||
id=metadata.id,
|
id=metadata.id,
|
||||||
metadata=[],
|
|
||||||
user=metadata.user,
|
user=metadata.user,
|
||||||
collection=metadata.collection,
|
collection=metadata.collection,
|
||||||
),
|
),
|
||||||
|
|
|
||||||
|
|
@ -219,7 +219,6 @@ class Processor(FlowProcessor):
|
||||||
flow("triples"),
|
flow("triples"),
|
||||||
Metadata(
|
Metadata(
|
||||||
id=v.metadata.id,
|
id=v.metadata.id,
|
||||||
metadata=[],
|
|
||||||
user=v.metadata.user,
|
user=v.metadata.user,
|
||||||
collection=v.metadata.collection,
|
collection=v.metadata.collection,
|
||||||
),
|
),
|
||||||
|
|
|
||||||
|
|
@ -272,7 +272,6 @@ class Processor(FlowProcessor):
|
||||||
extracted = ExtractedObject(
|
extracted = ExtractedObject(
|
||||||
metadata=Metadata(
|
metadata=Metadata(
|
||||||
id=f"{v.metadata.id}:{schema_name}",
|
id=f"{v.metadata.id}:{schema_name}",
|
||||||
metadata=[],
|
|
||||||
user=v.metadata.user,
|
user=v.metadata.user,
|
||||||
collection=v.metadata.collection,
|
collection=v.metadata.collection,
|
||||||
),
|
),
|
||||||
|
|
|
||||||
|
|
@ -53,7 +53,6 @@ class RowsImport:
|
||||||
elt = ExtractedObject(
|
elt = ExtractedObject(
|
||||||
metadata=Metadata(
|
metadata=Metadata(
|
||||||
id=data["metadata"]["id"],
|
id=data["metadata"]["id"],
|
||||||
metadata=to_subgraph(data["metadata"].get("metadata", [])),
|
|
||||||
user=data["metadata"]["user"],
|
user=data["metadata"]["user"],
|
||||||
collection=data["metadata"]["collection"],
|
collection=data["metadata"]["collection"],
|
||||||
),
|
),
|
||||||
|
|
|
||||||
|
|
@ -37,18 +37,17 @@ def serialize_triples(message):
|
||||||
return {
|
return {
|
||||||
"metadata": {
|
"metadata": {
|
||||||
"id": message.metadata.id,
|
"id": message.metadata.id,
|
||||||
"metadata": serialize_subgraph(message.metadata.metadata),
|
|
||||||
"user": message.metadata.user,
|
"user": message.metadata.user,
|
||||||
"collection": message.metadata.collection,
|
"collection": message.metadata.collection,
|
||||||
},
|
},
|
||||||
"triples": serialize_subgraph(message.triples),
|
"triples": serialize_subgraph(message.triples),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def serialize_graph_embeddings(message):
|
def serialize_graph_embeddings(message):
|
||||||
return {
|
return {
|
||||||
"metadata": {
|
"metadata": {
|
||||||
"id": message.metadata.id,
|
"id": message.metadata.id,
|
||||||
"metadata": serialize_subgraph(message.metadata.metadata),
|
|
||||||
"user": message.metadata.user,
|
"user": message.metadata.user,
|
||||||
"collection": message.metadata.collection,
|
"collection": message.metadata.collection,
|
||||||
},
|
},
|
||||||
|
|
@ -61,11 +60,11 @@ def serialize_graph_embeddings(message):
|
||||||
],
|
],
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def serialize_entity_contexts(message):
|
def serialize_entity_contexts(message):
|
||||||
return {
|
return {
|
||||||
"metadata": {
|
"metadata": {
|
||||||
"id": message.metadata.id,
|
"id": message.metadata.id,
|
||||||
"metadata": serialize_subgraph(message.metadata.metadata),
|
|
||||||
"user": message.metadata.user,
|
"user": message.metadata.user,
|
||||||
"collection": message.metadata.collection,
|
"collection": message.metadata.collection,
|
||||||
},
|
},
|
||||||
|
|
@ -78,11 +77,11 @@ def serialize_entity_contexts(message):
|
||||||
],
|
],
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def serialize_document_embeddings(message):
|
def serialize_document_embeddings(message):
|
||||||
return {
|
return {
|
||||||
"metadata": {
|
"metadata": {
|
||||||
"id": message.metadata.id,
|
"id": message.metadata.id,
|
||||||
"metadata": serialize_subgraph(message.metadata.metadata),
|
|
||||||
"user": message.metadata.user,
|
"user": message.metadata.user,
|
||||||
"collection": message.metadata.collection,
|
"collection": message.metadata.collection,
|
||||||
},
|
},
|
||||||
|
|
|
||||||
|
|
@ -48,7 +48,6 @@ class TriplesImport:
|
||||||
elt = Triples(
|
elt = Triples(
|
||||||
metadata=Metadata(
|
metadata=Metadata(
|
||||||
id=data["metadata"]["id"],
|
id=data["metadata"]["id"],
|
||||||
metadata=to_subgraph(data["metadata"]["metadata"]),
|
|
||||||
user=data["metadata"]["user"],
|
user=data["metadata"]["user"],
|
||||||
collection=data["metadata"]["collection"],
|
collection=data["metadata"]["collection"],
|
||||||
),
|
),
|
||||||
|
|
|
||||||
|
|
@ -334,7 +334,6 @@ class Processor(AsyncProcessor):
|
||||||
triples_msg = Triples(
|
triples_msg = Triples(
|
||||||
metadata=Metadata(
|
metadata=Metadata(
|
||||||
id=doc_uri,
|
id=doc_uri,
|
||||||
metadata=[],
|
|
||||||
user=processing.user,
|
user=processing.user,
|
||||||
collection=processing.collection,
|
collection=processing.collection,
|
||||||
),
|
),
|
||||||
|
|
@ -381,7 +380,6 @@ class Processor(AsyncProcessor):
|
||||||
doc = TextDocument(
|
doc = TextDocument(
|
||||||
metadata = Metadata(
|
metadata = Metadata(
|
||||||
id = document.id,
|
id = document.id,
|
||||||
metadata = document.metadata,
|
|
||||||
user = processing.user,
|
user = processing.user,
|
||||||
collection = processing.collection
|
collection = processing.collection
|
||||||
),
|
),
|
||||||
|
|
@ -392,7 +390,6 @@ class Processor(AsyncProcessor):
|
||||||
doc = TextDocument(
|
doc = TextDocument(
|
||||||
metadata = Metadata(
|
metadata = Metadata(
|
||||||
id = document.id,
|
id = document.id,
|
||||||
metadata = document.metadata,
|
|
||||||
user = processing.user,
|
user = processing.user,
|
||||||
collection = processing.collection
|
collection = processing.collection
|
||||||
),
|
),
|
||||||
|
|
@ -408,7 +405,6 @@ class Processor(AsyncProcessor):
|
||||||
doc = Document(
|
doc = Document(
|
||||||
metadata = Metadata(
|
metadata = Metadata(
|
||||||
id = document.id,
|
id = document.id,
|
||||||
metadata = document.metadata,
|
|
||||||
user = processing.user,
|
user = processing.user,
|
||||||
collection = processing.collection
|
collection = processing.collection
|
||||||
),
|
),
|
||||||
|
|
@ -419,7 +415,6 @@ class Processor(AsyncProcessor):
|
||||||
doc = Document(
|
doc = Document(
|
||||||
metadata = Metadata(
|
metadata = Metadata(
|
||||||
id = document.id,
|
id = document.id,
|
||||||
metadata = document.metadata,
|
|
||||||
user = processing.user,
|
user = processing.user,
|
||||||
collection = processing.collection
|
collection = processing.collection
|
||||||
),
|
),
|
||||||
|
|
|
||||||
|
|
@ -243,7 +243,6 @@ class Processor(FlowProcessor):
|
||||||
await flow("explainability").send(Triples(
|
await flow("explainability").send(Triples(
|
||||||
metadata=Metadata(
|
metadata=Metadata(
|
||||||
id=explain_id,
|
id=explain_id,
|
||||||
metadata=[],
|
|
||||||
user=v.user,
|
user=v.user,
|
||||||
collection=v.collection, # Store in user's collection, not separate explainability collection
|
collection=v.collection, # Store in user's collection, not separate explainability collection
|
||||||
),
|
),
|
||||||
|
|
|
||||||
|
|
@ -218,16 +218,6 @@ class KnowledgeTableStore:
|
||||||
|
|
||||||
when = int(time.time() * 1000)
|
when = int(time.time() * 1000)
|
||||||
|
|
||||||
if m.metadata.metadata:
|
|
||||||
metadata = [
|
|
||||||
(
|
|
||||||
*term_to_tuple(v.s), *term_to_tuple(v.p), *term_to_tuple(v.o)
|
|
||||||
)
|
|
||||||
for v in m.metadata.metadata
|
|
||||||
]
|
|
||||||
else:
|
|
||||||
metadata = []
|
|
||||||
|
|
||||||
triples = [
|
triples = [
|
||||||
(
|
(
|
||||||
*term_to_tuple(v.s), *term_to_tuple(v.p), *term_to_tuple(v.o)
|
*term_to_tuple(v.s), *term_to_tuple(v.p), *term_to_tuple(v.o)
|
||||||
|
|
@ -244,7 +234,7 @@ class KnowledgeTableStore:
|
||||||
(
|
(
|
||||||
uuid.uuid4(), m.metadata.user,
|
uuid.uuid4(), m.metadata.user,
|
||||||
m.metadata.id, when,
|
m.metadata.id, when,
|
||||||
metadata, triples,
|
[], triples,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -259,16 +249,6 @@ class KnowledgeTableStore:
|
||||||
|
|
||||||
when = int(time.time() * 1000)
|
when = int(time.time() * 1000)
|
||||||
|
|
||||||
if m.metadata.metadata:
|
|
||||||
metadata = [
|
|
||||||
(
|
|
||||||
*term_to_tuple(v.s), *term_to_tuple(v.p), *term_to_tuple(v.o)
|
|
||||||
)
|
|
||||||
for v in m.metadata.metadata
|
|
||||||
]
|
|
||||||
else:
|
|
||||||
metadata = []
|
|
||||||
|
|
||||||
entities = [
|
entities = [
|
||||||
(
|
(
|
||||||
term_to_tuple(v.entity),
|
term_to_tuple(v.entity),
|
||||||
|
|
@ -286,7 +266,7 @@ class KnowledgeTableStore:
|
||||||
(
|
(
|
||||||
uuid.uuid4(), m.metadata.user,
|
uuid.uuid4(), m.metadata.user,
|
||||||
m.metadata.id, when,
|
m.metadata.id, when,
|
||||||
metadata, entities,
|
[], entities,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -301,16 +281,6 @@ class KnowledgeTableStore:
|
||||||
|
|
||||||
when = int(time.time() * 1000)
|
when = int(time.time() * 1000)
|
||||||
|
|
||||||
if m.metadata.metadata:
|
|
||||||
metadata = [
|
|
||||||
(
|
|
||||||
*term_to_tuple(v.s), *term_to_tuple(v.p), *term_to_tuple(v.o)
|
|
||||||
)
|
|
||||||
for v in m.metadata.metadata
|
|
||||||
]
|
|
||||||
else:
|
|
||||||
metadata = []
|
|
||||||
|
|
||||||
chunks = [
|
chunks = [
|
||||||
(
|
(
|
||||||
v.chunk_id,
|
v.chunk_id,
|
||||||
|
|
@ -328,7 +298,7 @@ class KnowledgeTableStore:
|
||||||
(
|
(
|
||||||
uuid.uuid4(), m.metadata.user,
|
uuid.uuid4(), m.metadata.user,
|
||||||
m.metadata.id, when,
|
m.metadata.id, when,
|
||||||
metadata, chunks,
|
[], chunks,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -423,18 +393,6 @@ class KnowledgeTableStore:
|
||||||
|
|
||||||
for row in resp:
|
for row in resp:
|
||||||
|
|
||||||
if row[2]:
|
|
||||||
metadata = [
|
|
||||||
Triple(
|
|
||||||
s = tuple_to_term(elt[0], elt[1]),
|
|
||||||
p = tuple_to_term(elt[2], elt[3]),
|
|
||||||
o = tuple_to_term(elt[4], elt[5]),
|
|
||||||
)
|
|
||||||
for elt in row[2]
|
|
||||||
]
|
|
||||||
else:
|
|
||||||
metadata = []
|
|
||||||
|
|
||||||
if row[3]:
|
if row[3]:
|
||||||
triples = [
|
triples = [
|
||||||
Triple(
|
Triple(
|
||||||
|
|
@ -453,7 +411,6 @@ class KnowledgeTableStore:
|
||||||
id = document_id,
|
id = document_id,
|
||||||
user = user,
|
user = user,
|
||||||
collection = "default", # FIXME: What to put here?
|
collection = "default", # FIXME: What to put here?
|
||||||
metadata = metadata,
|
|
||||||
),
|
),
|
||||||
triples = triples
|
triples = triples
|
||||||
)
|
)
|
||||||
|
|
@ -482,18 +439,6 @@ class KnowledgeTableStore:
|
||||||
|
|
||||||
for row in resp:
|
for row in resp:
|
||||||
|
|
||||||
if row[2]:
|
|
||||||
metadata = [
|
|
||||||
Triple(
|
|
||||||
s = tuple_to_term(elt[0], elt[1]),
|
|
||||||
p = tuple_to_term(elt[2], elt[3]),
|
|
||||||
o = tuple_to_term(elt[4], elt[5]),
|
|
||||||
)
|
|
||||||
for elt in row[2]
|
|
||||||
]
|
|
||||||
else:
|
|
||||||
metadata = []
|
|
||||||
|
|
||||||
if row[3]:
|
if row[3]:
|
||||||
entities = [
|
entities = [
|
||||||
EntityEmbeddings(
|
EntityEmbeddings(
|
||||||
|
|
@ -511,7 +456,6 @@ class KnowledgeTableStore:
|
||||||
id = document_id,
|
id = document_id,
|
||||||
user = user,
|
user = user,
|
||||||
collection = "default", # FIXME: What to put here?
|
collection = "default", # FIXME: What to put here?
|
||||||
metadata = metadata,
|
|
||||||
),
|
),
|
||||||
entities = entities
|
entities = entities
|
||||||
)
|
)
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue