mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-04-25 00:16:23 +02:00
Remove redundant metadata (#685)
The metadata field (list of triples) in the pipeline Metadata class was redundant. Document metadata triples already flow directly from librarian to triple-store via emit_document_provenance() - they don't need to pass through the extraction pipeline. Additionally, chunker and PDF decoder were overwriting metadata to [] anyway, so any metadata passed through the pipeline was being discarded. Changes: - Remove metadata field from Metadata dataclass (schema/core/metadata.py) - Update all Metadata instantiations to remove metadata=[] parameter - Remove metadata handling from translators (document_loading, knowledge) - Remove metadata consumption from extractors (ontology, agent) - Update gateway serializers and import handlers - Update all unit, integration, and contract tests
This commit is contained in:
parent
1837d73f34
commit
aa4f5c6c00
37 changed files with 106 additions and 343 deletions
|
|
@ -231,7 +231,6 @@ class TestObjectExtractionServiceIntegration:
|
|||
id="customer-doc-001",
|
||||
user="integration_test",
|
||||
collection="test_documents",
|
||||
metadata=[]
|
||||
)
|
||||
|
||||
chunk_text = """
|
||||
|
|
@ -299,7 +298,6 @@ class TestObjectExtractionServiceIntegration:
|
|||
id="product-doc-001",
|
||||
user="integration_test",
|
||||
collection="test_documents",
|
||||
metadata=[]
|
||||
)
|
||||
|
||||
chunk_text = """
|
||||
|
|
@ -373,7 +371,6 @@ class TestObjectExtractionServiceIntegration:
|
|||
id=chunk_id,
|
||||
user="concurrent_test",
|
||||
collection="test_collection",
|
||||
metadata=[]
|
||||
)
|
||||
chunk = Chunk(metadata=metadata, chunk=text.encode('utf-8'))
|
||||
chunks.append(chunk)
|
||||
|
|
@ -470,7 +467,7 @@ class TestObjectExtractionServiceIntegration:
|
|||
await processor.on_schema_config(integration_config, version=1)
|
||||
|
||||
# Create test chunk
|
||||
metadata = Metadata(id="error-test", user="test", collection="test", metadata=[])
|
||||
metadata = Metadata(id="error-test", user="test", collection="test")
|
||||
chunk = Chunk(metadata=metadata, chunk=b"Some text that will fail to process")
|
||||
|
||||
mock_msg = MagicMock()
|
||||
|
|
@ -507,7 +504,6 @@ class TestObjectExtractionServiceIntegration:
|
|||
id="metadata-test-chunk",
|
||||
user="test_user",
|
||||
collection="test_collection",
|
||||
metadata=[] # Could include source document metadata
|
||||
)
|
||||
|
||||
chunk = Chunk(
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue