apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-07-09 05:12:12 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

OntoRAG: Ontology-Based Knowledge Extraction and Query Technical Specification (#523)

Browse source 
* Onto-rag tech spec

* New processor kg-extract-ontology, use 'ontology' objects from config to guide triple extraction

* Also entity contexts

* Integrate with ontology extractor from workbench

This is first phase, the extraction is tested and working, also GraphRAG with the extracted knowledge works
This commit is contained in:
cybermaggedon 2025-11-12 20:38:08 +00:00 • committed by GitHub
parent 4c3db4dbbe
commit c69f5207a4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
28 changed files with 11824 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

1
trustgraph-flow/trustgraph/extract/kg/ontology/__init__.py Normal file
View file

@ -0,0 +1 @@
from . extract import *

848
trustgraph-flow/trustgraph/extract/kg/ontology/extract.py Normal file
View file

@ -0,0 +1,848 @@
"""
OntoRAG: Ontology-based knowledge extraction service.
Extracts ontology-conformant triples from text chunks.
"""
import json
import logging
import asyncio
from typing import List, Dict, Any, Optional
from .... schema import Chunk, Triple, Triples, Metadata, Value
from .... schema import EntityContext, EntityContexts
from .... schema import PromptRequest, PromptResponse
from .... rdf import TRUSTGRAPH_ENTITIES, RDF_TYPE, RDF_LABEL, DEFINITION
from .... base import FlowProcessor, ConsumerSpec, ProducerSpec
from .... base import PromptClientSpec, EmbeddingsClientSpec
from .ontology_loader import OntologyLoader
from .ontology_embedder import OntologyEmbedder
from .vector_store import InMemoryVectorStore
from .text_processor import TextProcessor
from .ontology_selector import OntologySelector, OntologySubset
logger = logging.getLogger(__name__)
default_ident = "kg-extract-ontology"
default_concurrency = 1
# URI prefix mappings for common namespaces
URI_PREFIXES = {
"rdf:": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
"rdfs:": "http://www.w3.org/2000/01/rdf-schema#",
"owl:": "http://www.w3.org/2002/07/owl#",
"skos:": "http://www.w3.org/2004/02/skos/core#",
"schema:": "https://schema.org/",
"xsd:": "http://www.w3.org/2001/XMLSchema#",
}
class Processor(FlowProcessor):
"""Main OntoRAG extraction processor."""
def __init__(self, **params):
id = params.get("id", default_ident)
concurrency = params.get("concurrency", default_concurrency)
super(Processor, self).__init__(
**params | {
"id": id,
"concurrency": concurrency,
}
)
# Register specifications
self.register_specification(
ConsumerSpec(
name="input",
schema=Chunk,
handler=self.on_message,
concurrency=concurrency,
)
)
self.register_specification(
PromptClientSpec(
request_name="prompt-request",
response_name="prompt-response",
)
)
self.register_specification(
EmbeddingsClientSpec(
request_name="embeddings-request",
response_name="embeddings-response"
)
)
self.register_specification(
ProducerSpec(
name="triples",
schema=Triples
)
)
self.register_specification(
ProducerSpec(
name="entity-contexts",
schema=EntityContexts
)
)
# Register config handler for ontology updates
self.register_config_handler(self.on_ontology_config)
# Shared components (not flow-specific)
self.ontology_loader = OntologyLoader()
self.text_processor = TextProcessor()
# Per-flow components (each flow gets its own embedder/vector store/selector)
self.flow_components = {} # flow_id -> {embedder, vector_store, selector}
# Configuration
self.top_k = params.get("top_k", 10)
self.similarity_threshold = params.get("similarity_threshold", 0.3)
# Track loaded ontology version
self.current_ontology_version = None
self.loaded_ontology_ids = set()
async def initialize_flow_components(self, flow):
"""Initialize per-flow OntoRAG components.
Each flow gets its own vector store and embedder to support
different embedding models across flows. The vector store dimension
is auto-detected from the embeddings service.
Args:
flow: Flow object for this processing context
Returns:
flow_id: Identifier for this flow's components
"""
# Use flow object as identifier
flow_id = id(flow)
if flow_id in self.flow_components:
return flow_id # Already initialized for this flow
try:
logger.info(f"Initializing components for flow {flow_id}")
# Use embeddings client directly (no wrapper needed)
embeddings_client = flow("embeddings-request")
# Detect embedding dimension by embedding a test string
logger.info("Detecting embedding dimension from embeddings service...")
test_embedding_response = await embeddings_client.embed("test")
test_embedding = test_embedding_response[0] # Extract from [[vector]]
dimension = len(test_embedding)
logger.info(f"Detected embedding dimension: {dimension}")
# Initialize vector store with detected dimension
vector_store = InMemoryVectorStore(
dimension=dimension,
index_type='flat'
)
ontology_embedder = OntologyEmbedder(
embedding_service=embeddings_client,
vector_store=vector_store
)
# Embed all loaded ontologies for this flow
if self.ontology_loader.get_all_ontologies():
logger.info(f"Embedding ontologies for flow {flow_id}")
for ont_id, ontology in self.ontology_loader.get_all_ontologies().items():
await ontology_embedder.embed_ontology(ontology)
logger.info(f"Embedded {ontology_embedder.get_embedded_count()} ontology elements for flow {flow_id}")
# Initialize ontology selector
ontology_selector = OntologySelector(
ontology_embedder=ontology_embedder,
ontology_loader=self.ontology_loader,
top_k=self.top_k,
similarity_threshold=self.similarity_threshold
)
# Store flow-specific components
self.flow_components[flow_id] = {
'embedder': ontology_embedder,
'vector_store': vector_store,
'selector': ontology_selector,
'dimension': dimension
}
logger.info(f"Flow {flow_id} components initialized successfully (dimension={dimension})")
return flow_id
except Exception as e:
logger.error(f"Failed to initialize flow {flow_id} components: {e}", exc_info=True)
raise
async def on_ontology_config(self, config, version):
"""
Handle ontology configuration updates from ConfigPush queue.
Parses and stores ontologies. Embedding happens per-flow on first message.
Called automatically when:
- Processor starts (gets full config history via start_of_messages=True)
- Config service pushes updates (immediate event-driven notification)
Args:
config: Full configuration map - config[type][key] = value
version: Config version number (monotonically increasing)
"""
try:
logger.info(f"Received ontology config update, version={version}")
# Skip if we've already processed this version
if version == self.current_ontology_version:
logger.debug(f"Already at version {version}, skipping")
return
# Extract ontology configurations
if "ontology" not in config:
logger.warning("No 'ontology' section in config")
return
ontology_configs = config["ontology"]
# Parse ontology definitions
ontologies = {}
for ont_id, ont_json in ontology_configs.items():
try:
ontologies[ont_id] = json.loads(ont_json)
except json.JSONDecodeError as e:
logger.error(f"Failed to parse ontology '{ont_id}': {e}")
continue
logger.info(f"Loaded {len(ontologies)} ontology definitions")
# Determine what changed (for incremental updates)
new_ids = set(ontologies.keys())
added_ids = new_ids - self.loaded_ontology_ids
removed_ids = self.loaded_ontology_ids - new_ids
updated_ids = new_ids & self.loaded_ontology_ids # May have changed content
if added_ids:
logger.info(f"New ontologies: {added_ids}")
if removed_ids:
logger.info(f"Removed ontologies: {removed_ids}")
if updated_ids:
logger.info(f"Updated ontologies: {updated_ids}")
# Update ontology loader's internal state
self.ontology_loader.update_ontologies(ontologies)
# Clear all flow components to force re-embedding with new ontologies
if added_ids or removed_ids or updated_ids:
logger.info("Clearing flow components to trigger re-embedding")
self.flow_components.clear()
# Update tracking
self.current_ontology_version = version
self.loaded_ontology_ids = new_ids
logger.info(f"Ontology config update complete, version={version}")
except Exception as e:
logger.error(f"Failed to process ontology config: {e}", exc_info=True)
async def on_message(self, msg, consumer, flow):
"""Process incoming chunk message."""
v = msg.value()
logger.info(f"Extracting ontology-based triples from {v.metadata.id}...")
# Initialize flow-specific components if needed
flow_id = await self.initialize_flow_components(flow)
components = self.flow_components[flow_id]
chunk = v.chunk.decode("utf-8")
logger.debug(f"Processing chunk: {chunk[:200]}...")
try:
# Process text into segments
segments = self.text_processor.process_chunk(chunk, extract_phrases=True)
logger.debug(f"Split chunk into {len(segments)} segments")
# Select relevant ontology subset (using flow-specific selector)
ontology_subsets = await components['selector'].select_ontology_subset(segments)
if not ontology_subsets:
logger.warning("No relevant ontology elements found for chunk")
# Emit empty outputs
await self.emit_triples(
flow("triples"),
v.metadata,
[]
)
await self.emit_entity_contexts(
flow("entity-contexts"),
v.metadata,
[]
)
return
# Merge subsets if multiple ontologies matched
if len(ontology_subsets) > 1:
ontology_subset = components['selector'].merge_subsets(ontology_subsets)
else:
ontology_subset = ontology_subsets[0]
logger.debug(f"Selected ontology subset with {len(ontology_subset.classes)} classes, "
f"{len(ontology_subset.object_properties)} object properties, "
f"{len(ontology_subset.datatype_properties)} datatype properties")
# Build extraction prompt variables
prompt_variables = self.build_extraction_variables(chunk, ontology_subset)
# Call prompt service for extraction
try:
# Use prompt() method with extract-with-ontologies prompt ID
triples_response = await flow("prompt-request").prompt(
id="extract-with-ontologies",
variables=prompt_variables
)
logger.debug(f"Extraction response: {triples_response}")
if not isinstance(triples_response, list):
logger.error("Expected list of triples from prompt service")
triples_response = []
except Exception as e:
logger.error(f"Prompt service error: {e}", exc_info=True)
triples_response = []
# Parse and validate triples
triples = self.parse_and_validate_triples(triples_response, ontology_subset)
# Add metadata triples
for t in v.metadata.metadata:
triples.append(t)
# Generate ontology definition triples
ontology_triples = self.build_ontology_triples(ontology_subset)
# Combine extracted triples with ontology triples
all_triples = triples + ontology_triples
# Build entity contexts from all triples (including ontology elements)
entity_contexts = self.build_entity_contexts(all_triples)
# Emit all triples (extracted + ontology definitions)
await self.emit_triples(
flow("triples"),
v.metadata,
all_triples
)
# Emit entity contexts
await self.emit_entity_contexts(
flow("entity-contexts"),
v.metadata,
entity_contexts
)
logger.info(f"Extracted {len(triples)} content triples + {len(ontology_triples)} ontology triples "
f"= {len(all_triples)} total triples and {len(entity_contexts)} entity contexts")
except Exception as e:
logger.error(f"OntoRAG extraction exception: {e}", exc_info=True)
# Emit empty outputs on error
await self.emit_triples(
flow("triples"),
v.metadata,
[]
)
await self.emit_entity_contexts(
flow("entity-contexts"),
v.metadata,
[]
)
def build_extraction_variables(self, chunk: str, ontology_subset: OntologySubset) -> Dict[str, Any]:
"""Build variables for ontology-based extraction prompt template.
Args:
chunk: Text chunk to extract from
ontology_subset: Relevant ontology elements
Returns:
Dict with template variables: text, classes, object_properties, datatype_properties
"""
return {
"text": chunk,
"classes": ontology_subset.classes,
"object_properties": ontology_subset.object_properties,
"datatype_properties": ontology_subset.datatype_properties
}
def parse_and_validate_triples(self, triples_response: List[Any],
ontology_subset: OntologySubset) -> List[Triple]:
"""Parse and validate extracted triples against ontology."""
validated_triples = []
ontology_id = ontology_subset.ontology_id
for triple_data in triples_response:
try:
if isinstance(triple_data, dict):
subject = triple_data.get('subject', '')
predicate = triple_data.get('predicate', '')
object_val = triple_data.get('object', '')
if not subject or not predicate or not object_val:
continue
# Validate against ontology
if self.is_valid_triple(subject, predicate, object_val, ontology_subset):
# Expand URIs before creating Value objects
subject_uri = self.expand_uri(subject, ontology_subset, ontology_id)
predicate_uri = self.expand_uri(predicate, ontology_subset, ontology_id)
# Object might be URI or literal - check before expanding
if self.is_uri(object_val) or self.should_expand_as_uri(object_val, ontology_subset):
object_uri = self.expand_uri(object_val, ontology_subset, ontology_id)
is_object_uri = True
else:
object_uri = object_val
is_object_uri = False
# Create Triple object with expanded URIs
s_value = Value(value=subject_uri, is_uri=True)
p_value = Value(value=predicate_uri, is_uri=True)
o_value = Value(value=object_uri, is_uri=is_object_uri)
validated_triples.append(Triple(
s=s_value,
p=p_value,
o=o_value
))
else:
logger.debug(f"Invalid triple: ({subject}, {predicate}, {object_val})")
except Exception as e:
logger.error(f"Error parsing triple: {e}")
return validated_triples
def should_expand_as_uri(self, value: str, ontology_subset: OntologySubset) -> bool:
"""Check if a value should be treated as URI (not literal).
Returns True if value is a class name, property name, or entity reference.
"""
# Check if it's a class or property from ontology
if value in ontology_subset.classes:
return True
if value in ontology_subset.object_properties:
return True
if value in ontology_subset.datatype_properties:
return True
# Check if it starts with a known prefix
for prefix in URI_PREFIXES.keys():
if value.startswith(prefix):
return True
# Check if it looks like an entity reference (e.g., "recipe:cornish-pasty")
if ":" in value and not value.startswith("http"):
return True
return False
def is_valid_triple(self, subject: str, predicate: str, object_val: str,
ontology_subset: OntologySubset) -> bool:
"""Validate triple against ontology constraints."""
# Special case for rdf:type
if predicate == "rdf:type" or predicate == str(RDF_TYPE):
# Check if object is a valid class
return object_val in ontology_subset.classes
# Special case for rdfs:label
if predicate == "rdfs:label" or predicate == str(RDF_LABEL):
return True # Labels are always valid
# Check if predicate is a valid property
is_obj_prop = predicate in ontology_subset.object_properties
is_dt_prop = predicate in ontology_subset.datatype_properties
if not is_obj_prop and not is_dt_prop:
return False # Unknown property
# TODO: Add more sophisticated validation (domain/range checking)
return True
def expand_uri(self, value: str, ontology_subset: OntologySubset, ontology_id: str = "unknown") -> str:
"""Expand prefix notation or short names to full URIs.
Args:
value: Value to expand (e.g., "rdf:type", "Recipe", "has_ingredient")
ontology_subset: Ontology subset for class/property lookup
ontology_id: ID of the ontology for constructing instance URIs
Returns:
Full URI string
"""
# Already a full URI
if value.startswith("http://") or value.startswith("https://"):
return value
# Check standard prefixes (rdf:, rdfs:, etc.)
for prefix, namespace in URI_PREFIXES.items():
if value.startswith(prefix):
return namespace + value[len(prefix):]
# Check if it's an ontology class
if value in ontology_subset.classes:
class_def = ontology_subset.classes[value]
# class_def is a dict (from cls.__dict__ in ontology_selector)
if isinstance(class_def, dict) and 'uri' in class_def and class_def['uri']:
return class_def['uri']
# Fallback: construct URI
return f"https://trustgraph.ai/ontology/{ontology_id}#{value}"
# Check if it's an ontology property
if value in ontology_subset.object_properties:
prop_def = ontology_subset.object_properties[value]
# prop_def is a dict (from prop.__dict__ in ontology_selector)
if isinstance(prop_def, dict) and 'uri' in prop_def and prop_def['uri']:
return prop_def['uri']
return f"https://trustgraph.ai/ontology/{ontology_id}#{value}"
if value in ontology_subset.datatype_properties:
prop_def = ontology_subset.datatype_properties[value]
# prop_def is a dict (from prop.__dict__ in ontology_selector)
if isinstance(prop_def, dict) and 'uri' in prop_def and prop_def['uri']:
return prop_def['uri']
return f"https://trustgraph.ai/ontology/{ontology_id}#{value}"
# Otherwise, treat as entity instance - construct unique URI
# Normalize the value for URI (lowercase, replace spaces with hyphens)
normalized = value.replace(" ", "-").lower()
return f"https://trustgraph.ai/{ontology_id}/{normalized}"
def is_uri(self, value: str) -> bool:
"""Check if value is already a full URI."""
return value.startswith("http://") or value.startswith("https://")
async def emit_triples(self, pub, metadata: Metadata, triples: List[Triple]):
"""Emit triples to output."""
t = Triples(
metadata=Metadata(
id=metadata.id,
metadata=[],
user=metadata.user,
collection=metadata.collection,
),
triples=triples,
)
await pub.send(t)
async def emit_entity_contexts(self, pub, metadata: Metadata, entities: List[EntityContext]):
"""Emit entity contexts to output."""
ec = EntityContexts(
metadata=Metadata(
id=metadata.id,
metadata=[],
user=metadata.user,
collection=metadata.collection,
),
entities=entities,
)
await pub.send(ec)
def build_ontology_triples(self, ontology_subset: OntologySubset) -> List[Triple]:
"""Build triples describing the ontology elements themselves.
Generates triples for classes and properties so they exist in the knowledge graph.
Args:
ontology_subset: The ontology subset used for extraction
Returns:
List of Triple objects describing ontology elements
"""
ontology_triples = []
# Generate triples for classes
for class_id, class_def in ontology_subset.classes.items():
# Get URI for class
if isinstance(class_def, dict) and 'uri' in class_def and class_def['uri']:
class_uri = class_def['uri']
else:
# Fallback to constructed URI
class_uri = f"https://trustgraph.ai/ontology/{ontology_subset.ontology_id}#{class_id}"
# rdf:type owl:Class
ontology_triples.append(Triple(
s=Value(value=class_uri, is_uri=True),
p=Value(value="http://www.w3.org/1999/02/22-rdf-syntax-ns#type", is_uri=True),
o=Value(value="http://www.w3.org/2002/07/owl#Class", is_uri=True)
))
# rdfs:label (stored as 'labels' in OntologyClass.__dict__)
if isinstance(class_def, dict) and 'labels' in class_def:
labels = class_def['labels']
if isinstance(labels, list) and labels:
label_val = labels[0].get('value', class_id) if isinstance(labels[0], dict) else str(labels[0])
ontology_triples.append(Triple(
s=Value(value=class_uri, is_uri=True),
p=Value(value=RDF_LABEL, is_uri=True),
o=Value(value=label_val, is_uri=False)
))
# rdfs:comment (stored as 'comment' in OntologyClass.__dict__)
if isinstance(class_def, dict) and 'comment' in class_def and class_def['comment']:
comment = class_def['comment']
ontology_triples.append(Triple(
s=Value(value=class_uri, is_uri=True),
p=Value(value="http://www.w3.org/2000/01/rdf-schema#comment", is_uri=True),
o=Value(value=comment, is_uri=False)
))
# rdfs:subClassOf (stored as 'subclass_of' in OntologyClass.__dict__)
if isinstance(class_def, dict) and 'subclass_of' in class_def and class_def['subclass_of']:
parent = class_def['subclass_of']
# Get parent URI
if parent in ontology_subset.classes:
parent_class_def = ontology_subset.classes[parent]
if isinstance(parent_class_def, dict) and 'uri' in parent_class_def and parent_class_def['uri']:
parent_uri = parent_class_def['uri']
else:
parent_uri = f"https://trustgraph.ai/ontology/{ontology_subset.ontology_id}#{parent}"
else:
parent_uri = f"https://trustgraph.ai/ontology/{ontology_subset.ontology_id}#{parent}"
ontology_triples.append(Triple(
s=Value(value=class_uri, is_uri=True),
p=Value(value="http://www.w3.org/2000/01/rdf-schema#subClassOf", is_uri=True),
o=Value(value=parent_uri, is_uri=True)
))
# Generate triples for object properties
for prop_id, prop_def in ontology_subset.object_properties.items():
# Get URI for property
if isinstance(prop_def, dict) and 'uri' in prop_def and prop_def['uri']:
prop_uri = prop_def['uri']
else:
prop_uri = f"https://trustgraph.ai/ontology/{ontology_subset.ontology_id}#{prop_id}"
# rdf:type owl:ObjectProperty
ontology_triples.append(Triple(
s=Value(value=prop_uri, is_uri=True),
p=Value(value="http://www.w3.org/1999/02/22-rdf-syntax-ns#type", is_uri=True),
o=Value(value="http://www.w3.org/2002/07/owl#ObjectProperty", is_uri=True)
))
# rdfs:label (stored as 'labels' in OntologyProperty.__dict__)
if isinstance(prop_def, dict) and 'labels' in prop_def:
labels = prop_def['labels']
if isinstance(labels, list) and labels:
label_val = labels[0].get('value', prop_id) if isinstance(labels[0], dict) else str(labels[0])
ontology_triples.append(Triple(
s=Value(value=prop_uri, is_uri=True),
p=Value(value=RDF_LABEL, is_uri=True),
o=Value(value=label_val, is_uri=False)
))
# rdfs:comment (stored as 'comment' in OntologyProperty.__dict__)
if isinstance(prop_def, dict) and 'comment' in prop_def and prop_def['comment']:
comment = prop_def['comment']
ontology_triples.append(Triple(
s=Value(value=prop_uri, is_uri=True),
p=Value(value="http://www.w3.org/2000/01/rdf-schema#comment", is_uri=True),
o=Value(value=comment, is_uri=False)
))
# rdfs:domain (stored as 'domain' in OntologyProperty.__dict__)
if isinstance(prop_def, dict) and 'domain' in prop_def and prop_def['domain']:
domain = prop_def['domain']
# Get domain class URI
if domain in ontology_subset.classes:
domain_class_def = ontology_subset.classes[domain]
if isinstance(domain_class_def, dict) and 'uri' in domain_class_def and domain_class_def['uri']:
domain_uri = domain_class_def['uri']
else:
domain_uri = f"https://trustgraph.ai/ontology/{ontology_subset.ontology_id}#{domain}"
else:
domain_uri = f"https://trustgraph.ai/ontology/{ontology_subset.ontology_id}#{domain}"
ontology_triples.append(Triple(
s=Value(value=prop_uri, is_uri=True),
p=Value(value="http://www.w3.org/2000/01/rdf-schema#domain", is_uri=True),
o=Value(value=domain_uri, is_uri=True)
))
# rdfs:range (stored as 'range' in OntologyProperty.__dict__)
if isinstance(prop_def, dict) and 'range' in prop_def and prop_def['range']:
range_val = prop_def['range']
# Get range class URI
if range_val in ontology_subset.classes:
range_class_def = ontology_subset.classes[range_val]
if isinstance(range_class_def, dict) and 'uri' in range_class_def and range_class_def['uri']:
range_uri = range_class_def['uri']
else:
range_uri = f"https://trustgraph.ai/ontology/{ontology_subset.ontology_id}#{range_val}"
else:
range_uri = f"https://trustgraph.ai/ontology/{ontology_subset.ontology_id}#{range_val}"
ontology_triples.append(Triple(
s=Value(value=prop_uri, is_uri=True),
p=Value(value="http://www.w3.org/2000/01/rdf-schema#range", is_uri=True),
o=Value(value=range_uri, is_uri=True)
))
# Generate triples for datatype properties
for prop_id, prop_def in ontology_subset.datatype_properties.items():
# Get URI for property
if isinstance(prop_def, dict) and 'uri' in prop_def and prop_def['uri']:
prop_uri = prop_def['uri']
else:
prop_uri = f"https://trustgraph.ai/ontology/{ontology_subset.ontology_id}#{prop_id}"
# rdf:type owl:DatatypeProperty
ontology_triples.append(Triple(
s=Value(value=prop_uri, is_uri=True),
p=Value(value="http://www.w3.org/1999/02/22-rdf-syntax-ns#type", is_uri=True),
o=Value(value="http://www.w3.org/2002/07/owl#DatatypeProperty", is_uri=True)
))
# rdfs:label (stored as 'labels' in OntologyProperty.__dict__)
if isinstance(prop_def, dict) and 'labels' in prop_def:
labels = prop_def['labels']
if isinstance(labels, list) and labels:
label_val = labels[0].get('value', prop_id) if isinstance(labels[0], dict) else str(labels[0])
ontology_triples.append(Triple(
s=Value(value=prop_uri, is_uri=True),
p=Value(value=RDF_LABEL, is_uri=True),
o=Value(value=label_val, is_uri=False)
))
# rdfs:comment (stored as 'comment' in OntologyProperty.__dict__)
if isinstance(prop_def, dict) and 'comment' in prop_def and prop_def['comment']:
comment = prop_def['comment']
ontology_triples.append(Triple(
s=Value(value=prop_uri, is_uri=True),
p=Value(value="http://www.w3.org/2000/01/rdf-schema#comment", is_uri=True),
o=Value(value=comment, is_uri=False)
))
# rdfs:domain (stored as 'domain' in OntologyProperty.__dict__)
if isinstance(prop_def, dict) and 'domain' in prop_def and prop_def['domain']:
domain = prop_def['domain']
# Get domain class URI
if domain in ontology_subset.classes:
domain_class_def = ontology_subset.classes[domain]
if isinstance(domain_class_def, dict) and 'uri' in domain_class_def and domain_class_def['uri']:
domain_uri = domain_class_def['uri']
else:
domain_uri = f"https://trustgraph.ai/ontology/{ontology_subset.ontology_id}#{domain}"
else:
domain_uri = f"https://trustgraph.ai/ontology/{ontology_subset.ontology_id}#{domain}"
ontology_triples.append(Triple(
s=Value(value=prop_uri, is_uri=True),
p=Value(value="http://www.w3.org/2000/01/rdf-schema#domain", is_uri=True),
o=Value(value=domain_uri, is_uri=True)
))
# rdfs:range (datatype)
if isinstance(prop_def, dict) and 'rdfs:range' in prop_def and prop_def['rdfs:range']:
range_val = prop_def['rdfs:range']
# Range for datatype properties is usually xsd:string, xsd:int, etc.
if range_val.startswith('xsd:'):
range_uri = f"http://www.w3.org/2001/XMLSchema#{range_val[4:]}"
else:
range_uri = range_val
ontology_triples.append(Triple(
s=Value(value=prop_uri, is_uri=True),
p=Value(value="http://www.w3.org/2000/01/rdf-schema#range", is_uri=True),
o=Value(value=range_uri, is_uri=True)
))
logger.info(f"Generated {len(ontology_triples)} triples describing ontology elements")
return ontology_triples
def build_entity_contexts(self, triples: List[Triple]) -> List[EntityContext]:
"""Build entity contexts from extracted triples.
Collects rdfs:label and definition properties for each entity to create
contextual descriptions for embedding.
Args:
triples: List of extracted triples
Returns:
List of EntityContext objects
"""
# Group triples by subject to collect entity information
entity_data = {} # subject_uri -> {labels: [], definitions: []}
for triple in triples:
subject_uri = triple.s.value
predicate_uri = triple.p.value
object_val = triple.o.value
# Initialize entity data if not exists
if subject_uri not in entity_data:
entity_data[subject_uri] = {'labels': [], 'definitions': []}
# Collect labels (rdfs:label)
if predicate_uri == RDF_LABEL:
if not triple.o.is_uri: # Labels are literals
entity_data[subject_uri]['labels'].append(object_val)
# Collect definitions (skos:definition, schema:description)
elif predicate_uri == DEFINITION or predicate_uri == "https://schema.org/description":
if not triple.o.is_uri:
entity_data[subject_uri]['definitions'].append(object_val)
# Build EntityContext objects
entity_contexts = []
for subject_uri, data in entity_data.items():
# Build context text from labels and definitions
context_parts = []
if data['labels']:
context_parts.append(f"Label: {data['labels'][0]}")
if data['definitions']:
context_parts.extend(data['definitions'])
# Only create EntityContext if we have meaningful context
if context_parts:
context_text = ". ".join(context_parts)
entity_contexts.append(EntityContext(
entity=Value(value=subject_uri, is_uri=True),
context=context_text
))
logger.debug(f"Built {len(entity_contexts)} entity contexts from {len(triples)} triples")
return entity_contexts
@staticmethod
def add_args(parser):
"""Add command-line arguments."""
parser.add_argument(
'-c', '--concurrency',
type=int,
default=default_concurrency,
help=f'Concurrent processing threads (default: {default_concurrency})'
)
parser.add_argument(
'--top-k',
type=int,
default=10,
help='Number of top ontology elements to retrieve (default: 10)'
)
parser.add_argument(
'--similarity-threshold',
type=float,
default=0.3,
help='Similarity threshold for ontology matching (default: 0.3, range: 0.0-1.0)'
)
FlowProcessor.add_args(parser)
def run():
"""Launch the OntoRAG extraction service."""
Processor.launch(default_ident, __doc__)

310
trustgraph-flow/trustgraph/extract/kg/ontology/ontology_embedder.py Normal file
View file

@ -0,0 +1,310 @@
"""
Ontology embedder component for OntoRAG system.
Generates and stores embeddings for ontology elements.
"""
import asyncio
import logging
import numpy as np
from typing import Dict, List, Any, Optional
from dataclasses import dataclass
from .ontology_loader import Ontology, OntologyClass, OntologyProperty
from .vector_store import InMemoryVectorStore
logger = logging.getLogger(__name__)
@dataclass
class OntologyElementMetadata:
"""Metadata for an embedded ontology element."""
type: str # 'class', 'objectProperty', 'datatypeProperty'
ontology: str # Ontology ID
element: str # Element ID
definition: Dict[str, Any] # Full element definition
text: str # Text used for embedding
class OntologyEmbedder:
"""Generates embeddings for ontology elements and stores them in vector store."""
def __init__(self, embedding_service=None, vector_store: Optional[InMemoryVectorStore] = None):
"""Initialize the ontology embedder.
Args:
embedding_service: Service for generating embeddings
vector_store: Vector store instance (InMemoryVectorStore)
"""
self.embedding_service = embedding_service
self.vector_store = vector_store or InMemoryVectorStore()
self.embedded_ontologies = set()
def _create_text_representation(self, element_id: str, element: Any,
element_type: str) -> str:
"""Create text representation of an ontology element for embedding.
Args:
element_id: ID of the element
element: The element object (OntologyClass or OntologyProperty)
element_type: Type of element
Returns:
Text representation for embedding
"""
parts = []
# Add the element ID (often meaningful)
parts.append(element_id.replace('-', ' ').replace('_', ' '))
# Add labels
if hasattr(element, 'labels') and element.labels:
for label in element.labels:
if isinstance(label, dict):
parts.append(label.get('value', ''))
else:
parts.append(str(label))
# Add comment/description
if hasattr(element, 'comment') and element.comment:
parts.append(element.comment)
# Add type-specific information
if element_type == 'class':
if hasattr(element, 'subclass_of') and element.subclass_of:
parts.append(f"subclass of {element.subclass_of}")
elif element_type in ['objectProperty', 'datatypeProperty']:
if hasattr(element, 'domain') and element.domain:
parts.append(f"domain: {element.domain}")
if hasattr(element, 'range') and element.range:
parts.append(f"range: {element.range}")
# Join all parts with spaces
text = ' '.join(filter(None, parts))
return text
async def embed_ontology(self, ontology: Ontology) -> int:
"""Generate and store embeddings for all elements in an ontology.
Args:
ontology: The ontology to embed
Returns:
Number of elements embedded
"""
if not self.embedding_service:
logger.warning("No embedding service available, skipping embedding")
return 0
embedded_count = 0
batch_size = 50 # Process embeddings in batches
# Collect all elements to embed
elements_to_embed = []
# Process classes
for class_id, class_def in ontology.classes.items():
text = self._create_text_representation(class_id, class_def, 'class')
elements_to_embed.append({
'id': f"{ontology.id}:class:{class_id}",
'text': text,
'metadata': OntologyElementMetadata(
type='class',
ontology=ontology.id,
element=class_id,
definition=class_def.__dict__,
text=text
).__dict__
})
# Process object properties
for prop_id, prop_def in ontology.object_properties.items():
text = self._create_text_representation(prop_id, prop_def, 'objectProperty')
elements_to_embed.append({
'id': f"{ontology.id}:objectProperty:{prop_id}",
'text': text,
'metadata': OntologyElementMetadata(
type='objectProperty',
ontology=ontology.id,
element=prop_id,
definition=prop_def.__dict__,
text=text
).__dict__
})
# Process datatype properties
for prop_id, prop_def in ontology.datatype_properties.items():
text = self._create_text_representation(prop_id, prop_def, 'datatypeProperty')
elements_to_embed.append({
'id': f"{ontology.id}:datatypeProperty:{prop_id}",
'text': text,
'metadata': OntologyElementMetadata(
type='datatypeProperty',
ontology=ontology.id,
element=prop_id,
definition=prop_def.__dict__,
text=text
).__dict__
})
# Process in batches
for i in range(0, len(elements_to_embed), batch_size):
batch = elements_to_embed[i:i + batch_size]
# Get embeddings for batch
texts = [elem['text'] for elem in batch]
try:
# Call embedding service for each text
# Note: embed() returns 2D array [[vector]], so extract first element
embedding_tasks = [self.embedding_service.embed(text) for text in texts]
embeddings_responses = await asyncio.gather(*embedding_tasks)
# Extract vectors from responses (each is [[vector]])
embeddings_list = [resp[0] for resp in embeddings_responses]
# Convert to numpy array
embeddings = np.array(embeddings_list)
# Log embedding shape for debugging
logger.debug(f"Embeddings shape: {embeddings.shape}, expected: ({len(batch)}, {self.vector_store.dimension})")
# Store in vector store
ids = [elem['id'] for elem in batch]
metadata_list = [elem['metadata'] for elem in batch]
self.vector_store.add_batch(ids, embeddings, metadata_list)
embedded_count += len(batch)
logger.debug(f"Embedded batch of {len(batch)} elements from ontology {ontology.id}")
except Exception as e:
logger.error(f"Failed to embed batch for ontology {ontology.id}: {e}", exc_info=True)
self.embedded_ontologies.add(ontology.id)
logger.info(f"Embedded {embedded_count} elements from ontology {ontology.id}")
return embedded_count
async def embed_ontologies(self, ontologies: Dict[str, Ontology]) -> int:
"""Generate and store embeddings for multiple ontologies.
Args:
ontologies: Dictionary of ontology ID to Ontology objects
Returns:
Total number of elements embedded
"""
total_embedded = 0
for ont_id, ontology in ontologies.items():
if ont_id not in self.embedded_ontologies:
count = await self.embed_ontology(ontology)
total_embedded += count
else:
logger.debug(f"Ontology {ont_id} already embedded, skipping")
logger.info(f"Total embedded elements: {total_embedded} from {len(ontologies)} ontologies")
return total_embedded
async def embed_text(self, text: str) -> Optional[np.ndarray]:
"""Generate embedding for a single text.
Args:
text: Text to embed
Returns:
Embedding vector or None if failed
"""
if not self.embedding_service:
logger.warning("No embedding service available")
return None
try:
# embed() returns 2D array [[vector]], extract first element
embedding_response = await self.embedding_service.embed(text)
return np.array(embedding_response[0])
except Exception as e:
logger.error(f"Failed to embed text: {e}")
return None
async def embed_texts(self, texts: List[str]) -> Optional[np.ndarray]:
"""Generate embeddings for multiple texts.
Args:
texts: List of texts to embed
Returns:
Array of embeddings or None if failed
"""
if not self.embedding_service:
logger.warning("No embedding service available")
return None
try:
# Call embed() for each text (returns [[vector]] per call)
embedding_tasks = [self.embedding_service.embed(text) for text in texts]
embeddings_responses = await asyncio.gather(*embedding_tasks)
# Extract first vector from each response
embeddings_list = [resp[0] for resp in embeddings_responses]
return np.array(embeddings_list)
except Exception as e:
logger.error(f"Failed to embed texts: {e}")
return None
def remove_ontology(self, ontology_id: str):
"""Remove all embeddings for a specific ontology.
Note: FAISS doesn't support efficient deletion, so this currently
requires rebuilding the entire index without the removed ontology.
Args:
ontology_id: ID of ontology to remove
"""
if ontology_id not in self.embedded_ontologies:
logger.debug(f"Ontology '{ontology_id}' not embedded, nothing to remove")
return
# FAISS doesn't support selective deletion, so we'd need to rebuild the index
# For now, just remove from tracking set
# TODO: Implement index rebuilding if selective removal is needed
self.embedded_ontologies.discard(ontology_id)
logger.info(f"Removed ontology '{ontology_id}' from embedded set (note: vectors still in store)")
def clear_embeddings(self, ontology_id: Optional[str] = None):
"""Clear embeddings from vector store.
Args:
ontology_id: If provided, only clear embeddings for this ontology
Otherwise, clear all embeddings
"""
if ontology_id:
self.remove_ontology(ontology_id)
else:
self.vector_store.clear()
self.embedded_ontologies.clear()
logger.info("Cleared all embeddings from vector store")
def get_vector_store(self) -> InMemoryVectorStore:
"""Get the vector store instance.
Returns:
The vector store being used
"""
return self.vector_store
def get_embedded_count(self) -> int:
"""Get the number of embedded elements.
Returns:
Number of elements in the vector store
"""
return self.vector_store.size()
def is_ontology_embedded(self, ontology_id: str) -> bool:
"""Check if an ontology has been embedded.
Args:
ontology_id: ID of the ontology
Returns:
True if the ontology has been embedded
"""
return ontology_id in self.embedded_ontologies

247
trustgraph-flow/trustgraph/extract/kg/ontology/ontology_loader.py Normal file
View file

@ -0,0 +1,247 @@
"""
Ontology loader component for OntoRAG system.
Loads and manages ontologies from configuration service.
"""
import json
import logging
from typing import Dict, Any, Optional, List
from dataclasses import dataclass, field
logger = logging.getLogger(__name__)
@dataclass
class OntologyClass:
"""Represents an OWL-like class in the ontology."""
uri: str
type: str = "owl:Class"
labels: List[Dict[str, str]] = field(default_factory=list)
comment: Optional[str] = None
subclass_of: Optional[str] = None
equivalent_classes: List[str] = field(default_factory=list)
disjoint_with: List[str] = field(default_factory=list)
identifier: Optional[str] = None
@staticmethod
def from_dict(class_id: str, data: Dict[str, Any]) -> 'OntologyClass':
"""Create OntologyClass from dictionary representation."""
labels = data.get('rdfs:label', [])
if isinstance(labels, list):
labels = labels
else:
labels = [labels] if labels else []
return OntologyClass(
uri=data.get('uri', ''),
type=data.get('type', 'owl:Class'),
labels=labels,
comment=data.get('rdfs:comment'),
subclass_of=data.get('rdfs:subClassOf'),
equivalent_classes=data.get('owl:equivalentClass', []),
disjoint_with=data.get('owl:disjointWith', []),
identifier=data.get('dcterms:identifier')
)
@dataclass
class OntologyProperty:
"""Represents a property (object or datatype) in the ontology."""
uri: str
type: str
labels: List[Dict[str, str]] = field(default_factory=list)
comment: Optional[str] = None
domain: Optional[str] = None
range: Optional[str] = None
inverse_of: Optional[str] = None
functional: bool = False
inverse_functional: bool = False
min_cardinality: Optional[int] = None
max_cardinality: Optional[int] = None
cardinality: Optional[int] = None
@staticmethod
def from_dict(prop_id: str, data: Dict[str, Any]) -> 'OntologyProperty':
"""Create OntologyProperty from dictionary representation."""
labels = data.get('rdfs:label', [])
if isinstance(labels, list):
labels = labels
else:
labels = [labels] if labels else []
return OntologyProperty(
uri=data.get('uri', ''),
type=data.get('type', ''),
labels=labels,
comment=data.get('rdfs:comment'),
domain=data.get('rdfs:domain'),
range=data.get('rdfs:range'),
inverse_of=data.get('owl:inverseOf'),
functional=data.get('owl:functionalProperty', False),
inverse_functional=data.get('owl:inverseFunctionalProperty', False),
min_cardinality=data.get('owl:minCardinality'),
max_cardinality=data.get('owl:maxCardinality'),
cardinality=data.get('owl:cardinality')
)
@dataclass
class Ontology:
"""Represents a complete ontology with metadata, classes, and properties."""
id: str
metadata: Dict[str, Any]
classes: Dict[str, OntologyClass]
object_properties: Dict[str, OntologyProperty]
datatype_properties: Dict[str, OntologyProperty]
def get_class(self, class_id: str) -> Optional[OntologyClass]:
"""Get a class by ID."""
return self.classes.get(class_id)
def get_property(self, prop_id: str) -> Optional[OntologyProperty]:
"""Get a property (object or datatype) by ID."""
prop = self.object_properties.get(prop_id)
if prop is None:
prop = self.datatype_properties.get(prop_id)
return prop
def get_parent_classes(self, class_id: str) -> List[str]:
"""Get all parent classes (following subClassOf hierarchy)."""
parents = []
current = class_id
visited = set()
while current and current not in visited:
visited.add(current)
cls = self.get_class(current)
if cls and cls.subclass_of:
parents.append(cls.subclass_of)
current = cls.subclass_of
else:
break
return parents
def validate_structure(self) -> List[str]:
"""Validate ontology structure and return list of issues."""
issues = []
# Check for circular inheritance
for class_id in self.classes:
visited = set()
current = class_id
while current:
if current in visited:
issues.append(f"Circular inheritance detected for class {class_id}")
break
visited.add(current)
cls = self.get_class(current)
if cls:
current = cls.subclass_of
else:
break
# Check property domains and ranges exist
for prop_id, prop in {**self.object_properties, **self.datatype_properties}.items():
if prop.domain and prop.domain not in self.classes:
issues.append(f"Property {prop_id} has unknown domain {prop.domain}")
if prop.type == "owl:ObjectProperty" and prop.range and prop.range not in self.classes:
issues.append(f"Object property {prop_id} has unknown range class {prop.range}")
# Check disjoint classes
for class_id, cls in self.classes.items():
for disjoint_id in cls.disjoint_with:
if disjoint_id not in self.classes:
issues.append(f"Class {class_id} disjoint with unknown class {disjoint_id}")
return issues
class OntologyLoader:
"""Manages ontologies received via event-driven config updates.
No direct database access - receives ontologies via config handler.
"""
def __init__(self):
"""Initialize empty ontology store."""
self.ontologies: Dict[str, Ontology] = {}
def update_ontologies(self, ontology_configs: Dict[str, Any]):
"""Update ontology definitions from config.
Args:
ontology_configs: Dict mapping ontology_id -> ontology_definition (parsed dicts)
"""
self.ontologies.clear()
for ont_id, ont_data in ontology_configs.items():
try:
# Parse classes
classes = {}
for class_id, class_data in ont_data.get('classes', {}).items():
classes[class_id] = OntologyClass.from_dict(class_id, class_data)
# Parse object properties
object_props = {}
for prop_id, prop_data in ont_data.get('objectProperties', {}).items():
object_props[prop_id] = OntologyProperty.from_dict(prop_id, prop_data)
# Parse datatype properties
datatype_props = {}
for prop_id, prop_data in ont_data.get('datatypeProperties', {}).items():
datatype_props[prop_id] = OntologyProperty.from_dict(prop_id, prop_data)
# Create ontology
ontology = Ontology(
id=ont_id,
metadata=ont_data.get('metadata', {}),
classes=classes,
object_properties=object_props,
datatype_properties=datatype_props
)
# Validate structure
issues = ontology.validate_structure()
if issues:
logger.warning(f"Ontology {ont_id} has validation issues: {issues}")
self.ontologies[ont_id] = ontology
logger.info(f"Loaded ontology {ont_id} with {len(classes)} classes, "
f"{len(object_props)} object properties, "
f"{len(datatype_props)} datatype properties")
except Exception as e:
logger.error(f"Failed to load ontology {ont_id}: {e}", exc_info=True)
def get_ontology(self, ont_id: str) -> Optional[Ontology]:
"""Get a specific ontology by ID.
Args:
ont_id: Ontology identifier
Returns:
Ontology object or None if not found
"""
return self.ontologies.get(ont_id)
def get_all_ontologies(self) -> Dict[str, Ontology]:
"""Get all loaded ontologies.
Returns:
Dictionary of ontology ID to Ontology objects
"""
return self.ontologies
def list_ontology_ids(self) -> List[str]:
"""Get list of loaded ontology IDs.
Returns:
List of ontology IDs
"""
return list(self.ontologies.keys())
def clear(self):
"""Clear all loaded ontologies."""
self.ontologies.clear()
logger.info("Cleared all loaded ontologies")

356
trustgraph-flow/trustgraph/extract/kg/ontology/ontology_selector.py Normal file
View file

@ -0,0 +1,356 @@
"""
Ontology selection algorithm for OntoRAG system.
Selects relevant ontology subsets based on text similarity.
"""
import logging
from typing import List, Dict, Any, Set, Optional, Tuple
from dataclasses import dataclass
from collections import defaultdict
from .ontology_loader import Ontology, OntologyLoader
from .ontology_embedder import OntologyEmbedder
from .text_processor import TextSegment
from .vector_store import SearchResult
logger = logging.getLogger(__name__)
@dataclass
class OntologySubset:
"""Represents a subset of an ontology relevant to a text chunk."""
ontology_id: str
classes: Dict[str, Any]
object_properties: Dict[str, Any]
datatype_properties: Dict[str, Any]
metadata: Dict[str, Any]
relevance_score: float = 0.0
class OntologySelector:
"""Selects relevant ontology elements for text segments using vector similarity."""
def __init__(self, ontology_embedder: OntologyEmbedder,
ontology_loader: OntologyLoader,
top_k: int = 10,
similarity_threshold: float = 0.7):
"""Initialize the ontology selector.
Args:
ontology_embedder: Embedder with vector store
ontology_loader: Loader with ontology definitions
top_k: Number of top results to retrieve per segment
similarity_threshold: Minimum similarity score
"""
self.embedder = ontology_embedder
self.loader = ontology_loader
self.top_k = top_k
self.similarity_threshold = similarity_threshold
async def select_ontology_subset(self, segments: List[TextSegment]) -> List[OntologySubset]:
"""Select relevant ontology subsets for text segments.
Args:
segments: List of text segments to match
Returns:
List of ontology subsets with relevant elements
"""
# Collect all relevant elements
relevant_elements = await self._find_relevant_elements(segments)
# Group by ontology and build subsets
ontology_subsets = self._build_ontology_subsets(relevant_elements)
# Resolve dependencies
for subset in ontology_subsets:
self._resolve_dependencies(subset)
logger.info(f"Selected {len(ontology_subsets)} ontology subsets")
return ontology_subsets
async def _find_relevant_elements(self, segments: List[TextSegment]) -> Set[Tuple[str, str, str, Dict]]:
"""Find relevant ontology elements for text segments.
Args:
segments: Text segments to match
Returns:
Set of (ontology_id, element_type, element_id, definition) tuples
"""
relevant_elements = set()
element_scores = defaultdict(float)
# Check if vector store has any elements
vector_store = self.embedder.get_vector_store()
store_size = vector_store.size()
logger.info(f"Vector store size: {store_size} elements")
if store_size == 0:
logger.warning("Vector store is empty - no ontology elements embedded")
return relevant_elements
# Process each segment (log first few for debugging)
for i, segment in enumerate(segments):
# Get embedding for segment
embedding = await self.embedder.embed_text(segment.text)
if embedding is None:
logger.warning(f"Failed to embed segment: {segment.text[:50]}...")
continue
# Search vector store with no threshold to see all scores
all_results = vector_store.search(
embedding=embedding,
top_k=self.top_k,
threshold=0.0 # Get all results to see scores
)
# Log top scores for first 3 segments to debug
if i < 3 and all_results:
top_scores = [r.score for r in all_results[:3]]
top_elements = [r.metadata['element'] for r in all_results[:3]]
logger.info(f"Segment {i}: '{segment.text[:60]}...'")
logger.info(f" Top 3 scores: {top_scores} (threshold={self.similarity_threshold})")
logger.info(f" Top 3 elements: {top_elements}")
# Filter by threshold
results = [r for r in all_results if r.score >= self.similarity_threshold]
# Process results
for result in results:
metadata = result.metadata
element_key = (
metadata['ontology'],
metadata['type'],
metadata['element'],
str(metadata['definition']) # Convert dict to string for hashability
)
relevant_elements.add(element_key)
# Track scores for ranking
element_scores[element_key] = max(element_scores[element_key], result.score)
logger.info(f"Found {len(relevant_elements)} relevant elements from {len(segments)} segments")
return relevant_elements
def _build_ontology_subsets(self, relevant_elements: Set[Tuple[str, str, str, Dict]]) -> List[OntologySubset]:
"""Build ontology subsets from relevant elements.
Args:
relevant_elements: Set of relevant element tuples
Returns:
List of ontology subsets
"""
# Group elements by ontology
ontology_groups = defaultdict(lambda: {
'classes': {},
'object_properties': {},
'datatype_properties': {},
'scores': []
})
for ont_id, elem_type, elem_id, definition in relevant_elements:
# Parse definition back from string if needed
if isinstance(definition, str):
import json
try:
definition = json.loads(definition.replace("'", '"'))
except:
definition = eval(definition) # Fallback for dict-like strings
# Get the actual ontology and element
ontology = self.loader.get_ontology(ont_id)
if not ontology:
logger.warning(f"Ontology {ont_id} not found in loader")
continue
# Add element to appropriate category
if elem_type == 'class':
cls = ontology.get_class(elem_id)
if cls:
ontology_groups[ont_id]['classes'][elem_id] = cls.__dict__
elif elem_type == 'objectProperty':
prop = ontology.object_properties.get(elem_id)
if prop:
ontology_groups[ont_id]['object_properties'][elem_id] = prop.__dict__
elif elem_type == 'datatypeProperty':
prop = ontology.datatype_properties.get(elem_id)
if prop:
ontology_groups[ont_id]['datatype_properties'][elem_id] = prop.__dict__
# Create OntologySubset objects
subsets = []
for ont_id, elements in ontology_groups.items():
ontology = self.loader.get_ontology(ont_id)
if ontology:
subset = OntologySubset(
ontology_id=ont_id,
classes=elements['classes'],
object_properties=elements['object_properties'],
datatype_properties=elements['datatype_properties'],
metadata=ontology.metadata,
relevance_score=sum(elements['scores']) / len(elements['scores']) if elements['scores'] else 0.0
)
subsets.append(subset)
return subsets
def _resolve_dependencies(self, subset: OntologySubset):
"""Resolve dependencies for ontology subset elements.
Args:
subset: Ontology subset to resolve dependencies for
"""
ontology = self.loader.get_ontology(subset.ontology_id)
if not ontology:
return
# Track classes to add
classes_to_add = set()
# Resolve class hierarchies
for class_id in list(subset.classes.keys()):
# Add parent classes
parents = ontology.get_parent_classes(class_id)
for parent_id in parents:
parent_class = ontology.get_class(parent_id)
if parent_class and parent_id not in subset.classes:
classes_to_add.add(parent_id)
# Resolve property domains and ranges
for prop_id, prop_def in subset.object_properties.items():
# Add domain class
if 'domain' in prop_def and prop_def['domain']:
domain_id = prop_def['domain']
if domain_id not in subset.classes:
domain_class = ontology.get_class(domain_id)
if domain_class:
classes_to_add.add(domain_id)
# Add range class
if 'range' in prop_def and prop_def['range']:
range_id = prop_def['range']
if range_id not in subset.classes:
range_class = ontology.get_class(range_id)
if range_class:
classes_to_add.add(range_id)
# Resolve datatype property domains
for prop_id, prop_def in subset.datatype_properties.items():
if 'domain' in prop_def and prop_def['domain']:
domain_id = prop_def['domain']
if domain_id not in subset.classes:
domain_class = ontology.get_class(domain_id)
if domain_class:
classes_to_add.add(domain_id)
# Add inverse properties
for prop_id, prop_def in list(subset.object_properties.items()):
if 'inverse_of' in prop_def and prop_def['inverse_of']:
inverse_id = prop_def['inverse_of']
if inverse_id not in subset.object_properties:
inverse_prop = ontology.object_properties.get(inverse_id)
if inverse_prop:
subset.object_properties[inverse_id] = inverse_prop.__dict__
# NEW: Auto-include properties related to selected classes
# For each selected class, find all properties that reference it in domain or range
properties_added = 0
datatype_properties_added = 0
for class_id in list(subset.classes.keys()):
# Check all object properties in the ontology
for prop_id, prop_def in ontology.object_properties.items():
if prop_id not in subset.object_properties:
# Check if this class is in the property's domain or range
prop_domain = getattr(prop_def, 'domain', None)
prop_range = getattr(prop_def, 'range', None)
if prop_domain == class_id or prop_range == class_id:
subset.object_properties[prop_id] = prop_def.__dict__
properties_added += 1
# Also add the other class (domain or range) if not already present
if prop_domain and prop_domain != class_id and prop_domain not in subset.classes:
other_class = ontology.get_class(prop_domain)
if other_class:
classes_to_add.add(prop_domain)
if prop_range and prop_range != class_id and prop_range not in subset.classes:
other_class = ontology.get_class(prop_range)
if other_class:
classes_to_add.add(prop_range)
# Check all datatype properties in the ontology
for prop_id, prop_def in ontology.datatype_properties.items():
if prop_id not in subset.datatype_properties:
# Check if this class is in the property's domain
prop_domain = getattr(prop_def, 'domain', None)
if prop_domain == class_id:
subset.datatype_properties[prop_id] = prop_def.__dict__
datatype_properties_added += 1
# Add collected classes
for class_id in classes_to_add:
cls = ontology.get_class(class_id)
if cls:
subset.classes[class_id] = cls.__dict__
logger.debug(f"Resolved dependencies for subset {subset.ontology_id}: "
f"added {len(classes_to_add)} classes, "
f"{properties_added} object properties, "
f"{datatype_properties_added} datatype properties")
def merge_subsets(self, subsets: List[OntologySubset]) -> OntologySubset:
"""Merge multiple ontology subsets into one.
Args:
subsets: List of subsets to merge
Returns:
Merged ontology subset
"""
if not subsets:
return None
if len(subsets) == 1:
return subsets[0]
# Use first subset as base
merged = OntologySubset(
ontology_id="merged",
classes={},
object_properties={},
datatype_properties={},
metadata={},
relevance_score=0.0
)
# Merge all subsets
total_score = 0.0
for subset in subsets:
# Merge classes
for class_id, class_def in subset.classes.items():
key = f"{subset.ontology_id}:{class_id}"
merged.classes[key] = class_def
# Merge object properties
for prop_id, prop_def in subset.object_properties.items():
key = f"{subset.ontology_id}:{prop_id}"
merged.object_properties[key] = prop_def
# Merge datatype properties
for prop_id, prop_def in subset.datatype_properties.items():
key = f"{subset.ontology_id}:{prop_id}"
merged.datatype_properties[key] = prop_def
total_score += subset.relevance_score
# Average relevance score
merged.relevance_score = total_score / len(subsets)
logger.info(f"Merged {len(subsets)} subsets into one with "
f"{len(merged.classes)} classes, "
f"{len(merged.object_properties)} object properties, "
f"{len(merged.datatype_properties)} datatype properties")
return merged

10
trustgraph-flow/trustgraph/extract/kg/ontology/run.py Normal file
View file

@ -0,0 +1,10 @@
#!/usr/bin/env python3
"""
OntoRAG extraction service launcher.
"""
from . extract import run
if __name__ == "__main__":
run()

240
trustgraph-flow/trustgraph/extract/kg/ontology/text_processor.py Normal file
View file

@ -0,0 +1,240 @@
"""
Text processing components for OntoRAG system.
Splits text into sentences and extracts phrases for granular matching.
"""
import logging
import re
from typing import List, Dict, Any, Optional
from dataclasses import dataclass
import nltk
from nltk.corpus import stopwords
logger = logging.getLogger(__name__)
# Ensure required NLTK data is downloaded
try:
nltk.data.find('tokenizers/punkt_tab')
except LookupError:
try:
nltk.download('punkt_tab', quiet=True)
except:
# Fallback to older punkt if punkt_tab not available
try:
nltk.download('punkt', quiet=True)
except:
pass
try:
nltk.data.find('taggers/averaged_perceptron_tagger_eng')
except LookupError:
try:
nltk.download('averaged_perceptron_tagger_eng', quiet=True)
except:
# Fallback to older name
try:
nltk.download('averaged_perceptron_tagger', quiet=True)
except:
pass
try:
nltk.data.find('corpora/stopwords')
except LookupError:
nltk.download('stopwords', quiet=True)
@dataclass
class TextSegment:
"""Represents a segment of text (sentence or phrase)."""
text: str
type: str # 'sentence', 'phrase', 'noun_phrase', 'verb_phrase'
position: int
parent_sentence: Optional[str] = None
metadata: Dict[str, Any] = None
class SentenceSplitter:
"""Splits text into sentences using NLTK."""
def __init__(self):
"""Initialize sentence splitter."""
try:
# Try newer punkt_tab first
self.sent_detector = nltk.data.load('tokenizers/punkt_tab/english/')
logger.info("Using NLTK sentence tokenizer (punkt_tab)")
except:
# Fallback to older punkt
self.sent_detector = nltk.data.load('tokenizers/punkt/english.pickle')
logger.info("Using NLTK sentence tokenizer (punkt)")
def split(self, text: str) -> List[str]:
"""Split text into sentences.
Args:
text: Text to split
Returns:
List of sentences
"""
sentences = self.sent_detector.tokenize(text)
return sentences
class PhraseExtractor:
"""Extracts meaningful phrases from sentences using NLTK."""
def __init__(self):
"""Initialize phrase extractor."""
logger.info("Using NLTK phrase extraction")
def extract(self, sentence: str) -> List[Dict[str, str]]:
"""Extract phrases from a sentence.
Args:
sentence: Sentence to extract phrases from
Returns:
List of phrases with their types
"""
phrases = []
# Tokenize and POS tag
tokens = nltk.word_tokenize(sentence)
pos_tags = nltk.pos_tag(tokens)
# Extract noun phrases (simple pattern)
noun_phrase = []
for word, pos in pos_tags:
if pos.startswith('NN') or pos.startswith('JJ'):
noun_phrase.append(word)
elif noun_phrase:
if len(noun_phrase) > 1:
phrases.append({
'text': ' '.join(noun_phrase),
'type': 'noun_phrase'
})
noun_phrase = []
# Add last noun phrase if exists
if noun_phrase and len(noun_phrase) > 1:
phrases.append({
'text': ' '.join(noun_phrase),
'type': 'noun_phrase'
})
# Extract verb phrases (simple pattern)
verb_phrase = []
for word, pos in pos_tags:
if pos.startswith('VB') or pos.startswith('RB'):
verb_phrase.append(word)
elif verb_phrase:
if len(verb_phrase) > 1:
phrases.append({
'text': ' '.join(verb_phrase),
'type': 'verb_phrase'
})
verb_phrase = []
# Add last verb phrase if exists
if verb_phrase and len(verb_phrase) > 1:
phrases.append({
'text': ' '.join(verb_phrase),
'type': 'verb_phrase'
})
return phrases
class TextProcessor:
"""Main text processing class that coordinates sentence splitting and phrase extraction."""
def __init__(self):
"""Initialize text processor."""
self.sentence_splitter = SentenceSplitter()
self.phrase_extractor = PhraseExtractor()
def process_chunk(self, chunk_text: str, extract_phrases: bool = True) -> List[TextSegment]:
"""Process a text chunk into segments.
Args:
chunk_text: Text chunk to process
extract_phrases: Whether to extract phrases from sentences
Returns:
List of TextSegment objects
"""
segments = []
position = 0
# Split into sentences
sentences = self.sentence_splitter.split(chunk_text)
for sentence in sentences:
# Add sentence segment
segments.append(TextSegment(
text=sentence,
type='sentence',
position=position
))
position += 1
# Extract phrases if requested
if extract_phrases:
phrases = self.phrase_extractor.extract(sentence)
for phrase_data in phrases:
segments.append(TextSegment(
text=phrase_data['text'],
type=phrase_data['type'],
position=position,
parent_sentence=sentence
))
position += 1
logger.debug(f"Processed chunk into {len(segments)} segments")
return segments
def extract_key_terms(self, text: str) -> List[str]:
"""Extract key terms from text for matching.
Args:
text: Text to extract terms from
Returns:
List of key terms
"""
terms = []
# Split on word boundaries
words = re.findall(r'\b\w+\b', text.lower())
# Use NLTK stopwords
stop_words = set(stopwords.words('english'))
# Filter stopwords and short words
terms = [w for w in words if w not in stop_words and len(w) > 2]
# Also extract multi-word terms (bigrams)
for i in range(len(words) - 1):
if words[i] not in stop_words and words[i+1] not in stop_words:
bigram = f"{words[i]} {words[i+1]}"
terms.append(bigram)
return terms
def normalize_text(self, text: str) -> str:
"""Normalize text for consistent processing.
Args:
text: Text to normalize
Returns:
Normalized text
"""
# Remove extra whitespace
text = re.sub(r'\s+', ' ', text)
# Remove leading/trailing whitespace
text = text.strip()
# Normalize quotes
text = text.replace('"', '"').replace('"', '"')
text = text.replace(''', "'").replace(''', "'")
return text

130
trustgraph-flow/trustgraph/extract/kg/ontology/vector_store.py Normal file
View file

@ -0,0 +1,130 @@
"""
Vector store implementation for OntoRAG system.
Provides FAISS-based vector storage for ontology embeddings.
"""
import logging
import numpy as np
from typing import List, Dict, Any
from dataclasses import dataclass
import faiss
logger = logging.getLogger(__name__)
@dataclass
class SearchResult:
"""Result from vector similarity search."""
id: str
score: float
metadata: Dict[str, Any]
class InMemoryVectorStore:
"""FAISS-based vector store implementation for ontology embeddings."""
def __init__(self, dimension: int = 1536, index_type: str = 'flat'):
"""Initialize FAISS vector store.
Args:
dimension: Embedding dimension (1536 for text-embedding-3-small)
index_type: 'flat' for exact search, 'ivf' for larger datasets
"""
self.dimension = dimension
self.metadata = []
self.ids = []
if index_type == 'flat':
# Exact search - best for ontologies with <10k elements
self.index = faiss.IndexFlatIP(dimension)
logger.info(f"Created FAISS flat index with dimension {dimension}")
else:
# Approximate search - for larger ontologies
quantizer = faiss.IndexFlatIP(dimension)
self.index = faiss.IndexIVFFlat(quantizer, dimension, 100)
# Train with random vectors for initialization
training_data = np.random.randn(1000, dimension).astype('float32')
training_data = training_data / np.linalg.norm(
training_data, axis=1, keepdims=True
)
self.index.train(training_data)
logger.info(f"Created FAISS IVF index with dimension {dimension}")
def add(self, id: str, embedding: np.ndarray, metadata: Dict[str, Any]):
"""Add single embedding with metadata."""
# Normalize for cosine similarity
embedding = embedding / np.linalg.norm(embedding)
self.index.add(np.array([embedding], dtype=np.float32))
self.metadata.append(metadata)
self.ids.append(id)
def add_batch(self, ids: List[str], embeddings: np.ndarray,
metadata_list: List[Dict[str, Any]]):
"""Batch add for initial ontology loading."""
# Normalize all embeddings
norms = np.linalg.norm(embeddings, axis=1, keepdims=True)
normalized = embeddings / norms
self.index.add(normalized.astype(np.float32))
self.metadata.extend(metadata_list)
self.ids.extend(ids)
logger.debug(f"Added batch of {len(ids)} embeddings to FAISS index")
def search(self, embedding: np.ndarray, top_k: int = 10,
threshold: float = 0.0) -> List[SearchResult]:
"""Search for similar vectors."""
# Normalize query
embedding = embedding / np.linalg.norm(embedding)
# Search
scores, indices = self.index.search(
np.array([embedding], dtype=np.float32),
min(top_k, self.index.ntotal)
)
# Filter by threshold and format results
results = []
for score, idx in zip(scores[0], indices[0]):
if idx >= 0 and score >= threshold: # FAISS returns -1 for empty slots
results.append(SearchResult(
id=self.ids[idx],
score=float(score),
metadata=self.metadata[idx]
))
return results
def clear(self):
"""Reset the store."""
self.index.reset()
self.metadata = []
self.ids = []
logger.info("Cleared FAISS vector store")
def size(self) -> int:
"""Return number of stored vectors."""
return self.index.ntotal
# Utility functions for vector operations
def cosine_similarity(a: np.ndarray, b: np.ndarray) -> float:
"""Compute cosine similarity between two vectors."""
return np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b))
def batch_cosine_similarity(queries: np.ndarray, targets: np.ndarray) -> np.ndarray:
"""Compute cosine similarity between query vectors and target vectors.
Args:
queries: Array of shape (n_queries, dimension)
targets: Array of shape (n_targets, dimension)
Returns:
Array of shape (n_queries, n_targets) with similarity scores
"""
# Normalize queries and targets
queries_norm = queries / np.linalg.norm(queries, axis=1, keepdims=True)
targets_norm = targets / np.linalg.norm(targets, axis=1, keepdims=True)
# Compute dot product
similarities = np.dot(queries_norm, targets_norm.T)
return similarities