From a117fc032a99b2d921cdae940dcfba8d8766d06e Mon Sep 17 00:00:00 2001 From: Cyber MacGeddon Date: Wed, 3 Dec 2025 11:05:19 +0000 Subject: [PATCH] Removed old code --- docs/tech-specs/ontology-extract-phase-2.md | 52 ++++++++++++------- .../trustgraph/extract/kg/ontology/extract.py | 38 ++------------ 2 files changed, 37 insertions(+), 53 deletions(-) diff --git a/docs/tech-specs/ontology-extract-phase-2.md b/docs/tech-specs/ontology-extract-phase-2.md index 687c177d..ac1a0543 100644 --- a/docs/tech-specs/ontology-extract-phase-2.md +++ b/docs/tech-specs/ontology-extract-phase-2.md @@ -254,36 +254,50 @@ But these are never surfaced to the LLM. The LLM doesn't know: - Longer prompt - More complex template -## Recommended Approach +## Implemented Approach -**Option B** (Full Prefixed IDs Consistently) is recommended because: +**Simplified Entity-Relationship-Attribute Format** - completely replaces the old triple-based format. + +The new approach was chosen because: 1. **No Information Loss**: Original URIs preserved correctly 2. **Simpler Logic**: No transformation needed, direct dict lookups work 3. **Namespace Safety**: Handles multiple ontologies without collisions 4. **Semantic Correctness**: Maintains RDF/OWL semantics -## Implementation Plan +## Implementation Complete -### Phase 1: Fix Prompt Examples -- [ ] Update `ontology-prompt.md` example to use prefixed IDs -- [ ] Add namespace prefix explanation section -- [ ] Add guidance for entity instance naming +### What Was Built: -### Phase 2: Enhance Prompt Template -- [ ] Optionally add labels alongside IDs (Option C enhancement) -- [ ] Document namespace prefixes from ontology metadata -- [ ] Clarify entity instance URI format +1. **New Prompt Template** (`prompts/ontology-extract-v2.txt`) + - ✅ Clear sections: Entity Types, Relationships, Attributes + - ✅ Example using full type identifiers (`fo/Recipe`, `fo/has_ingredient`) + - ✅ Instructions to use exact identifiers from schema + - ✅ New JSON format with entities/relationships/attributes arrays -### Phase 3: Add Validation Tests -- [ ] Test that LLM outputs match input format -- [ ] Verify URI expansion preserves original URIs -- [ ] Test with multiple ontologies (collision scenarios) +2. **Entity Normalization** (`entity_normalizer.py`) + - ✅ `normalize_entity_name()` - Converts names to URI-safe format + - ✅ `normalize_type_identifier()` - Handles slashes in types (`fo/Recipe` → `fo-recipe`) + - ✅ `build_entity_uri()` - Creates unique URIs using (name, type) tuple + - ✅ `EntityRegistry` - Tracks entities for deduplication -### Phase 4: Improve Error Handling -- [ ] Better logging when URI lookup fails -- [ ] Warn when falling back to constructed URIs -- [ ] Validate LLM output format before parsing +3. **JSON Parser** (`simplified_parser.py`) + - ✅ Parses new format: `{entities: [...], relationships: [...], attributes: [...]}` + - ✅ Supports kebab-case and snake_case field names + - ✅ Returns structured dataclasses + - ✅ Graceful error handling with logging + +4. **Triple Converter** (`triple_converter.py`) + - ✅ `convert_entity()` - Generates type + label triples automatically + - ✅ `convert_relationship()` - Connects entity URIs via properties + - ✅ `convert_attribute()` - Adds literal values + - ✅ Looks up full URIs from ontology definitions + +5. **Updated Main Processor** (`extract.py`) + - ✅ Removed old triple-based extraction code + - ✅ Added `extract_with_simplified_format()` method + - ✅ Now exclusively uses new simplified format + - ✅ Calls prompt with `extract-with-ontologies-v2` ID ## Test Cases diff --git a/trustgraph-flow/trustgraph/extract/kg/ontology/extract.py b/trustgraph-flow/trustgraph/extract/kg/ontology/extract.py index 61097881..54f0903a 100644 --- a/trustgraph-flow/trustgraph/extract/kg/ontology/extract.py +++ b/trustgraph-flow/trustgraph/extract/kg/ontology/extract.py @@ -104,7 +104,6 @@ class Processor(FlowProcessor): # Configuration self.top_k = params.get("top_k", 10) self.similarity_threshold = params.get("similarity_threshold", 0.3) - self.use_simplified_format = params.get("use_simplified_format", False) # Track loaded ontology version self.current_ontology_version = None @@ -301,33 +300,10 @@ class Processor(FlowProcessor): # Build extraction prompt variables prompt_variables = self.build_extraction_variables(chunk, ontology_subset) - # Choose extraction format based on configuration - if self.use_simplified_format: - # NEW: Simplified format extraction - triples = await self.extract_with_simplified_format( - flow, chunk, ontology_subset, prompt_variables - ) - else: - # OLD: Original triple-based format extraction - # Call prompt service for extraction - try: - # Use prompt() method with extract-with-ontologies prompt ID - triples_response = await flow("prompt-request").prompt( - id="extract-with-ontologies", - variables=prompt_variables - ) - logger.debug(f"Extraction response: {triples_response}") - - if not isinstance(triples_response, list): - logger.error("Expected list of triples from prompt service") - triples_response = [] - - except Exception as e: - logger.error(f"Prompt service error: {e}", exc_info=True) - triples_response = [] - - # Parse and validate triples - triples = self.parse_and_validate_triples(triples_response, ontology_subset) + # Extract using simplified entity-relationship-attribute format + triples = await self.extract_with_simplified_format( + flow, chunk, ontology_subset, prompt_variables + ) # Add metadata triples for t in v.metadata.metadata: @@ -900,12 +876,6 @@ class Processor(FlowProcessor): default=0.3, help='Similarity threshold for ontology matching (default: 0.3, range: 0.0-1.0)' ) - parser.add_argument( - '--use-simplified-format', - action='store_true', - default=False, - help='Use simplified entity-relationship-attribute extraction format (default: False)' - ) FlowProcessor.add_args(parser)