diff --git a/tests/unit/test_cli/test_error_handling_edge_cases.py b/tests/unit/test_cli/test_error_handling_edge_cases.py
new file mode 100644
index 00000000..77eb88e0
--- /dev/null
+++ b/tests/unit/test_cli/test_error_handling_edge_cases.py
@@ -0,0 +1,520 @@
+"""
+Error handling and edge case tests for tg-load-structured-data CLI command.
+Tests various failure scenarios, malformed data, and boundary conditions.
+"""
+
+import pytest
+import json
+import tempfile
+import os
+import csv
+from unittest.mock import Mock, patch, AsyncMock
+from io import StringIO
+
+from trustgraph.cli.load_structured_data import load_structured_data
+
+
+class TestErrorHandlingEdgeCases:
+ """Tests for error handling and edge cases"""
+
+ def setup_method(self):
+ """Set up test fixtures"""
+ self.api_url = "http://localhost:8088"
+
+ # Valid descriptor for testing
+ self.valid_descriptor = {
+ "version": "1.0",
+ "format": {
+ "type": "csv",
+ "encoding": "utf-8",
+ "options": {"header": True, "delimiter": ","}
+ },
+ "mappings": [
+ {"source_field": "name", "target_field": "name", "transforms": [{"type": "trim"}]},
+ {"source_field": "email", "target_field": "email", "transforms": [{"type": "lower"}]}
+ ],
+ "output": {
+ "format": "trustgraph-objects",
+ "schema_name": "test_schema",
+ "options": {"confidence": 0.9, "batch_size": 10}
+ }
+ }
+
+ def create_temp_file(self, content, suffix='.txt'):
+ """Create a temporary file with given content"""
+ temp_file = tempfile.NamedTemporaryFile(mode='w', suffix=suffix, delete=False)
+ temp_file.write(content)
+ temp_file.flush()
+ temp_file.close()
+ return temp_file.name
+
+ def cleanup_temp_file(self, file_path):
+ """Clean up temporary file"""
+ try:
+ os.unlink(file_path)
+ except:
+ pass
+
+ # File Access Error Tests
+ def test_nonexistent_input_file(self):
+ """Test handling of nonexistent input file"""
+ with pytest.raises(FileNotFoundError):
+ load_structured_data(
+ api_url=self.api_url,
+ input_file="/nonexistent/path/file.csv"
+ )
+
+ def test_nonexistent_descriptor_file(self):
+ """Test handling of nonexistent descriptor file"""
+ input_file = self.create_temp_file("name,email\nJohn,john@email.com", '.csv')
+
+ try:
+ with pytest.raises(FileNotFoundError):
+ load_structured_data(
+ api_url=self.api_url,
+ input_file=input_file,
+ descriptor_file="/nonexistent/descriptor.json"
+ )
+ finally:
+ self.cleanup_temp_file(input_file)
+
+ def test_permission_denied_file(self):
+ """Test handling of permission denied errors"""
+ # This test would need to create a file with restricted permissions
+ # Skip on systems where this can't be easily tested
+ pass
+
+ def test_empty_input_file(self):
+ """Test handling of completely empty input file"""
+ input_file = self.create_temp_file("", '.csv')
+ descriptor_file = self.create_temp_file(json.dumps(self.valid_descriptor), '.json')
+
+ try:
+ result = load_structured_data(
+ api_url=self.api_url,
+ input_file=input_file,
+ descriptor_file=descriptor_file,
+ dry_run=True
+ )
+ # Should handle gracefully, possibly with warning
+
+ finally:
+ self.cleanup_temp_file(input_file)
+ self.cleanup_temp_file(descriptor_file)
+
+ # Descriptor Format Error Tests
+ def test_invalid_json_descriptor(self):
+ """Test handling of invalid JSON in descriptor file"""
+ input_file = self.create_temp_file("name,email\nJohn,john@email.com", '.csv')
+ descriptor_file = self.create_temp_file('{"invalid": json}', '.json') # Invalid JSON
+
+ try:
+ with pytest.raises(json.JSONDecodeError):
+ load_structured_data(
+ api_url=self.api_url,
+ input_file=input_file,
+ descriptor_file=descriptor_file
+ )
+ finally:
+ self.cleanup_temp_file(input_file)
+ self.cleanup_temp_file(descriptor_file)
+
+ def test_missing_required_descriptor_fields(self):
+ """Test handling of descriptor missing required fields"""
+ incomplete_descriptor = {"version": "1.0"} # Missing format, mappings, output
+
+ input_file = self.create_temp_file("name,email\nJohn,john@email.com", '.csv')
+ descriptor_file = self.create_temp_file(json.dumps(incomplete_descriptor), '.json')
+
+ try:
+ with pytest.raises(KeyError):
+ load_structured_data(
+ api_url=self.api_url,
+ input_file=input_file,
+ descriptor_file=descriptor_file
+ )
+ finally:
+ self.cleanup_temp_file(input_file)
+ self.cleanup_temp_file(descriptor_file)
+
+ def test_invalid_format_type(self):
+ """Test handling of invalid format type in descriptor"""
+ invalid_descriptor = {
+ **self.valid_descriptor,
+ "format": {"type": "unsupported_format", "encoding": "utf-8"}
+ }
+
+ input_file = self.create_temp_file("name,email\nJohn,john@email.com", '.csv')
+ descriptor_file = self.create_temp_file(json.dumps(invalid_descriptor), '.json')
+
+ try:
+ with pytest.raises(ValueError):
+ load_structured_data(
+ api_url=self.api_url,
+ input_file=input_file,
+ descriptor_file=descriptor_file
+ )
+ finally:
+ self.cleanup_temp_file(input_file)
+ self.cleanup_temp_file(descriptor_file)
+
+ # Data Parsing Error Tests
+ def test_malformed_csv_data(self):
+ """Test handling of malformed CSV data"""
+ malformed_csv = '''name,email,age
+John Smith,john@email.com,35
+Jane "unclosed quote,jane@email.com,28
+Bob,bob@email.com,"age with quote,42'''
+
+ format_info = {"type": "csv", "encoding": "utf-8", "options": {"header": True, "delimiter": ","}}
+
+ # Should handle parsing errors gracefully
+ try:
+ records = parse_csv_data(malformed_csv, format_info)
+ # May return partial results or raise exception
+ except Exception as e:
+ # Exception is expected for malformed CSV
+ assert isinstance(e, (csv.Error, ValueError))
+
+ def test_csv_wrong_delimiter(self):
+ """Test CSV with wrong delimiter configuration"""
+ csv_data = "name;email;age\nJohn Smith;john@email.com;35\nJane Doe;jane@email.com;28"
+ format_info = {"type": "csv", "encoding": "utf-8", "options": {"header": True, "delimiter": ","}} # Wrong delimiter
+
+ records = parse_csv_data(csv_data, format_info)
+
+ # Should still parse but data will be in wrong format
+ assert len(records) == 2
+ # The entire row will be in the first field due to wrong delimiter
+ assert "John Smith;john@email.com;35" in records[0].values()
+
+ def test_malformed_json_data(self):
+ """Test handling of malformed JSON data"""
+ malformed_json = '{"name": "John", "age": 35, "email": }' # Missing value
+ format_info = {"type": "json", "encoding": "utf-8"}
+
+ with pytest.raises(json.JSONDecodeError):
+ parse_json_data(malformed_json, format_info)
+
+ def test_json_wrong_structure(self):
+ """Test JSON with unexpected structure"""
+ wrong_json = '{"not_an_array": "single_object"}'
+ format_info = {"type": "json", "encoding": "utf-8"}
+
+ with pytest.raises((ValueError, TypeError)):
+ parse_json_data(wrong_json, format_info)
+
+ def test_malformed_xml_data(self):
+ """Test handling of malformed XML data"""
+ malformed_xml = '''
+
+
+ John
+
+
+'''
+
+ format_info = {"type": "xml", "encoding": "utf-8", "options": {"record_path": "//record"}}
+
+ with pytest.raises(Exception): # XML parsing error
+ parse_xml_data(malformed_xml, format_info)
+
+ def test_xml_invalid_xpath(self):
+ """Test XML with invalid XPath expression"""
+ xml_data = '''
+
+ John
+'''
+
+ format_info = {
+ "type": "xml",
+ "encoding": "utf-8",
+ "options": {"record_path": "//[invalid xpath syntax"}
+ }
+
+ with pytest.raises(Exception):
+ parse_xml_data(xml_data, format_info)
+
+ # Transformation Error Tests
+ def test_invalid_transformation_type(self):
+ """Test handling of invalid transformation types"""
+ record = {"age": "35", "name": "John"}
+ mappings = [
+ {
+ "source_field": "age",
+ "target_field": "age",
+ "transforms": [{"type": "invalid_transform"}] # Invalid transform type
+ }
+ ]
+
+ # Should handle gracefully, possibly ignoring invalid transforms
+ result = apply_transformations(record, mappings)
+ assert "age" in result
+
+ def test_type_conversion_errors(self):
+ """Test handling of type conversion errors"""
+ record = {"age": "not_a_number", "price": "invalid_float", "active": "not_boolean"}
+ mappings = [
+ {"source_field": "age", "target_field": "age", "transforms": [{"type": "to_int"}]},
+ {"source_field": "price", "target_field": "price", "transforms": [{"type": "to_float"}]},
+ {"source_field": "active", "target_field": "active", "transforms": [{"type": "to_bool"}]}
+ ]
+
+ # Should handle conversion errors gracefully
+ result = apply_transformations(record, mappings)
+
+ # Should still have the fields, possibly with original or default values
+ assert "age" in result
+ assert "price" in result
+ assert "active" in result
+
+ def test_missing_source_fields(self):
+ """Test handling of mappings referencing missing source fields"""
+ record = {"name": "John", "email": "john@email.com"} # Missing 'age' field
+ mappings = [
+ {"source_field": "name", "target_field": "name", "transforms": []},
+ {"source_field": "age", "target_field": "age", "transforms": []}, # Missing field
+ {"source_field": "nonexistent", "target_field": "other", "transforms": []} # Also missing
+ ]
+
+ result = apply_transformations(record, mappings)
+
+ # Should include existing fields
+ assert result["name"] == "John"
+ # Missing fields should be handled (possibly skipped or empty)
+ # The exact behavior depends on implementation
+
+ # Network and API Error Tests
+ @patch('trustgraph.cli.load_structured_data.TrustGraphAPI')
+ def test_api_connection_failure(self, mock_api_class):
+ """Test handling of API connection failures"""
+ mock_api_class.side_effect = ConnectionError("Cannot connect to API")
+
+ input_file = self.create_temp_file("name,email\nJohn,john@email.com", '.csv')
+
+ try:
+ with pytest.raises(ConnectionError):
+ load_structured_data(
+ api_url=self.api_url,
+ input_file=input_file,
+ suggest_schema=True
+ )
+ finally:
+ self.cleanup_temp_file(input_file)
+
+ def test_websocket_connection_failure(self):
+ """Test WebSocket connection failure handling"""
+ input_file = self.create_temp_file("name,email\nJohn,john@email.com", '.csv')
+ descriptor_file = self.create_temp_file(json.dumps(self.valid_descriptor), '.json')
+
+ try:
+ # Test with invalid URL
+ with pytest.raises(Exception):
+ load_structured_data(
+ api_url="http://invalid-host:9999",
+ input_file=input_file,
+ descriptor_file=descriptor_file,
+ batch_size=1,
+ flow='obj-ex'
+ )
+ finally:
+ self.cleanup_temp_file(input_file)
+ self.cleanup_temp_file(descriptor_file)
+
+ # Edge Case Data Tests
+ def test_extremely_long_lines(self):
+ """Test handling of extremely long data lines"""
+ # Create CSV with very long line
+ long_description = "A" * 10000 # 10K character string
+ csv_data = f"name,description\nJohn,{long_description}\nJane,Short description"
+
+ format_info = {"type": "csv", "encoding": "utf-8", "options": {"header": True}}
+
+ records = parse_csv_data(csv_data, format_info)
+
+ assert len(records) == 2
+ assert records[0]["description"] == long_description
+ assert records[1]["name"] == "Jane"
+
+ def test_special_characters_handling(self):
+ """Test handling of special characters"""
+ special_csv = '''name,description,notes
+"John O'Connor","Senior Developer, Team Lead","Works on UI/UX & backend"
+"María García","Data Scientist","Specializes in NLP & ML"
+"张三","Software Engineer","Focuses on 中文 processing"'''
+
+ format_info = {"type": "csv", "encoding": "utf-8", "options": {"header": True}}
+
+ records = parse_csv_data(special_csv, format_info)
+
+ assert len(records) == 3
+ assert records[0]["name"] == "John O'Connor"
+ assert records[1]["name"] == "María García"
+ assert records[2]["name"] == "张三"
+
+ def test_unicode_and_encoding_issues(self):
+ """Test handling of Unicode and encoding issues"""
+ # This test would need specific encoding scenarios
+ unicode_data = "name,city\nJohn,München\nJane,Zürich\nBob,Kraków"
+
+ format_info = {"type": "csv", "encoding": "utf-8", "options": {"header": True}}
+
+ records = parse_csv_data(unicode_data, format_info)
+
+ assert len(records) == 3
+ assert records[0]["city"] == "München"
+ assert records[2]["city"] == "Kraków"
+
+ def test_null_and_empty_values(self):
+ """Test handling of null and empty values"""
+ csv_with_nulls = '''name,email,age,notes
+John,john@email.com,35,
+Jane,,28,Some notes
+,missing@email.com,,
+Bob,bob@email.com,42,'''
+
+ format_info = {"type": "csv", "encoding": "utf-8", "options": {"header": True}}
+
+ records = parse_csv_data(csv_with_nulls, format_info)
+
+ assert len(records) == 4
+ # Check empty values are handled
+ assert records[0]["notes"] == ""
+ assert records[1]["email"] == ""
+ assert records[2]["name"] == ""
+ assert records[2]["age"] == ""
+
+ def test_extremely_large_dataset(self):
+ """Test handling of extremely large datasets"""
+ # Generate large CSV
+ num_records = 10000
+ large_csv_lines = ["name,email,age"]
+
+ for i in range(num_records):
+ large_csv_lines.append(f"User{i},user{i}@example.com,{25 + i % 50}")
+
+ large_csv = "\n".join(large_csv_lines)
+
+ format_info = {"type": "csv", "encoding": "utf-8", "options": {"header": True}}
+
+ # This should not crash due to memory issues
+ records = parse_csv_data(large_csv, format_info)
+
+ assert len(records) == num_records
+ assert records[0]["name"] == "User0"
+ assert records[-1]["name"] == f"User{num_records-1}"
+
+ # Batch Processing Edge Cases
+ def test_batch_size_edge_cases(self):
+ """Test edge cases in batch size handling"""
+ records = [{"id": str(i), "name": f"User{i}"} for i in range(10)]
+
+ # Test batch size larger than data
+ batch_size = 20
+ batches = []
+ for i in range(0, len(records), batch_size):
+ batch_records = records[i:i + batch_size]
+ batches.append(batch_records)
+
+ assert len(batches) == 1
+ assert len(batches[0]) == 10
+
+ # Test batch size of 1
+ batch_size = 1
+ batches = []
+ for i in range(0, len(records), batch_size):
+ batch_records = records[i:i + batch_size]
+ batches.append(batch_records)
+
+ assert len(batches) == 10
+ assert all(len(batch) == 1 for batch in batches)
+
+ def test_zero_batch_size(self):
+ """Test handling of zero or invalid batch size"""
+ input_file = self.create_temp_file("name\nJohn\nJane", '.csv')
+ descriptor_file = self.create_temp_file(json.dumps(self.valid_descriptor), '.json')
+
+ try:
+ # Should handle invalid batch size gracefully
+ with pytest.raises(ValueError):
+ load_structured_data(
+ api_url=self.api_url,
+ input_file=input_file,
+ descriptor_file=descriptor_file,
+ batch_size=0, # Invalid batch size
+ dry_run=True
+ )
+ finally:
+ self.cleanup_temp_file(input_file)
+ self.cleanup_temp_file(descriptor_file)
+
+ # Memory and Performance Edge Cases
+ def test_memory_efficient_processing(self):
+ """Test that processing doesn't consume excessive memory"""
+ # This would be a performance test to ensure memory efficiency
+ # For unit testing, we just verify it doesn't crash
+ pass
+
+ def test_concurrent_access_safety(self):
+ """Test handling of concurrent access to temp files"""
+ # This would test file locking and concurrent access scenarios
+ pass
+
+ # Output File Error Tests
+ def test_output_file_permission_error(self):
+ """Test handling of output file permission errors"""
+ input_file = self.create_temp_file("name\nJohn", '.csv')
+ descriptor_file = self.create_temp_file(json.dumps(self.valid_descriptor), '.json')
+
+ try:
+ # Try to write to invalid location
+ with pytest.raises((PermissionError, OSError)):
+ load_structured_data(
+ api_url=self.api_url,
+ input_file=input_file,
+ descriptor_file=descriptor_file,
+ parse_only=True,
+ output_file="/root/forbidden.json" # Should fail on most systems
+ )
+ except Exception:
+ # Different systems may handle this differently
+ pass
+ finally:
+ self.cleanup_temp_file(input_file)
+ self.cleanup_temp_file(descriptor_file)
+
+ # Configuration Edge Cases
+ def test_invalid_flow_parameter(self):
+ """Test handling of invalid flow parameter"""
+ input_file = self.create_temp_file("name\nJohn", '.csv')
+ descriptor_file = self.create_temp_file(json.dumps(self.valid_descriptor), '.json')
+
+ try:
+ # Invalid flow should be handled gracefully (may just use as-is)
+ result = load_structured_data(
+ api_url=self.api_url,
+ input_file=input_file,
+ descriptor_file=descriptor_file,
+ flow="", # Empty flow
+ dry_run=True
+ )
+ finally:
+ self.cleanup_temp_file(input_file)
+ self.cleanup_temp_file(descriptor_file)
+
+ def test_conflicting_parameters(self):
+ """Test handling of conflicting command line parameters"""
+ input_file = self.create_temp_file("name\nJohn", '.csv')
+
+ try:
+ # Should handle conflicting modes gracefully or raise appropriate error
+ with pytest.raises(ValueError):
+ load_structured_data(
+ api_url=self.api_url,
+ input_file=input_file,
+ suggest_schema=True,
+ generate_descriptor=True, # Conflicting with suggest_schema
+ parse_only=True # Also conflicting
+ )
+ finally:
+ self.cleanup_temp_file(input_file)
\ No newline at end of file
diff --git a/tests/unit/test_cli/test_schema_descriptor_generation.py b/tests/unit/test_cli/test_schema_descriptor_generation.py
new file mode 100644
index 00000000..6ae63f3b
--- /dev/null
+++ b/tests/unit/test_cli/test_schema_descriptor_generation.py
@@ -0,0 +1,707 @@
+"""
+Unit tests for schema suggestion and descriptor generation functionality in tg-load-structured-data.
+Tests the --suggest-schema and --generate-descriptor modes.
+"""
+
+import pytest
+import json
+import tempfile
+import os
+from unittest.mock import Mock, patch, MagicMock
+
+from trustgraph.cli.load_structured_data import load_structured_data
+
+
+class TestSchemaDescriptorGeneration:
+ """Tests for schema suggestion and descriptor generation"""
+
+ def setup_method(self):
+ """Set up test fixtures"""
+ self.api_url = "http://localhost:8088"
+
+ # Sample data for different formats
+ self.customer_csv = """name,email,age,country,registration_date,status
+John Smith,john@email.com,35,USA,2024-01-15,active
+Jane Doe,jane@email.com,28,Canada,2024-01-20,active
+Bob Johnson,bob@company.org,42,UK,2024-01-10,inactive"""
+
+ self.product_json = [
+ {
+ "id": "PROD001",
+ "name": "Wireless Headphones",
+ "category": "Electronics",
+ "price": 99.99,
+ "in_stock": True,
+ "specifications": {
+ "battery_life": "24 hours",
+ "wireless": True,
+ "noise_cancellation": True
+ }
+ },
+ {
+ "id": "PROD002",
+ "name": "Coffee Maker",
+ "category": "Home & Kitchen",
+ "price": 129.99,
+ "in_stock": False,
+ "specifications": {
+ "capacity": "12 cups",
+ "programmable": True,
+ "auto_shutoff": True
+ }
+ }
+ ]
+
+ self.trade_xml = """
+
+
+
+ USA
+ Wheat
+ 1000000
+ 250000000
+ export
+
+
+ China
+ Electronics
+ 500000
+ 750000000
+ import
+
+
+"""
+
+ # Mock schema definitions
+ self.mock_schemas = {
+ "customer": json.dumps({
+ "name": "customer",
+ "description": "Customer information records",
+ "fields": [
+ {"name": "name", "type": "string", "required": True},
+ {"name": "email", "type": "string", "required": True},
+ {"name": "age", "type": "integer"},
+ {"name": "country", "type": "string"},
+ {"name": "status", "type": "string"}
+ ]
+ }),
+ "product": json.dumps({
+ "name": "product",
+ "description": "Product catalog information",
+ "fields": [
+ {"name": "id", "type": "string", "required": True, "primary_key": True},
+ {"name": "name", "type": "string", "required": True},
+ {"name": "category", "type": "string"},
+ {"name": "price", "type": "float"},
+ {"name": "in_stock", "type": "boolean"}
+ ]
+ }),
+ "trade_data": json.dumps({
+ "name": "trade_data",
+ "description": "International trade statistics",
+ "fields": [
+ {"name": "country", "type": "string", "required": True},
+ {"name": "product", "type": "string", "required": True},
+ {"name": "quantity", "type": "integer"},
+ {"name": "value_usd", "type": "float"},
+ {"name": "trade_type", "type": "string"}
+ ]
+ }),
+ "financial_record": json.dumps({
+ "name": "financial_record",
+ "description": "Financial transaction records",
+ "fields": [
+ {"name": "transaction_id", "type": "string", "primary_key": True},
+ {"name": "amount", "type": "float", "required": True},
+ {"name": "currency", "type": "string"},
+ {"name": "date", "type": "timestamp"}
+ ]
+ })
+ }
+
+ def create_temp_file(self, content, suffix='.txt'):
+ """Create a temporary file with given content"""
+ temp_file = tempfile.NamedTemporaryFile(mode='w', suffix=suffix, delete=False)
+ temp_file.write(content)
+ temp_file.flush()
+ temp_file.close()
+ return temp_file.name
+
+ def cleanup_temp_file(self, file_path):
+ """Clean up temporary file"""
+ try:
+ os.unlink(file_path)
+ except:
+ pass
+
+ # Schema Suggestion Tests
+ @patch('trustgraph.cli.load_structured_data.TrustGraphAPI')
+ def test_suggest_schema_csv_data(self, mock_api_class):
+ """Test schema suggestion for CSV data"""
+ # Setup mock API
+ mock_api = Mock()
+ mock_api_class.return_value = mock_api
+ mock_config_api = Mock()
+ mock_api.config.return_value = mock_config_api
+ mock_config_api.get_config_items.return_value = {"schema": self.mock_schemas}
+
+ mock_flow = Mock()
+ mock_api.flow.return_value = mock_flow
+ mock_flow.id.return_value = mock_flow
+ mock_prompt_client = Mock()
+ mock_flow.prompt.return_value = mock_prompt_client
+
+ # Mock schema selection response
+ mock_prompt_client.schema_selection.return_value = (
+ "Based on the data containing customer names, emails, ages, and countries, "
+ "the **customer** schema is the most appropriate choice. This schema includes "
+ "all the necessary fields for customer information and aligns well with the "
+ "structure of your data."
+ )
+
+ input_file = self.create_temp_file(self.customer_csv, '.csv')
+
+ try:
+ result = load_structured_data(
+ api_url=self.api_url,
+ input_file=input_file,
+ suggest_schema=True,
+ sample_size=100,
+ sample_chars=500
+ )
+
+ # Verify API calls were made correctly
+ mock_config_api.get_config_items.assert_called_once()
+ mock_prompt_client.schema_selection.assert_called_once()
+
+ # Check arguments passed to schema_selection
+ call_args = mock_prompt_client.schema_selection.call_args
+ assert 'schemas' in call_args.kwargs
+ assert 'sample' in call_args.kwargs
+
+ # Verify schemas were passed correctly
+ passed_schemas = call_args.kwargs['schemas']
+ assert len(passed_schemas) == len(self.mock_schemas)
+
+ # Check sample data was included
+ sample_data = call_args.kwargs['sample']
+ assert 'John Smith' in sample_data
+ assert 'jane@email.com' in sample_data
+
+ finally:
+ self.cleanup_temp_file(input_file)
+
+ @patch('trustgraph.cli.load_structured_data.TrustGraphAPI')
+ def test_suggest_schema_json_data(self, mock_api_class):
+ """Test schema suggestion for JSON data"""
+ mock_api = Mock()
+ mock_api_class.return_value = mock_api
+ mock_config_api = Mock()
+ mock_api.config.return_value = mock_config_api
+ mock_config_api.get_config_items.return_value = {"schema": self.mock_schemas}
+
+ mock_flow = Mock()
+ mock_api.flow.return_value = mock_flow
+ mock_flow.id.return_value = mock_flow
+ mock_prompt_client = Mock()
+ mock_flow.prompt.return_value = mock_prompt_client
+
+ mock_prompt_client.schema_selection.return_value = (
+ "The **product** schema is ideal for this dataset containing product IDs, "
+ "names, categories, prices, and stock status. This matches perfectly with "
+ "the product schema structure."
+ )
+
+ input_file = self.create_temp_file(json.dumps(self.product_json), '.json')
+
+ try:
+ result = load_structured_data(
+ api_url=self.api_url,
+ input_file=input_file,
+ suggest_schema=True,
+ sample_chars=1000
+ )
+
+ # Verify the call was made
+ mock_prompt_client.schema_selection.assert_called_once()
+
+ # Check that JSON data was properly sampled
+ call_args = mock_prompt_client.schema_selection.call_args
+ sample_data = call_args.kwargs['sample']
+ assert 'PROD001' in sample_data
+ assert 'Wireless Headphones' in sample_data
+ assert 'Electronics' in sample_data
+
+ finally:
+ self.cleanup_temp_file(input_file)
+
+ @patch('trustgraph.cli.load_structured_data.TrustGraphAPI')
+ def test_suggest_schema_xml_data(self, mock_api_class):
+ """Test schema suggestion for XML data"""
+ mock_api = Mock()
+ mock_api_class.return_value = mock_api
+ mock_config_api = Mock()
+ mock_api.config.return_value = mock_config_api
+ mock_config_api.get_config_items.return_value = {"schema": self.mock_schemas}
+
+ mock_flow = Mock()
+ mock_api.flow.return_value = mock_flow
+ mock_flow.id.return_value = mock_flow
+ mock_prompt_client = Mock()
+ mock_flow.prompt.return_value = mock_prompt_client
+
+ mock_prompt_client.schema_selection.return_value = (
+ "The **trade_data** schema is the best fit for this XML data containing "
+ "country, product, quantity, value, and trade type information. This aligns "
+ "perfectly with international trade statistics."
+ )
+
+ input_file = self.create_temp_file(self.trade_xml, '.xml')
+
+ try:
+ result = load_structured_data(
+ api_url=self.api_url,
+ input_file=input_file,
+ suggest_schema=True,
+ sample_chars=800
+ )
+
+ mock_prompt_client.schema_selection.assert_called_once()
+
+ # Verify XML content was included in sample
+ call_args = mock_prompt_client.schema_selection.call_args
+ sample_data = call_args.kwargs['sample']
+ assert 'field name="country"' in sample_data or 'country' in sample_data
+ assert 'USA' in sample_data
+ assert 'export' in sample_data
+
+ finally:
+ self.cleanup_temp_file(input_file)
+
+ @patch('trustgraph.cli.load_structured_data.TrustGraphAPI')
+ def test_suggest_schema_sample_size_limiting(self, mock_api_class):
+ """Test that sample size is properly limited"""
+ mock_api = Mock()
+ mock_api_class.return_value = mock_api
+ mock_config_api = Mock()
+ mock_api.config.return_value = mock_config_api
+ mock_config_api.get_config_items.return_value = {"schema": self.mock_schemas}
+
+ mock_flow = Mock()
+ mock_api.flow.return_value = mock_flow
+ mock_flow.id.return_value = mock_flow
+ mock_prompt_client = Mock()
+ mock_flow.prompt.return_value = mock_prompt_client
+ mock_prompt_client.schema_selection.return_value = "customer schema recommended"
+
+ # Create large CSV file
+ large_csv = "name,email,age\n" + "\n".join([f"User{i},user{i}@example.com,{20+i}" for i in range(1000)])
+ input_file = self.create_temp_file(large_csv, '.csv')
+
+ try:
+ result = load_structured_data(
+ api_url=self.api_url,
+ input_file=input_file,
+ suggest_schema=True,
+ sample_size=10, # Limit to 10 records
+ sample_chars=200 # Limit to 200 characters
+ )
+
+ # Check that sample was limited
+ call_args = mock_prompt_client.schema_selection.call_args
+ sample_data = call_args.kwargs['sample']
+
+ # Should be limited by sample_chars
+ assert len(sample_data) <= 250 # Some margin for formatting
+
+ # Should not contain all 1000 users
+ user_count = sample_data.count('User')
+ assert user_count < 20 # Much less than 1000
+
+ finally:
+ self.cleanup_temp_file(input_file)
+
+ # Descriptor Generation Tests
+ @patch('trustgraph.cli.load_structured_data.TrustGraphAPI')
+ def test_generate_descriptor_csv_format(self, mock_api_class):
+ """Test descriptor generation for CSV format"""
+ mock_api = Mock()
+ mock_api_class.return_value = mock_api
+ mock_config_api = Mock()
+ mock_api.config.return_value = mock_config_api
+ mock_config_api.get_config_items.return_value = {"schema": self.mock_schemas}
+
+ mock_flow = Mock()
+ mock_api.flow.return_value = mock_flow
+ mock_flow.id.return_value = mock_flow
+ mock_prompt_client = Mock()
+ mock_flow.prompt.return_value = mock_prompt_client
+
+ # Mock descriptor generation response
+ generated_descriptor = {
+ "version": "1.0",
+ "metadata": {
+ "name": "CustomerDataImport",
+ "description": "Import customer data from CSV",
+ "author": "TrustGraph"
+ },
+ "format": {
+ "type": "csv",
+ "encoding": "utf-8",
+ "options": {
+ "header": True,
+ "delimiter": ","
+ }
+ },
+ "mappings": [
+ {
+ "source_field": "name",
+ "target_field": "name",
+ "transforms": [{"type": "trim"}],
+ "validation": [{"type": "required"}]
+ },
+ {
+ "source_field": "email",
+ "target_field": "email",
+ "transforms": [{"type": "trim"}, {"type": "lower"}],
+ "validation": [{"type": "required"}]
+ },
+ {
+ "source_field": "age",
+ "target_field": "age",
+ "transforms": [{"type": "to_int"}],
+ "validation": [{"type": "required"}]
+ }
+ ],
+ "output": {
+ "format": "trustgraph-objects",
+ "schema_name": "customer",
+ "options": {
+ "confidence": 0.85,
+ "batch_size": 100
+ }
+ }
+ }
+
+ mock_prompt_client.diagnose_structured_data.return_value = json.dumps(generated_descriptor)
+
+ input_file = self.create_temp_file(self.customer_csv, '.csv')
+
+ try:
+ result = load_structured_data(
+ api_url=self.api_url,
+ input_file=input_file,
+ generate_descriptor=True,
+ sample_chars=1000
+ )
+
+ # Verify API calls
+ mock_prompt_client.diagnose_structured_data.assert_called_once()
+
+ # Check call arguments
+ call_args = mock_prompt_client.diagnose_structured_data.call_args
+ assert 'schemas' in call_args.kwargs
+ assert 'sample' in call_args.kwargs
+
+ # Verify CSV data was included
+ sample_data = call_args.kwargs['sample']
+ assert 'name,email,age,country' in sample_data # Header
+ assert 'John Smith' in sample_data
+
+ # Verify schemas were passed
+ passed_schemas = call_args.kwargs['schemas']
+ assert len(passed_schemas) > 0
+
+ finally:
+ self.cleanup_temp_file(input_file)
+
+ @patch('trustgraph.cli.load_structured_data.TrustGraphAPI')
+ def test_generate_descriptor_json_format(self, mock_api_class):
+ """Test descriptor generation for JSON format"""
+ mock_api = Mock()
+ mock_api_class.return_value = mock_api
+ mock_config_api = Mock()
+ mock_api.config.return_value = mock_config_api
+ mock_config_api.get_config_items.return_value = {"schema": self.mock_schemas}
+
+ mock_flow = Mock()
+ mock_api.flow.return_value = mock_flow
+ mock_flow.id.return_value = mock_flow
+ mock_prompt_client = Mock()
+ mock_flow.prompt.return_value = mock_prompt_client
+
+ generated_descriptor = {
+ "version": "1.0",
+ "format": {
+ "type": "json",
+ "encoding": "utf-8"
+ },
+ "mappings": [
+ {
+ "source_field": "id",
+ "target_field": "product_id",
+ "transforms": [{"type": "trim"}],
+ "validation": [{"type": "required"}]
+ },
+ {
+ "source_field": "name",
+ "target_field": "product_name",
+ "transforms": [{"type": "trim"}],
+ "validation": [{"type": "required"}]
+ },
+ {
+ "source_field": "price",
+ "target_field": "price",
+ "transforms": [{"type": "to_float"}],
+ "validation": []
+ }
+ ],
+ "output": {
+ "format": "trustgraph-objects",
+ "schema_name": "product",
+ "options": {"confidence": 0.9, "batch_size": 50}
+ }
+ }
+
+ mock_prompt_client.diagnose_structured_data.return_value = json.dumps(generated_descriptor)
+
+ input_file = self.create_temp_file(json.dumps(self.product_json), '.json')
+
+ try:
+ result = load_structured_data(
+ api_url=self.api_url,
+ input_file=input_file,
+ generate_descriptor=True
+ )
+
+ mock_prompt_client.diagnose_structured_data.assert_called_once()
+
+ # Verify JSON structure was analyzed
+ call_args = mock_prompt_client.diagnose_structured_data.call_args
+ sample_data = call_args.kwargs['sample']
+ assert 'PROD001' in sample_data
+ assert 'Wireless Headphones' in sample_data
+ assert '99.99' in sample_data
+
+ finally:
+ self.cleanup_temp_file(input_file)
+
+ @patch('trustgraph.cli.load_structured_data.TrustGraphAPI')
+ def test_generate_descriptor_xml_format(self, mock_api_class):
+ """Test descriptor generation for XML format"""
+ mock_api = Mock()
+ mock_api_class.return_value = mock_api
+ mock_config_api = Mock()
+ mock_api.config.return_value = mock_config_api
+ mock_config_api.get_config_items.return_value = {"schema": self.mock_schemas}
+
+ mock_flow = Mock()
+ mock_api.flow.return_value = mock_flow
+ mock_flow.id.return_value = mock_flow
+ mock_prompt_client = Mock()
+ mock_flow.prompt.return_value = mock_prompt_client
+
+ # XML descriptor should include XPath configuration
+ xml_descriptor = {
+ "version": "1.0",
+ "format": {
+ "type": "xml",
+ "encoding": "utf-8",
+ "options": {
+ "record_path": "/ROOT/data/record",
+ "field_attribute": "name"
+ }
+ },
+ "mappings": [
+ {
+ "source_field": "country",
+ "target_field": "country",
+ "transforms": [{"type": "trim"}, {"type": "upper"}],
+ "validation": [{"type": "required"}]
+ },
+ {
+ "source_field": "value_usd",
+ "target_field": "trade_value",
+ "transforms": [{"type": "to_float"}],
+ "validation": []
+ }
+ ],
+ "output": {
+ "format": "trustgraph-objects",
+ "schema_name": "trade_data",
+ "options": {"confidence": 0.8, "batch_size": 25}
+ }
+ }
+
+ mock_prompt_client.diagnose_structured_data.return_value = json.dumps(xml_descriptor)
+
+ input_file = self.create_temp_file(self.trade_xml, '.xml')
+
+ try:
+ result = load_structured_data(
+ api_url=self.api_url,
+ input_file=input_file,
+ generate_descriptor=True
+ )
+
+ mock_prompt_client.diagnose_structured_data.assert_called_once()
+
+ # Verify XML structure was included
+ call_args = mock_prompt_client.diagnose_structured_data.call_args
+ sample_data = call_args.kwargs['sample']
+ assert '' in sample_data
+ assert 'field name=' in sample_data
+ assert 'USA' in sample_data
+
+ finally:
+ self.cleanup_temp_file(input_file)
+
+ # Error Handling Tests
+ @patch('trustgraph.cli.load_structured_data.TrustGraphAPI')
+ def test_suggest_schema_no_schemas_available(self, mock_api_class):
+ """Test schema suggestion when no schemas are available"""
+ mock_api = Mock()
+ mock_api_class.return_value = mock_api
+ mock_config_api = Mock()
+ mock_api.config.return_value = mock_config_api
+ mock_config_api.get_config_items.return_value = {"schema": {}} # Empty schemas
+
+ input_file = self.create_temp_file(self.customer_csv, '.csv')
+
+ try:
+ with pytest.raises(ValueError) as exc_info:
+ result = load_structured_data(
+ api_url=self.api_url,
+ input_file=input_file,
+ suggest_schema=True
+ )
+
+ assert "no schemas" in str(exc_info.value).lower()
+
+ finally:
+ self.cleanup_temp_file(input_file)
+
+ @patch('trustgraph.cli.load_structured_data.TrustGraphAPI')
+ def test_generate_descriptor_api_error(self, mock_api_class):
+ """Test descriptor generation when API returns error"""
+ mock_api = Mock()
+ mock_api_class.return_value = mock_api
+ mock_config_api = Mock()
+ mock_api.config.return_value = mock_config_api
+ mock_config_api.get_config_items.return_value = {"schema": self.mock_schemas}
+
+ mock_flow = Mock()
+ mock_api.flow.return_value = mock_flow
+ mock_flow.id.return_value = mock_flow
+ mock_prompt_client = Mock()
+ mock_flow.prompt.return_value = mock_prompt_client
+
+ # Mock API error
+ mock_prompt_client.diagnose_structured_data.side_effect = Exception("API connection failed")
+
+ input_file = self.create_temp_file(self.customer_csv, '.csv')
+
+ try:
+ with pytest.raises(Exception) as exc_info:
+ result = load_structured_data(
+ api_url=self.api_url,
+ input_file=input_file,
+ generate_descriptor=True
+ )
+
+ assert "API connection failed" in str(exc_info.value)
+
+ finally:
+ self.cleanup_temp_file(input_file)
+
+ @patch('trustgraph.cli.load_structured_data.TrustGraphAPI')
+ def test_generate_descriptor_invalid_response(self, mock_api_class):
+ """Test descriptor generation with invalid API response"""
+ mock_api = Mock()
+ mock_api_class.return_value = mock_api
+ mock_config_api = Mock()
+ mock_api.config.return_value = mock_config_api
+ mock_config_api.get_config_items.return_value = {"schema": self.mock_schemas}
+
+ mock_flow = Mock()
+ mock_api.flow.return_value = mock_flow
+ mock_flow.id.return_value = mock_flow
+ mock_prompt_client = Mock()
+ mock_flow.prompt.return_value = mock_prompt_client
+
+ # Return invalid JSON
+ mock_prompt_client.diagnose_structured_data.return_value = "invalid json response"
+
+ input_file = self.create_temp_file(self.customer_csv, '.csv')
+
+ try:
+ with pytest.raises(json.JSONDecodeError):
+ result = load_structured_data(
+ api_url=self.api_url,
+ input_file=input_file,
+ generate_descriptor=True
+ )
+
+ finally:
+ self.cleanup_temp_file(input_file)
+
+ # Output Format Tests
+ def test_suggest_schema_output_format(self):
+ """Test that schema suggestion produces proper output format"""
+ # This would be tested with actual TrustGraph instance
+ # Here we verify the expected behavior structure
+ pass
+
+ def test_generate_descriptor_output_to_file(self):
+ """Test descriptor generation with file output"""
+ # Test would verify descriptor is written to specified file
+ pass
+
+ # Sample Data Quality Tests
+ @patch('trustgraph.cli.load_structured_data.TrustGraphAPI')
+ def test_sample_data_quality_csv(self, mock_api_class):
+ """Test that sample data quality is maintained for CSV"""
+ mock_api = Mock()
+ mock_api_class.return_value = mock_api
+ mock_config_api = Mock()
+ mock_api.config.return_value = mock_config_api
+ mock_config_api.get_config_items.return_value = {"schema": self.mock_schemas}
+
+ mock_flow = Mock()
+ mock_api.flow.return_value = mock_flow
+ mock_flow.id.return_value = mock_flow
+ mock_prompt_client = Mock()
+ mock_flow.prompt.return_value = mock_prompt_client
+ mock_prompt_client.schema_selection.return_value = "customer schema recommended"
+
+ # CSV with various data types and edge cases
+ complex_csv = """name,email,age,salary,join_date,is_active,notes
+John O'Connor,"john@company.com",35,75000.50,2024-01-15,true,"Senior Developer, Team Lead"
+Jane "Smith" Doe,jane@email.com,28,65000,2024-02-01,true,"Data Scientist, ML Expert"
+Bob,bob@temp.org,42,,2023-12-01,false,"Contractor, Part-time"
+,missing@email.com,25,45000,2024-03-01,true,"Junior Developer, New Hire" """
+
+ input_file = self.create_temp_file(complex_csv, '.csv')
+
+ try:
+ result = load_structured_data(
+ api_url=self.api_url,
+ input_file=input_file,
+ suggest_schema=True,
+ sample_chars=1000
+ )
+
+ # Check that sample preserves important characteristics
+ call_args = mock_prompt_client.schema_selection.call_args
+ sample_data = call_args.kwargs['sample']
+
+ # Should preserve header
+ assert 'name,email,age,salary' in sample_data
+
+ # Should include examples of data variety
+ assert "John O'Connor" in sample_data or 'John' in sample_data
+ assert '@' in sample_data # Email format
+ assert '75000' in sample_data or '65000' in sample_data # Numeric data
+
+ finally:
+ self.cleanup_temp_file(input_file)
\ No newline at end of file
diff --git a/tests/unit/test_cli/test_xml_xpath_parsing.py b/tests/unit/test_cli/test_xml_xpath_parsing.py
new file mode 100644
index 00000000..e56f7369
--- /dev/null
+++ b/tests/unit/test_cli/test_xml_xpath_parsing.py
@@ -0,0 +1,626 @@
+"""
+Specialized unit tests for XML parsing and XPath functionality in tg-load-structured-data.
+Tests complex XML structures, XPath expressions, and field attribute handling.
+"""
+
+import pytest
+import json
+import tempfile
+import os
+import xml.etree.ElementTree as ET
+
+from trustgraph.cli.load_structured_data import load_structured_data
+
+
+class TestXMLXPathParsing:
+ """Specialized tests for XML parsing with XPath support"""
+
+ def setup_method(self):
+ """Set up test fixtures"""
+ # UN Trade Data format (real-world complex XML)
+ self.un_trade_xml = """
+
+
+
+ Albania
+ 2024
+ Coffee; not roasted or decaffeinated
+ import
+ 24445532.903
+ 5305568.05
+
+
+ Algeria
+ 2024
+ Tea
+ export
+ 12345678.90
+ 2500000.00
+
+
+"""
+
+ # Standard XML with attributes
+ self.product_xml = """
+
+
+ Laptop
+ 999.99
+ High-performance laptop
+
+ Intel i7
+ 16GB
+ 512GB SSD
+
+
+
+ Python Programming
+ 49.99
+ Learn Python programming
+
+ 500
+ English
+ Paperback
+
+
+"""
+
+ # Nested XML structure
+ self.nested_xml = """
+
+
+
+ John Smith
+ john@email.com
+
+ 123 Main St
+ New York
+ USA
+
+
+
+ -
+ Widget A
+ 19.99
+
+ -
+ Widget B
+ 29.99
+
+
+
+"""
+
+ # XML with mixed content and namespaces
+ self.namespace_xml = """
+
+
+
+ Smartphone
+ 599.99
+
+
+ Tablet
+ 399.99
+
+
+"""
+
+ def create_temp_file(self, content, suffix='.txt'):
+ """Create a temporary file with given content"""
+ temp_file = tempfile.NamedTemporaryFile(mode='w', suffix=suffix, delete=False)
+ temp_file.write(content)
+ temp_file.flush()
+ temp_file.close()
+ return temp_file.name
+
+ def cleanup_temp_file(self, file_path):
+ """Clean up temporary file"""
+ try:
+ os.unlink(file_path)
+ except:
+ pass
+
+ # UN Data Format Tests (CLI-level testing)
+ def test_un_trade_data_xpath_parsing(self):
+ """Test parsing UN trade data format with field attributes via CLI"""
+ descriptor = {
+ "version": "1.0",
+ "format": {
+ "type": "xml",
+ "encoding": "utf-8",
+ "options": {
+ "record_path": "/ROOT/data/record",
+ "field_attribute": "name"
+ }
+ },
+ "mappings": [
+ {"source_field": "country_or_area", "target_field": "country", "transforms": []},
+ {"source_field": "commodity", "target_field": "product", "transforms": []},
+ {"source_field": "trade_usd", "target_field": "value", "transforms": []}
+ ],
+ "output": {
+ "format": "trustgraph-objects",
+ "schema_name": "trade_data",
+ "options": {"confidence": 0.9, "batch_size": 10}
+ }
+ }
+
+ input_file = self.create_temp_file(self.un_trade_xml, '.xml')
+ descriptor_file = self.create_temp_file(json.dumps(descriptor), '.json')
+ output_file = tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False)
+ output_file.close()
+
+ try:
+ # Test parse-only mode to verify XML parsing works
+ load_structured_data(
+ api_url="http://localhost:8088",
+ input_file=input_file,
+ descriptor_file=descriptor_file,
+ parse_only=True,
+ output_file=output_file.name
+ )
+
+ # Verify parsing worked
+ assert os.path.exists(output_file.name)
+ with open(output_file.name, 'r') as f:
+ parsed_data = json.load(f)
+ assert len(parsed_data) == 2
+ assert parsed_data[0]["country"] == "Albania"
+ assert parsed_data[0]["product"] == "Coffee; not roasted or decaffeinated"
+ assert parsed_data[1]["country"] == "Algeria"
+
+ finally:
+ self.cleanup_temp_file(input_file)
+ self.cleanup_temp_file(descriptor_file)
+ self.cleanup_temp_file(output_file.name)
+
+ def test_xpath_record_path_variations(self):
+ """Test different XPath record path expressions"""
+ # Test with leading slash
+ format_info_1 = {
+ "type": "xml",
+ "encoding": "utf-8",
+ "options": {
+ "record_path": "/ROOT/data/record",
+ "field_attribute": "name"
+ }
+ }
+
+ records_1 = parse_xml_data(self.un_trade_xml, format_info_1)
+ assert len(records_1) == 2
+
+ # Test with double slash (descendant)
+ format_info_2 = {
+ "type": "xml",
+ "encoding": "utf-8",
+ "options": {
+ "record_path": "//record",
+ "field_attribute": "name"
+ }
+ }
+
+ records_2 = parse_xml_data(self.un_trade_xml, format_info_2)
+ assert len(records_2) == 2
+
+ # Results should be the same
+ assert records_1[0]["country_or_area"] == records_2[0]["country_or_area"]
+
+ def test_field_attribute_parsing(self):
+ """Test field attribute parsing mechanism"""
+ format_info = {
+ "type": "xml",
+ "encoding": "utf-8",
+ "options": {
+ "record_path": "/ROOT/data/record",
+ "field_attribute": "name"
+ }
+ }
+
+ records = parse_xml_data(self.un_trade_xml, format_info)
+
+ # Should extract all fields defined by 'name' attribute
+ expected_fields = ["country_or_area", "year", "commodity", "flow", "trade_usd", "weight_kg"]
+
+ for record in records:
+ for field in expected_fields:
+ assert field in record, f"Field {field} should be extracted from XML"
+ assert record[field], f"Field {field} should have a value"
+
+ # Standard XML Structure Tests
+ def test_standard_xml_with_attributes(self):
+ """Test parsing standard XML with element attributes"""
+ format_info = {
+ "type": "xml",
+ "encoding": "utf-8",
+ "options": {
+ "record_path": "//product"
+ }
+ }
+
+ records = parse_xml_data(self.product_xml, format_info)
+
+ assert len(records) == 2
+
+ # Check attributes are captured
+ first_product = records[0]
+ assert first_product["id"] == "1"
+ assert first_product["category"] == "electronics"
+ assert first_product["name"] == "Laptop"
+ assert first_product["price"] == "999.99"
+
+ second_product = records[1]
+ assert second_product["id"] == "2"
+ assert second_product["category"] == "books"
+ assert second_product["name"] == "Python Programming"
+
+ def test_nested_xml_structure_parsing(self):
+ """Test parsing deeply nested XML structures"""
+ # Test extracting order-level data
+ format_info = {
+ "type": "xml",
+ "encoding": "utf-8",
+ "options": {
+ "record_path": "//order"
+ }
+ }
+
+ records = parse_xml_data(self.nested_xml, format_info)
+
+ assert len(records) == 1
+
+ order = records[0]
+ assert order["order_id"] == "ORD001"
+ assert order["date"] == "2024-01-15"
+ # Nested elements should be flattened
+ assert "name" in order # Customer name
+ assert order["name"] == "John Smith"
+
+ def test_nested_item_extraction(self):
+ """Test extracting items from nested XML"""
+ # Test extracting individual items
+ format_info = {
+ "type": "xml",
+ "encoding": "utf-8",
+ "options": {
+ "record_path": "//item"
+ }
+ }
+
+ records = parse_xml_data(self.nested_xml, format_info)
+
+ assert len(records) == 2
+
+ first_item = records[0]
+ assert first_item["sku"] == "ITEM001"
+ assert first_item["quantity"] == "2"
+ assert first_item["name"] == "Widget A"
+ assert first_item["price"] == "19.99"
+
+ second_item = records[1]
+ assert second_item["sku"] == "ITEM002"
+ assert second_item["quantity"] == "1"
+ assert second_item["name"] == "Widget B"
+
+ # Complex XPath Expression Tests
+ def test_complex_xpath_expressions(self):
+ """Test complex XPath expressions"""
+ # Test with predicate - only electronics products
+ electronics_xml = """
+
+
+ Laptop
+ 999.99
+
+
+ Novel
+ 19.99
+
+
+ Phone
+ 599.99
+
+"""
+
+ # XPath with attribute filter
+ format_info = {
+ "type": "xml",
+ "encoding": "utf-8",
+ "options": {
+ "record_path": "//product[@category='electronics']"
+ }
+ }
+
+ records = parse_xml_data(electronics_xml, format_info)
+
+ # Should only get electronics products
+ assert len(records) == 2
+ assert records[0]["name"] == "Laptop"
+ assert records[1]["name"] == "Phone"
+
+ # Both should have electronics category
+ for record in records:
+ assert record["category"] == "electronics"
+
+ def test_xpath_with_position(self):
+ """Test XPath expressions with position predicates"""
+ format_info = {
+ "type": "xml",
+ "encoding": "utf-8",
+ "options": {
+ "record_path": "//product[1]" # First product only
+ }
+ }
+
+ records = parse_xml_data(self.product_xml, format_info)
+
+ # Should only get first product
+ assert len(records) == 1
+ assert records[0]["name"] == "Laptop"
+ assert records[0]["id"] == "1"
+
+ # Namespace Handling Tests
+ def test_xml_with_namespaces(self):
+ """Test XML parsing with namespaces"""
+ # Note: ElementTree has limited namespace support in XPath
+ format_info = {
+ "type": "xml",
+ "encoding": "utf-8",
+ "options": {
+ "record_path": "//{http://example.com/products}item"
+ }
+ }
+
+ try:
+ records = parse_xml_data(self.namespace_xml, format_info)
+
+ # Should find items with namespace
+ assert len(records) >= 1
+
+ except Exception:
+ # ElementTree may not support full namespace XPath
+ # This is expected behavior - document the limitation
+ pass
+
+ # Error Handling Tests
+ def test_invalid_xpath_expression(self):
+ """Test handling of invalid XPath expressions"""
+ format_info = {
+ "type": "xml",
+ "encoding": "utf-8",
+ "options": {
+ "record_path": "//[invalid xpath" # Malformed XPath
+ }
+ }
+
+ with pytest.raises(Exception):
+ records = parse_xml_data(self.un_trade_xml, format_info)
+
+ def test_xpath_no_matches(self):
+ """Test XPath that matches no elements"""
+ format_info = {
+ "type": "xml",
+ "encoding": "utf-8",
+ "options": {
+ "record_path": "//nonexistent"
+ }
+ }
+
+ records = parse_xml_data(self.un_trade_xml, format_info)
+
+ # Should return empty list
+ assert len(records) == 0
+ assert isinstance(records, list)
+
+ def test_malformed_xml_handling(self):
+ """Test handling of malformed XML"""
+ malformed_xml = """
+
+
+ value
+
+
+"""
+
+ format_info = {
+ "type": "xml",
+ "encoding": "utf-8",
+ "options": {
+ "record_path": "//record"
+ }
+ }
+
+ with pytest.raises(ET.ParseError):
+ records = parse_xml_data(malformed_xml, format_info)
+
+ # Field Attribute Variations Tests
+ def test_different_field_attribute_names(self):
+ """Test different field attribute names"""
+ custom_xml = """
+
+
+ John
+ 35
+ NYC
+
+"""
+
+ format_info = {
+ "type": "xml",
+ "encoding": "utf-8",
+ "options": {
+ "record_path": "//record",
+ "field_attribute": "key" # Using 'key' instead of 'name'
+ }
+ }
+
+ records = parse_xml_data(custom_xml, format_info)
+
+ assert len(records) == 1
+ record = records[0]
+ assert record["name"] == "John"
+ assert record["age"] == "35"
+ assert record["city"] == "NYC"
+
+ def test_missing_field_attribute(self):
+ """Test handling when field_attribute is specified but not found"""
+ xml_without_attributes = """
+
+
+ John
+ 35
+
+"""
+
+ format_info = {
+ "type": "xml",
+ "encoding": "utf-8",
+ "options": {
+ "record_path": "//record",
+ "field_attribute": "name" # Looking for 'name' attribute but elements don't have it
+ }
+ }
+
+ records = parse_xml_data(xml_without_attributes, format_info)
+
+ assert len(records) == 1
+ # Should fall back to standard parsing
+ record = records[0]
+ assert record["name"] == "John"
+ assert record["age"] == "35"
+
+ # Mixed Content Tests
+ def test_xml_with_mixed_content(self):
+ """Test XML with mixed text and element content"""
+ mixed_xml = """
+
+
+ John Smith works at ACME Corp in NYC
+
+
+ Jane Doe works at Tech Inc in SF
+
+"""
+
+ format_info = {
+ "type": "xml",
+ "encoding": "utf-8",
+ "options": {
+ "record_path": "//person"
+ }
+ }
+
+ records = parse_xml_data(mixed_xml, format_info)
+
+ assert len(records) == 2
+
+ # Should capture both attributes and child elements
+ first_person = records[0]
+ assert first_person["id"] == "1"
+ assert first_person["company"] == "ACME Corp"
+ assert first_person["city"] == "NYC"
+
+ # Integration with Transformation Tests
+ def test_xml_with_transformations(self):
+ """Test XML parsing with data transformations"""
+ records = parse_xml_data(self.un_trade_xml, {
+ "type": "xml",
+ "encoding": "utf-8",
+ "options": {
+ "record_path": "/ROOT/data/record",
+ "field_attribute": "name"
+ }
+ })
+
+ # Apply transformations
+ mappings = [
+ {
+ "source_field": "country_or_area",
+ "target_field": "country",
+ "transforms": [{"type": "upper"}]
+ },
+ {
+ "source_field": "trade_usd",
+ "target_field": "trade_value",
+ "transforms": [{"type": "to_float"}]
+ },
+ {
+ "source_field": "year",
+ "target_field": "year",
+ "transforms": [{"type": "to_int"}]
+ }
+ ]
+
+ transformed_records = []
+ for record in records:
+ transformed = apply_transformations(record, mappings)
+ transformed_records.append(transformed)
+
+ # Check transformations were applied
+ first_transformed = transformed_records[0]
+ assert first_transformed["country"] == "ALBANIA"
+ assert first_transformed["trade_value"] == "24445532.903" # Converted to string for ExtractedObject
+ assert first_transformed["year"] == "2024"
+
+ # Real-world Complexity Tests
+ def test_complex_real_world_xml(self):
+ """Test with complex real-world XML structure"""
+ complex_xml = """
+
+
+ 2024-01-15T10:30:00Z
+ Trade Statistics Database
+
+
+
+ United States
+ China
+ 854232
+ Integrated circuits
+ Import
+ 202401
+
+ 15000000.50
+ 125000.75
+ 120.00
+
+
+
+ United States
+ Germany
+ 870323
+ Motor cars
+ Import
+ 202401
+
+ 5000000.00
+ 250
+ 20000.00
+
+
+
+"""
+
+ format_info = {
+ "type": "xml",
+ "encoding": "utf-8",
+ "options": {
+ "record_path": "//trade_record"
+ }
+ }
+
+ records = parse_xml_data(complex_xml, format_info)
+
+ assert len(records) == 2
+
+ # Check first record structure
+ first_record = records[0]
+ assert first_record["reporting_country"] == "United States"
+ assert first_record["partner_country"] == "China"
+ assert first_record["commodity_code"] == "854232"
+ assert first_record["trade_flow"] == "Import"
+
+ # Check second record
+ second_record = records[1]
+ assert second_record["partner_country"] == "Germany"
+ assert second_record["commodity_description"] == "Motor cars"
\ No newline at end of file