diff --git a/tests/unit/test_cli/test_error_handling_edge_cases.py b/tests/unit/test_cli/test_error_handling_edge_cases.py new file mode 100644 index 00000000..77eb88e0 --- /dev/null +++ b/tests/unit/test_cli/test_error_handling_edge_cases.py @@ -0,0 +1,520 @@ +""" +Error handling and edge case tests for tg-load-structured-data CLI command. +Tests various failure scenarios, malformed data, and boundary conditions. +""" + +import pytest +import json +import tempfile +import os +import csv +from unittest.mock import Mock, patch, AsyncMock +from io import StringIO + +from trustgraph.cli.load_structured_data import load_structured_data + + +class TestErrorHandlingEdgeCases: + """Tests for error handling and edge cases""" + + def setup_method(self): + """Set up test fixtures""" + self.api_url = "http://localhost:8088" + + # Valid descriptor for testing + self.valid_descriptor = { + "version": "1.0", + "format": { + "type": "csv", + "encoding": "utf-8", + "options": {"header": True, "delimiter": ","} + }, + "mappings": [ + {"source_field": "name", "target_field": "name", "transforms": [{"type": "trim"}]}, + {"source_field": "email", "target_field": "email", "transforms": [{"type": "lower"}]} + ], + "output": { + "format": "trustgraph-objects", + "schema_name": "test_schema", + "options": {"confidence": 0.9, "batch_size": 10} + } + } + + def create_temp_file(self, content, suffix='.txt'): + """Create a temporary file with given content""" + temp_file = tempfile.NamedTemporaryFile(mode='w', suffix=suffix, delete=False) + temp_file.write(content) + temp_file.flush() + temp_file.close() + return temp_file.name + + def cleanup_temp_file(self, file_path): + """Clean up temporary file""" + try: + os.unlink(file_path) + except: + pass + + # File Access Error Tests + def test_nonexistent_input_file(self): + """Test handling of nonexistent input file""" + with pytest.raises(FileNotFoundError): + load_structured_data( + api_url=self.api_url, + input_file="/nonexistent/path/file.csv" + ) + + def test_nonexistent_descriptor_file(self): + """Test handling of nonexistent descriptor file""" + input_file = self.create_temp_file("name,email\nJohn,john@email.com", '.csv') + + try: + with pytest.raises(FileNotFoundError): + load_structured_data( + api_url=self.api_url, + input_file=input_file, + descriptor_file="/nonexistent/descriptor.json" + ) + finally: + self.cleanup_temp_file(input_file) + + def test_permission_denied_file(self): + """Test handling of permission denied errors""" + # This test would need to create a file with restricted permissions + # Skip on systems where this can't be easily tested + pass + + def test_empty_input_file(self): + """Test handling of completely empty input file""" + input_file = self.create_temp_file("", '.csv') + descriptor_file = self.create_temp_file(json.dumps(self.valid_descriptor), '.json') + + try: + result = load_structured_data( + api_url=self.api_url, + input_file=input_file, + descriptor_file=descriptor_file, + dry_run=True + ) + # Should handle gracefully, possibly with warning + + finally: + self.cleanup_temp_file(input_file) + self.cleanup_temp_file(descriptor_file) + + # Descriptor Format Error Tests + def test_invalid_json_descriptor(self): + """Test handling of invalid JSON in descriptor file""" + input_file = self.create_temp_file("name,email\nJohn,john@email.com", '.csv') + descriptor_file = self.create_temp_file('{"invalid": json}', '.json') # Invalid JSON + + try: + with pytest.raises(json.JSONDecodeError): + load_structured_data( + api_url=self.api_url, + input_file=input_file, + descriptor_file=descriptor_file + ) + finally: + self.cleanup_temp_file(input_file) + self.cleanup_temp_file(descriptor_file) + + def test_missing_required_descriptor_fields(self): + """Test handling of descriptor missing required fields""" + incomplete_descriptor = {"version": "1.0"} # Missing format, mappings, output + + input_file = self.create_temp_file("name,email\nJohn,john@email.com", '.csv') + descriptor_file = self.create_temp_file(json.dumps(incomplete_descriptor), '.json') + + try: + with pytest.raises(KeyError): + load_structured_data( + api_url=self.api_url, + input_file=input_file, + descriptor_file=descriptor_file + ) + finally: + self.cleanup_temp_file(input_file) + self.cleanup_temp_file(descriptor_file) + + def test_invalid_format_type(self): + """Test handling of invalid format type in descriptor""" + invalid_descriptor = { + **self.valid_descriptor, + "format": {"type": "unsupported_format", "encoding": "utf-8"} + } + + input_file = self.create_temp_file("name,email\nJohn,john@email.com", '.csv') + descriptor_file = self.create_temp_file(json.dumps(invalid_descriptor), '.json') + + try: + with pytest.raises(ValueError): + load_structured_data( + api_url=self.api_url, + input_file=input_file, + descriptor_file=descriptor_file + ) + finally: + self.cleanup_temp_file(input_file) + self.cleanup_temp_file(descriptor_file) + + # Data Parsing Error Tests + def test_malformed_csv_data(self): + """Test handling of malformed CSV data""" + malformed_csv = '''name,email,age +John Smith,john@email.com,35 +Jane "unclosed quote,jane@email.com,28 +Bob,bob@email.com,"age with quote,42''' + + format_info = {"type": "csv", "encoding": "utf-8", "options": {"header": True, "delimiter": ","}} + + # Should handle parsing errors gracefully + try: + records = parse_csv_data(malformed_csv, format_info) + # May return partial results or raise exception + except Exception as e: + # Exception is expected for malformed CSV + assert isinstance(e, (csv.Error, ValueError)) + + def test_csv_wrong_delimiter(self): + """Test CSV with wrong delimiter configuration""" + csv_data = "name;email;age\nJohn Smith;john@email.com;35\nJane Doe;jane@email.com;28" + format_info = {"type": "csv", "encoding": "utf-8", "options": {"header": True, "delimiter": ","}} # Wrong delimiter + + records = parse_csv_data(csv_data, format_info) + + # Should still parse but data will be in wrong format + assert len(records) == 2 + # The entire row will be in the first field due to wrong delimiter + assert "John Smith;john@email.com;35" in records[0].values() + + def test_malformed_json_data(self): + """Test handling of malformed JSON data""" + malformed_json = '{"name": "John", "age": 35, "email": }' # Missing value + format_info = {"type": "json", "encoding": "utf-8"} + + with pytest.raises(json.JSONDecodeError): + parse_json_data(malformed_json, format_info) + + def test_json_wrong_structure(self): + """Test JSON with unexpected structure""" + wrong_json = '{"not_an_array": "single_object"}' + format_info = {"type": "json", "encoding": "utf-8"} + + with pytest.raises((ValueError, TypeError)): + parse_json_data(wrong_json, format_info) + + def test_malformed_xml_data(self): + """Test handling of malformed XML data""" + malformed_xml = ''' + + + John + + +''' + + format_info = {"type": "xml", "encoding": "utf-8", "options": {"record_path": "//record"}} + + with pytest.raises(Exception): # XML parsing error + parse_xml_data(malformed_xml, format_info) + + def test_xml_invalid_xpath(self): + """Test XML with invalid XPath expression""" + xml_data = ''' + + John +''' + + format_info = { + "type": "xml", + "encoding": "utf-8", + "options": {"record_path": "//[invalid xpath syntax"} + } + + with pytest.raises(Exception): + parse_xml_data(xml_data, format_info) + + # Transformation Error Tests + def test_invalid_transformation_type(self): + """Test handling of invalid transformation types""" + record = {"age": "35", "name": "John"} + mappings = [ + { + "source_field": "age", + "target_field": "age", + "transforms": [{"type": "invalid_transform"}] # Invalid transform type + } + ] + + # Should handle gracefully, possibly ignoring invalid transforms + result = apply_transformations(record, mappings) + assert "age" in result + + def test_type_conversion_errors(self): + """Test handling of type conversion errors""" + record = {"age": "not_a_number", "price": "invalid_float", "active": "not_boolean"} + mappings = [ + {"source_field": "age", "target_field": "age", "transforms": [{"type": "to_int"}]}, + {"source_field": "price", "target_field": "price", "transforms": [{"type": "to_float"}]}, + {"source_field": "active", "target_field": "active", "transforms": [{"type": "to_bool"}]} + ] + + # Should handle conversion errors gracefully + result = apply_transformations(record, mappings) + + # Should still have the fields, possibly with original or default values + assert "age" in result + assert "price" in result + assert "active" in result + + def test_missing_source_fields(self): + """Test handling of mappings referencing missing source fields""" + record = {"name": "John", "email": "john@email.com"} # Missing 'age' field + mappings = [ + {"source_field": "name", "target_field": "name", "transforms": []}, + {"source_field": "age", "target_field": "age", "transforms": []}, # Missing field + {"source_field": "nonexistent", "target_field": "other", "transforms": []} # Also missing + ] + + result = apply_transformations(record, mappings) + + # Should include existing fields + assert result["name"] == "John" + # Missing fields should be handled (possibly skipped or empty) + # The exact behavior depends on implementation + + # Network and API Error Tests + @patch('trustgraph.cli.load_structured_data.TrustGraphAPI') + def test_api_connection_failure(self, mock_api_class): + """Test handling of API connection failures""" + mock_api_class.side_effect = ConnectionError("Cannot connect to API") + + input_file = self.create_temp_file("name,email\nJohn,john@email.com", '.csv') + + try: + with pytest.raises(ConnectionError): + load_structured_data( + api_url=self.api_url, + input_file=input_file, + suggest_schema=True + ) + finally: + self.cleanup_temp_file(input_file) + + def test_websocket_connection_failure(self): + """Test WebSocket connection failure handling""" + input_file = self.create_temp_file("name,email\nJohn,john@email.com", '.csv') + descriptor_file = self.create_temp_file(json.dumps(self.valid_descriptor), '.json') + + try: + # Test with invalid URL + with pytest.raises(Exception): + load_structured_data( + api_url="http://invalid-host:9999", + input_file=input_file, + descriptor_file=descriptor_file, + batch_size=1, + flow='obj-ex' + ) + finally: + self.cleanup_temp_file(input_file) + self.cleanup_temp_file(descriptor_file) + + # Edge Case Data Tests + def test_extremely_long_lines(self): + """Test handling of extremely long data lines""" + # Create CSV with very long line + long_description = "A" * 10000 # 10K character string + csv_data = f"name,description\nJohn,{long_description}\nJane,Short description" + + format_info = {"type": "csv", "encoding": "utf-8", "options": {"header": True}} + + records = parse_csv_data(csv_data, format_info) + + assert len(records) == 2 + assert records[0]["description"] == long_description + assert records[1]["name"] == "Jane" + + def test_special_characters_handling(self): + """Test handling of special characters""" + special_csv = '''name,description,notes +"John O'Connor","Senior Developer, Team Lead","Works on UI/UX & backend" +"María García","Data Scientist","Specializes in NLP & ML" +"张三","Software Engineer","Focuses on 中文 processing"''' + + format_info = {"type": "csv", "encoding": "utf-8", "options": {"header": True}} + + records = parse_csv_data(special_csv, format_info) + + assert len(records) == 3 + assert records[0]["name"] == "John O'Connor" + assert records[1]["name"] == "María García" + assert records[2]["name"] == "张三" + + def test_unicode_and_encoding_issues(self): + """Test handling of Unicode and encoding issues""" + # This test would need specific encoding scenarios + unicode_data = "name,city\nJohn,München\nJane,Zürich\nBob,Kraków" + + format_info = {"type": "csv", "encoding": "utf-8", "options": {"header": True}} + + records = parse_csv_data(unicode_data, format_info) + + assert len(records) == 3 + assert records[0]["city"] == "München" + assert records[2]["city"] == "Kraków" + + def test_null_and_empty_values(self): + """Test handling of null and empty values""" + csv_with_nulls = '''name,email,age,notes +John,john@email.com,35, +Jane,,28,Some notes +,missing@email.com,, +Bob,bob@email.com,42,''' + + format_info = {"type": "csv", "encoding": "utf-8", "options": {"header": True}} + + records = parse_csv_data(csv_with_nulls, format_info) + + assert len(records) == 4 + # Check empty values are handled + assert records[0]["notes"] == "" + assert records[1]["email"] == "" + assert records[2]["name"] == "" + assert records[2]["age"] == "" + + def test_extremely_large_dataset(self): + """Test handling of extremely large datasets""" + # Generate large CSV + num_records = 10000 + large_csv_lines = ["name,email,age"] + + for i in range(num_records): + large_csv_lines.append(f"User{i},user{i}@example.com,{25 + i % 50}") + + large_csv = "\n".join(large_csv_lines) + + format_info = {"type": "csv", "encoding": "utf-8", "options": {"header": True}} + + # This should not crash due to memory issues + records = parse_csv_data(large_csv, format_info) + + assert len(records) == num_records + assert records[0]["name"] == "User0" + assert records[-1]["name"] == f"User{num_records-1}" + + # Batch Processing Edge Cases + def test_batch_size_edge_cases(self): + """Test edge cases in batch size handling""" + records = [{"id": str(i), "name": f"User{i}"} for i in range(10)] + + # Test batch size larger than data + batch_size = 20 + batches = [] + for i in range(0, len(records), batch_size): + batch_records = records[i:i + batch_size] + batches.append(batch_records) + + assert len(batches) == 1 + assert len(batches[0]) == 10 + + # Test batch size of 1 + batch_size = 1 + batches = [] + for i in range(0, len(records), batch_size): + batch_records = records[i:i + batch_size] + batches.append(batch_records) + + assert len(batches) == 10 + assert all(len(batch) == 1 for batch in batches) + + def test_zero_batch_size(self): + """Test handling of zero or invalid batch size""" + input_file = self.create_temp_file("name\nJohn\nJane", '.csv') + descriptor_file = self.create_temp_file(json.dumps(self.valid_descriptor), '.json') + + try: + # Should handle invalid batch size gracefully + with pytest.raises(ValueError): + load_structured_data( + api_url=self.api_url, + input_file=input_file, + descriptor_file=descriptor_file, + batch_size=0, # Invalid batch size + dry_run=True + ) + finally: + self.cleanup_temp_file(input_file) + self.cleanup_temp_file(descriptor_file) + + # Memory and Performance Edge Cases + def test_memory_efficient_processing(self): + """Test that processing doesn't consume excessive memory""" + # This would be a performance test to ensure memory efficiency + # For unit testing, we just verify it doesn't crash + pass + + def test_concurrent_access_safety(self): + """Test handling of concurrent access to temp files""" + # This would test file locking and concurrent access scenarios + pass + + # Output File Error Tests + def test_output_file_permission_error(self): + """Test handling of output file permission errors""" + input_file = self.create_temp_file("name\nJohn", '.csv') + descriptor_file = self.create_temp_file(json.dumps(self.valid_descriptor), '.json') + + try: + # Try to write to invalid location + with pytest.raises((PermissionError, OSError)): + load_structured_data( + api_url=self.api_url, + input_file=input_file, + descriptor_file=descriptor_file, + parse_only=True, + output_file="/root/forbidden.json" # Should fail on most systems + ) + except Exception: + # Different systems may handle this differently + pass + finally: + self.cleanup_temp_file(input_file) + self.cleanup_temp_file(descriptor_file) + + # Configuration Edge Cases + def test_invalid_flow_parameter(self): + """Test handling of invalid flow parameter""" + input_file = self.create_temp_file("name\nJohn", '.csv') + descriptor_file = self.create_temp_file(json.dumps(self.valid_descriptor), '.json') + + try: + # Invalid flow should be handled gracefully (may just use as-is) + result = load_structured_data( + api_url=self.api_url, + input_file=input_file, + descriptor_file=descriptor_file, + flow="", # Empty flow + dry_run=True + ) + finally: + self.cleanup_temp_file(input_file) + self.cleanup_temp_file(descriptor_file) + + def test_conflicting_parameters(self): + """Test handling of conflicting command line parameters""" + input_file = self.create_temp_file("name\nJohn", '.csv') + + try: + # Should handle conflicting modes gracefully or raise appropriate error + with pytest.raises(ValueError): + load_structured_data( + api_url=self.api_url, + input_file=input_file, + suggest_schema=True, + generate_descriptor=True, # Conflicting with suggest_schema + parse_only=True # Also conflicting + ) + finally: + self.cleanup_temp_file(input_file) \ No newline at end of file diff --git a/tests/unit/test_cli/test_schema_descriptor_generation.py b/tests/unit/test_cli/test_schema_descriptor_generation.py new file mode 100644 index 00000000..6ae63f3b --- /dev/null +++ b/tests/unit/test_cli/test_schema_descriptor_generation.py @@ -0,0 +1,707 @@ +""" +Unit tests for schema suggestion and descriptor generation functionality in tg-load-structured-data. +Tests the --suggest-schema and --generate-descriptor modes. +""" + +import pytest +import json +import tempfile +import os +from unittest.mock import Mock, patch, MagicMock + +from trustgraph.cli.load_structured_data import load_structured_data + + +class TestSchemaDescriptorGeneration: + """Tests for schema suggestion and descriptor generation""" + + def setup_method(self): + """Set up test fixtures""" + self.api_url = "http://localhost:8088" + + # Sample data for different formats + self.customer_csv = """name,email,age,country,registration_date,status +John Smith,john@email.com,35,USA,2024-01-15,active +Jane Doe,jane@email.com,28,Canada,2024-01-20,active +Bob Johnson,bob@company.org,42,UK,2024-01-10,inactive""" + + self.product_json = [ + { + "id": "PROD001", + "name": "Wireless Headphones", + "category": "Electronics", + "price": 99.99, + "in_stock": True, + "specifications": { + "battery_life": "24 hours", + "wireless": True, + "noise_cancellation": True + } + }, + { + "id": "PROD002", + "name": "Coffee Maker", + "category": "Home & Kitchen", + "price": 129.99, + "in_stock": False, + "specifications": { + "capacity": "12 cups", + "programmable": True, + "auto_shutoff": True + } + } + ] + + self.trade_xml = """ + + + + USA + Wheat + 1000000 + 250000000 + export + + + China + Electronics + 500000 + 750000000 + import + + +""" + + # Mock schema definitions + self.mock_schemas = { + "customer": json.dumps({ + "name": "customer", + "description": "Customer information records", + "fields": [ + {"name": "name", "type": "string", "required": True}, + {"name": "email", "type": "string", "required": True}, + {"name": "age", "type": "integer"}, + {"name": "country", "type": "string"}, + {"name": "status", "type": "string"} + ] + }), + "product": json.dumps({ + "name": "product", + "description": "Product catalog information", + "fields": [ + {"name": "id", "type": "string", "required": True, "primary_key": True}, + {"name": "name", "type": "string", "required": True}, + {"name": "category", "type": "string"}, + {"name": "price", "type": "float"}, + {"name": "in_stock", "type": "boolean"} + ] + }), + "trade_data": json.dumps({ + "name": "trade_data", + "description": "International trade statistics", + "fields": [ + {"name": "country", "type": "string", "required": True}, + {"name": "product", "type": "string", "required": True}, + {"name": "quantity", "type": "integer"}, + {"name": "value_usd", "type": "float"}, + {"name": "trade_type", "type": "string"} + ] + }), + "financial_record": json.dumps({ + "name": "financial_record", + "description": "Financial transaction records", + "fields": [ + {"name": "transaction_id", "type": "string", "primary_key": True}, + {"name": "amount", "type": "float", "required": True}, + {"name": "currency", "type": "string"}, + {"name": "date", "type": "timestamp"} + ] + }) + } + + def create_temp_file(self, content, suffix='.txt'): + """Create a temporary file with given content""" + temp_file = tempfile.NamedTemporaryFile(mode='w', suffix=suffix, delete=False) + temp_file.write(content) + temp_file.flush() + temp_file.close() + return temp_file.name + + def cleanup_temp_file(self, file_path): + """Clean up temporary file""" + try: + os.unlink(file_path) + except: + pass + + # Schema Suggestion Tests + @patch('trustgraph.cli.load_structured_data.TrustGraphAPI') + def test_suggest_schema_csv_data(self, mock_api_class): + """Test schema suggestion for CSV data""" + # Setup mock API + mock_api = Mock() + mock_api_class.return_value = mock_api + mock_config_api = Mock() + mock_api.config.return_value = mock_config_api + mock_config_api.get_config_items.return_value = {"schema": self.mock_schemas} + + mock_flow = Mock() + mock_api.flow.return_value = mock_flow + mock_flow.id.return_value = mock_flow + mock_prompt_client = Mock() + mock_flow.prompt.return_value = mock_prompt_client + + # Mock schema selection response + mock_prompt_client.schema_selection.return_value = ( + "Based on the data containing customer names, emails, ages, and countries, " + "the **customer** schema is the most appropriate choice. This schema includes " + "all the necessary fields for customer information and aligns well with the " + "structure of your data." + ) + + input_file = self.create_temp_file(self.customer_csv, '.csv') + + try: + result = load_structured_data( + api_url=self.api_url, + input_file=input_file, + suggest_schema=True, + sample_size=100, + sample_chars=500 + ) + + # Verify API calls were made correctly + mock_config_api.get_config_items.assert_called_once() + mock_prompt_client.schema_selection.assert_called_once() + + # Check arguments passed to schema_selection + call_args = mock_prompt_client.schema_selection.call_args + assert 'schemas' in call_args.kwargs + assert 'sample' in call_args.kwargs + + # Verify schemas were passed correctly + passed_schemas = call_args.kwargs['schemas'] + assert len(passed_schemas) == len(self.mock_schemas) + + # Check sample data was included + sample_data = call_args.kwargs['sample'] + assert 'John Smith' in sample_data + assert 'jane@email.com' in sample_data + + finally: + self.cleanup_temp_file(input_file) + + @patch('trustgraph.cli.load_structured_data.TrustGraphAPI') + def test_suggest_schema_json_data(self, mock_api_class): + """Test schema suggestion for JSON data""" + mock_api = Mock() + mock_api_class.return_value = mock_api + mock_config_api = Mock() + mock_api.config.return_value = mock_config_api + mock_config_api.get_config_items.return_value = {"schema": self.mock_schemas} + + mock_flow = Mock() + mock_api.flow.return_value = mock_flow + mock_flow.id.return_value = mock_flow + mock_prompt_client = Mock() + mock_flow.prompt.return_value = mock_prompt_client + + mock_prompt_client.schema_selection.return_value = ( + "The **product** schema is ideal for this dataset containing product IDs, " + "names, categories, prices, and stock status. This matches perfectly with " + "the product schema structure." + ) + + input_file = self.create_temp_file(json.dumps(self.product_json), '.json') + + try: + result = load_structured_data( + api_url=self.api_url, + input_file=input_file, + suggest_schema=True, + sample_chars=1000 + ) + + # Verify the call was made + mock_prompt_client.schema_selection.assert_called_once() + + # Check that JSON data was properly sampled + call_args = mock_prompt_client.schema_selection.call_args + sample_data = call_args.kwargs['sample'] + assert 'PROD001' in sample_data + assert 'Wireless Headphones' in sample_data + assert 'Electronics' in sample_data + + finally: + self.cleanup_temp_file(input_file) + + @patch('trustgraph.cli.load_structured_data.TrustGraphAPI') + def test_suggest_schema_xml_data(self, mock_api_class): + """Test schema suggestion for XML data""" + mock_api = Mock() + mock_api_class.return_value = mock_api + mock_config_api = Mock() + mock_api.config.return_value = mock_config_api + mock_config_api.get_config_items.return_value = {"schema": self.mock_schemas} + + mock_flow = Mock() + mock_api.flow.return_value = mock_flow + mock_flow.id.return_value = mock_flow + mock_prompt_client = Mock() + mock_flow.prompt.return_value = mock_prompt_client + + mock_prompt_client.schema_selection.return_value = ( + "The **trade_data** schema is the best fit for this XML data containing " + "country, product, quantity, value, and trade type information. This aligns " + "perfectly with international trade statistics." + ) + + input_file = self.create_temp_file(self.trade_xml, '.xml') + + try: + result = load_structured_data( + api_url=self.api_url, + input_file=input_file, + suggest_schema=True, + sample_chars=800 + ) + + mock_prompt_client.schema_selection.assert_called_once() + + # Verify XML content was included in sample + call_args = mock_prompt_client.schema_selection.call_args + sample_data = call_args.kwargs['sample'] + assert 'field name="country"' in sample_data or 'country' in sample_data + assert 'USA' in sample_data + assert 'export' in sample_data + + finally: + self.cleanup_temp_file(input_file) + + @patch('trustgraph.cli.load_structured_data.TrustGraphAPI') + def test_suggest_schema_sample_size_limiting(self, mock_api_class): + """Test that sample size is properly limited""" + mock_api = Mock() + mock_api_class.return_value = mock_api + mock_config_api = Mock() + mock_api.config.return_value = mock_config_api + mock_config_api.get_config_items.return_value = {"schema": self.mock_schemas} + + mock_flow = Mock() + mock_api.flow.return_value = mock_flow + mock_flow.id.return_value = mock_flow + mock_prompt_client = Mock() + mock_flow.prompt.return_value = mock_prompt_client + mock_prompt_client.schema_selection.return_value = "customer schema recommended" + + # Create large CSV file + large_csv = "name,email,age\n" + "\n".join([f"User{i},user{i}@example.com,{20+i}" for i in range(1000)]) + input_file = self.create_temp_file(large_csv, '.csv') + + try: + result = load_structured_data( + api_url=self.api_url, + input_file=input_file, + suggest_schema=True, + sample_size=10, # Limit to 10 records + sample_chars=200 # Limit to 200 characters + ) + + # Check that sample was limited + call_args = mock_prompt_client.schema_selection.call_args + sample_data = call_args.kwargs['sample'] + + # Should be limited by sample_chars + assert len(sample_data) <= 250 # Some margin for formatting + + # Should not contain all 1000 users + user_count = sample_data.count('User') + assert user_count < 20 # Much less than 1000 + + finally: + self.cleanup_temp_file(input_file) + + # Descriptor Generation Tests + @patch('trustgraph.cli.load_structured_data.TrustGraphAPI') + def test_generate_descriptor_csv_format(self, mock_api_class): + """Test descriptor generation for CSV format""" + mock_api = Mock() + mock_api_class.return_value = mock_api + mock_config_api = Mock() + mock_api.config.return_value = mock_config_api + mock_config_api.get_config_items.return_value = {"schema": self.mock_schemas} + + mock_flow = Mock() + mock_api.flow.return_value = mock_flow + mock_flow.id.return_value = mock_flow + mock_prompt_client = Mock() + mock_flow.prompt.return_value = mock_prompt_client + + # Mock descriptor generation response + generated_descriptor = { + "version": "1.0", + "metadata": { + "name": "CustomerDataImport", + "description": "Import customer data from CSV", + "author": "TrustGraph" + }, + "format": { + "type": "csv", + "encoding": "utf-8", + "options": { + "header": True, + "delimiter": "," + } + }, + "mappings": [ + { + "source_field": "name", + "target_field": "name", + "transforms": [{"type": "trim"}], + "validation": [{"type": "required"}] + }, + { + "source_field": "email", + "target_field": "email", + "transforms": [{"type": "trim"}, {"type": "lower"}], + "validation": [{"type": "required"}] + }, + { + "source_field": "age", + "target_field": "age", + "transforms": [{"type": "to_int"}], + "validation": [{"type": "required"}] + } + ], + "output": { + "format": "trustgraph-objects", + "schema_name": "customer", + "options": { + "confidence": 0.85, + "batch_size": 100 + } + } + } + + mock_prompt_client.diagnose_structured_data.return_value = json.dumps(generated_descriptor) + + input_file = self.create_temp_file(self.customer_csv, '.csv') + + try: + result = load_structured_data( + api_url=self.api_url, + input_file=input_file, + generate_descriptor=True, + sample_chars=1000 + ) + + # Verify API calls + mock_prompt_client.diagnose_structured_data.assert_called_once() + + # Check call arguments + call_args = mock_prompt_client.diagnose_structured_data.call_args + assert 'schemas' in call_args.kwargs + assert 'sample' in call_args.kwargs + + # Verify CSV data was included + sample_data = call_args.kwargs['sample'] + assert 'name,email,age,country' in sample_data # Header + assert 'John Smith' in sample_data + + # Verify schemas were passed + passed_schemas = call_args.kwargs['schemas'] + assert len(passed_schemas) > 0 + + finally: + self.cleanup_temp_file(input_file) + + @patch('trustgraph.cli.load_structured_data.TrustGraphAPI') + def test_generate_descriptor_json_format(self, mock_api_class): + """Test descriptor generation for JSON format""" + mock_api = Mock() + mock_api_class.return_value = mock_api + mock_config_api = Mock() + mock_api.config.return_value = mock_config_api + mock_config_api.get_config_items.return_value = {"schema": self.mock_schemas} + + mock_flow = Mock() + mock_api.flow.return_value = mock_flow + mock_flow.id.return_value = mock_flow + mock_prompt_client = Mock() + mock_flow.prompt.return_value = mock_prompt_client + + generated_descriptor = { + "version": "1.0", + "format": { + "type": "json", + "encoding": "utf-8" + }, + "mappings": [ + { + "source_field": "id", + "target_field": "product_id", + "transforms": [{"type": "trim"}], + "validation": [{"type": "required"}] + }, + { + "source_field": "name", + "target_field": "product_name", + "transforms": [{"type": "trim"}], + "validation": [{"type": "required"}] + }, + { + "source_field": "price", + "target_field": "price", + "transforms": [{"type": "to_float"}], + "validation": [] + } + ], + "output": { + "format": "trustgraph-objects", + "schema_name": "product", + "options": {"confidence": 0.9, "batch_size": 50} + } + } + + mock_prompt_client.diagnose_structured_data.return_value = json.dumps(generated_descriptor) + + input_file = self.create_temp_file(json.dumps(self.product_json), '.json') + + try: + result = load_structured_data( + api_url=self.api_url, + input_file=input_file, + generate_descriptor=True + ) + + mock_prompt_client.diagnose_structured_data.assert_called_once() + + # Verify JSON structure was analyzed + call_args = mock_prompt_client.diagnose_structured_data.call_args + sample_data = call_args.kwargs['sample'] + assert 'PROD001' in sample_data + assert 'Wireless Headphones' in sample_data + assert '99.99' in sample_data + + finally: + self.cleanup_temp_file(input_file) + + @patch('trustgraph.cli.load_structured_data.TrustGraphAPI') + def test_generate_descriptor_xml_format(self, mock_api_class): + """Test descriptor generation for XML format""" + mock_api = Mock() + mock_api_class.return_value = mock_api + mock_config_api = Mock() + mock_api.config.return_value = mock_config_api + mock_config_api.get_config_items.return_value = {"schema": self.mock_schemas} + + mock_flow = Mock() + mock_api.flow.return_value = mock_flow + mock_flow.id.return_value = mock_flow + mock_prompt_client = Mock() + mock_flow.prompt.return_value = mock_prompt_client + + # XML descriptor should include XPath configuration + xml_descriptor = { + "version": "1.0", + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "/ROOT/data/record", + "field_attribute": "name" + } + }, + "mappings": [ + { + "source_field": "country", + "target_field": "country", + "transforms": [{"type": "trim"}, {"type": "upper"}], + "validation": [{"type": "required"}] + }, + { + "source_field": "value_usd", + "target_field": "trade_value", + "transforms": [{"type": "to_float"}], + "validation": [] + } + ], + "output": { + "format": "trustgraph-objects", + "schema_name": "trade_data", + "options": {"confidence": 0.8, "batch_size": 25} + } + } + + mock_prompt_client.diagnose_structured_data.return_value = json.dumps(xml_descriptor) + + input_file = self.create_temp_file(self.trade_xml, '.xml') + + try: + result = load_structured_data( + api_url=self.api_url, + input_file=input_file, + generate_descriptor=True + ) + + mock_prompt_client.diagnose_structured_data.assert_called_once() + + # Verify XML structure was included + call_args = mock_prompt_client.diagnose_structured_data.call_args + sample_data = call_args.kwargs['sample'] + assert '' in sample_data + assert 'field name=' in sample_data + assert 'USA' in sample_data + + finally: + self.cleanup_temp_file(input_file) + + # Error Handling Tests + @patch('trustgraph.cli.load_structured_data.TrustGraphAPI') + def test_suggest_schema_no_schemas_available(self, mock_api_class): + """Test schema suggestion when no schemas are available""" + mock_api = Mock() + mock_api_class.return_value = mock_api + mock_config_api = Mock() + mock_api.config.return_value = mock_config_api + mock_config_api.get_config_items.return_value = {"schema": {}} # Empty schemas + + input_file = self.create_temp_file(self.customer_csv, '.csv') + + try: + with pytest.raises(ValueError) as exc_info: + result = load_structured_data( + api_url=self.api_url, + input_file=input_file, + suggest_schema=True + ) + + assert "no schemas" in str(exc_info.value).lower() + + finally: + self.cleanup_temp_file(input_file) + + @patch('trustgraph.cli.load_structured_data.TrustGraphAPI') + def test_generate_descriptor_api_error(self, mock_api_class): + """Test descriptor generation when API returns error""" + mock_api = Mock() + mock_api_class.return_value = mock_api + mock_config_api = Mock() + mock_api.config.return_value = mock_config_api + mock_config_api.get_config_items.return_value = {"schema": self.mock_schemas} + + mock_flow = Mock() + mock_api.flow.return_value = mock_flow + mock_flow.id.return_value = mock_flow + mock_prompt_client = Mock() + mock_flow.prompt.return_value = mock_prompt_client + + # Mock API error + mock_prompt_client.diagnose_structured_data.side_effect = Exception("API connection failed") + + input_file = self.create_temp_file(self.customer_csv, '.csv') + + try: + with pytest.raises(Exception) as exc_info: + result = load_structured_data( + api_url=self.api_url, + input_file=input_file, + generate_descriptor=True + ) + + assert "API connection failed" in str(exc_info.value) + + finally: + self.cleanup_temp_file(input_file) + + @patch('trustgraph.cli.load_structured_data.TrustGraphAPI') + def test_generate_descriptor_invalid_response(self, mock_api_class): + """Test descriptor generation with invalid API response""" + mock_api = Mock() + mock_api_class.return_value = mock_api + mock_config_api = Mock() + mock_api.config.return_value = mock_config_api + mock_config_api.get_config_items.return_value = {"schema": self.mock_schemas} + + mock_flow = Mock() + mock_api.flow.return_value = mock_flow + mock_flow.id.return_value = mock_flow + mock_prompt_client = Mock() + mock_flow.prompt.return_value = mock_prompt_client + + # Return invalid JSON + mock_prompt_client.diagnose_structured_data.return_value = "invalid json response" + + input_file = self.create_temp_file(self.customer_csv, '.csv') + + try: + with pytest.raises(json.JSONDecodeError): + result = load_structured_data( + api_url=self.api_url, + input_file=input_file, + generate_descriptor=True + ) + + finally: + self.cleanup_temp_file(input_file) + + # Output Format Tests + def test_suggest_schema_output_format(self): + """Test that schema suggestion produces proper output format""" + # This would be tested with actual TrustGraph instance + # Here we verify the expected behavior structure + pass + + def test_generate_descriptor_output_to_file(self): + """Test descriptor generation with file output""" + # Test would verify descriptor is written to specified file + pass + + # Sample Data Quality Tests + @patch('trustgraph.cli.load_structured_data.TrustGraphAPI') + def test_sample_data_quality_csv(self, mock_api_class): + """Test that sample data quality is maintained for CSV""" + mock_api = Mock() + mock_api_class.return_value = mock_api + mock_config_api = Mock() + mock_api.config.return_value = mock_config_api + mock_config_api.get_config_items.return_value = {"schema": self.mock_schemas} + + mock_flow = Mock() + mock_api.flow.return_value = mock_flow + mock_flow.id.return_value = mock_flow + mock_prompt_client = Mock() + mock_flow.prompt.return_value = mock_prompt_client + mock_prompt_client.schema_selection.return_value = "customer schema recommended" + + # CSV with various data types and edge cases + complex_csv = """name,email,age,salary,join_date,is_active,notes +John O'Connor,"john@company.com",35,75000.50,2024-01-15,true,"Senior Developer, Team Lead" +Jane "Smith" Doe,jane@email.com,28,65000,2024-02-01,true,"Data Scientist, ML Expert" +Bob,bob@temp.org,42,,2023-12-01,false,"Contractor, Part-time" +,missing@email.com,25,45000,2024-03-01,true,"Junior Developer, New Hire" """ + + input_file = self.create_temp_file(complex_csv, '.csv') + + try: + result = load_structured_data( + api_url=self.api_url, + input_file=input_file, + suggest_schema=True, + sample_chars=1000 + ) + + # Check that sample preserves important characteristics + call_args = mock_prompt_client.schema_selection.call_args + sample_data = call_args.kwargs['sample'] + + # Should preserve header + assert 'name,email,age,salary' in sample_data + + # Should include examples of data variety + assert "John O'Connor" in sample_data or 'John' in sample_data + assert '@' in sample_data # Email format + assert '75000' in sample_data or '65000' in sample_data # Numeric data + + finally: + self.cleanup_temp_file(input_file) \ No newline at end of file diff --git a/tests/unit/test_cli/test_xml_xpath_parsing.py b/tests/unit/test_cli/test_xml_xpath_parsing.py new file mode 100644 index 00000000..e56f7369 --- /dev/null +++ b/tests/unit/test_cli/test_xml_xpath_parsing.py @@ -0,0 +1,626 @@ +""" +Specialized unit tests for XML parsing and XPath functionality in tg-load-structured-data. +Tests complex XML structures, XPath expressions, and field attribute handling. +""" + +import pytest +import json +import tempfile +import os +import xml.etree.ElementTree as ET + +from trustgraph.cli.load_structured_data import load_structured_data + + +class TestXMLXPathParsing: + """Specialized tests for XML parsing with XPath support""" + + def setup_method(self): + """Set up test fixtures""" + # UN Trade Data format (real-world complex XML) + self.un_trade_xml = """ + + + + Albania + 2024 + Coffee; not roasted or decaffeinated + import + 24445532.903 + 5305568.05 + + + Algeria + 2024 + Tea + export + 12345678.90 + 2500000.00 + + +""" + + # Standard XML with attributes + self.product_xml = """ + + + Laptop + 999.99 + High-performance laptop + + Intel i7 + 16GB + 512GB SSD + + + + Python Programming + 49.99 + Learn Python programming + + 500 + English + Paperback + + +""" + + # Nested XML structure + self.nested_xml = """ + + + + John Smith + john@email.com +
+ 123 Main St + New York + USA +
+
+ + + Widget A + 19.99 + + + Widget B + 29.99 + + +
+
""" + + # XML with mixed content and namespaces + self.namespace_xml = """ + + + + Smartphone + 599.99 + + + Tablet + 399.99 + + +""" + + def create_temp_file(self, content, suffix='.txt'): + """Create a temporary file with given content""" + temp_file = tempfile.NamedTemporaryFile(mode='w', suffix=suffix, delete=False) + temp_file.write(content) + temp_file.flush() + temp_file.close() + return temp_file.name + + def cleanup_temp_file(self, file_path): + """Clean up temporary file""" + try: + os.unlink(file_path) + except: + pass + + # UN Data Format Tests (CLI-level testing) + def test_un_trade_data_xpath_parsing(self): + """Test parsing UN trade data format with field attributes via CLI""" + descriptor = { + "version": "1.0", + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "/ROOT/data/record", + "field_attribute": "name" + } + }, + "mappings": [ + {"source_field": "country_or_area", "target_field": "country", "transforms": []}, + {"source_field": "commodity", "target_field": "product", "transforms": []}, + {"source_field": "trade_usd", "target_field": "value", "transforms": []} + ], + "output": { + "format": "trustgraph-objects", + "schema_name": "trade_data", + "options": {"confidence": 0.9, "batch_size": 10} + } + } + + input_file = self.create_temp_file(self.un_trade_xml, '.xml') + descriptor_file = self.create_temp_file(json.dumps(descriptor), '.json') + output_file = tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) + output_file.close() + + try: + # Test parse-only mode to verify XML parsing works + load_structured_data( + api_url="http://localhost:8088", + input_file=input_file, + descriptor_file=descriptor_file, + parse_only=True, + output_file=output_file.name + ) + + # Verify parsing worked + assert os.path.exists(output_file.name) + with open(output_file.name, 'r') as f: + parsed_data = json.load(f) + assert len(parsed_data) == 2 + assert parsed_data[0]["country"] == "Albania" + assert parsed_data[0]["product"] == "Coffee; not roasted or decaffeinated" + assert parsed_data[1]["country"] == "Algeria" + + finally: + self.cleanup_temp_file(input_file) + self.cleanup_temp_file(descriptor_file) + self.cleanup_temp_file(output_file.name) + + def test_xpath_record_path_variations(self): + """Test different XPath record path expressions""" + # Test with leading slash + format_info_1 = { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "/ROOT/data/record", + "field_attribute": "name" + } + } + + records_1 = parse_xml_data(self.un_trade_xml, format_info_1) + assert len(records_1) == 2 + + # Test with double slash (descendant) + format_info_2 = { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//record", + "field_attribute": "name" + } + } + + records_2 = parse_xml_data(self.un_trade_xml, format_info_2) + assert len(records_2) == 2 + + # Results should be the same + assert records_1[0]["country_or_area"] == records_2[0]["country_or_area"] + + def test_field_attribute_parsing(self): + """Test field attribute parsing mechanism""" + format_info = { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "/ROOT/data/record", + "field_attribute": "name" + } + } + + records = parse_xml_data(self.un_trade_xml, format_info) + + # Should extract all fields defined by 'name' attribute + expected_fields = ["country_or_area", "year", "commodity", "flow", "trade_usd", "weight_kg"] + + for record in records: + for field in expected_fields: + assert field in record, f"Field {field} should be extracted from XML" + assert record[field], f"Field {field} should have a value" + + # Standard XML Structure Tests + def test_standard_xml_with_attributes(self): + """Test parsing standard XML with element attributes""" + format_info = { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//product" + } + } + + records = parse_xml_data(self.product_xml, format_info) + + assert len(records) == 2 + + # Check attributes are captured + first_product = records[0] + assert first_product["id"] == "1" + assert first_product["category"] == "electronics" + assert first_product["name"] == "Laptop" + assert first_product["price"] == "999.99" + + second_product = records[1] + assert second_product["id"] == "2" + assert second_product["category"] == "books" + assert second_product["name"] == "Python Programming" + + def test_nested_xml_structure_parsing(self): + """Test parsing deeply nested XML structures""" + # Test extracting order-level data + format_info = { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//order" + } + } + + records = parse_xml_data(self.nested_xml, format_info) + + assert len(records) == 1 + + order = records[0] + assert order["order_id"] == "ORD001" + assert order["date"] == "2024-01-15" + # Nested elements should be flattened + assert "name" in order # Customer name + assert order["name"] == "John Smith" + + def test_nested_item_extraction(self): + """Test extracting items from nested XML""" + # Test extracting individual items + format_info = { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//item" + } + } + + records = parse_xml_data(self.nested_xml, format_info) + + assert len(records) == 2 + + first_item = records[0] + assert first_item["sku"] == "ITEM001" + assert first_item["quantity"] == "2" + assert first_item["name"] == "Widget A" + assert first_item["price"] == "19.99" + + second_item = records[1] + assert second_item["sku"] == "ITEM002" + assert second_item["quantity"] == "1" + assert second_item["name"] == "Widget B" + + # Complex XPath Expression Tests + def test_complex_xpath_expressions(self): + """Test complex XPath expressions""" + # Test with predicate - only electronics products + electronics_xml = """ + + + Laptop + 999.99 + + + Novel + 19.99 + + + Phone + 599.99 + +""" + + # XPath with attribute filter + format_info = { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//product[@category='electronics']" + } + } + + records = parse_xml_data(electronics_xml, format_info) + + # Should only get electronics products + assert len(records) == 2 + assert records[0]["name"] == "Laptop" + assert records[1]["name"] == "Phone" + + # Both should have electronics category + for record in records: + assert record["category"] == "electronics" + + def test_xpath_with_position(self): + """Test XPath expressions with position predicates""" + format_info = { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//product[1]" # First product only + } + } + + records = parse_xml_data(self.product_xml, format_info) + + # Should only get first product + assert len(records) == 1 + assert records[0]["name"] == "Laptop" + assert records[0]["id"] == "1" + + # Namespace Handling Tests + def test_xml_with_namespaces(self): + """Test XML parsing with namespaces""" + # Note: ElementTree has limited namespace support in XPath + format_info = { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//{http://example.com/products}item" + } + } + + try: + records = parse_xml_data(self.namespace_xml, format_info) + + # Should find items with namespace + assert len(records) >= 1 + + except Exception: + # ElementTree may not support full namespace XPath + # This is expected behavior - document the limitation + pass + + # Error Handling Tests + def test_invalid_xpath_expression(self): + """Test handling of invalid XPath expressions""" + format_info = { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//[invalid xpath" # Malformed XPath + } + } + + with pytest.raises(Exception): + records = parse_xml_data(self.un_trade_xml, format_info) + + def test_xpath_no_matches(self): + """Test XPath that matches no elements""" + format_info = { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//nonexistent" + } + } + + records = parse_xml_data(self.un_trade_xml, format_info) + + # Should return empty list + assert len(records) == 0 + assert isinstance(records, list) + + def test_malformed_xml_handling(self): + """Test handling of malformed XML""" + malformed_xml = """ + + + value + + +""" + + format_info = { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//record" + } + } + + with pytest.raises(ET.ParseError): + records = parse_xml_data(malformed_xml, format_info) + + # Field Attribute Variations Tests + def test_different_field_attribute_names(self): + """Test different field attribute names""" + custom_xml = """ + + + John + 35 + NYC + +""" + + format_info = { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//record", + "field_attribute": "key" # Using 'key' instead of 'name' + } + } + + records = parse_xml_data(custom_xml, format_info) + + assert len(records) == 1 + record = records[0] + assert record["name"] == "John" + assert record["age"] == "35" + assert record["city"] == "NYC" + + def test_missing_field_attribute(self): + """Test handling when field_attribute is specified but not found""" + xml_without_attributes = """ + + + John + 35 + +""" + + format_info = { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//record", + "field_attribute": "name" # Looking for 'name' attribute but elements don't have it + } + } + + records = parse_xml_data(xml_without_attributes, format_info) + + assert len(records) == 1 + # Should fall back to standard parsing + record = records[0] + assert record["name"] == "John" + assert record["age"] == "35" + + # Mixed Content Tests + def test_xml_with_mixed_content(self): + """Test XML with mixed text and element content""" + mixed_xml = """ + + + John Smith works at ACME Corp in NYC + + + Jane Doe works at Tech Inc in SF + +""" + + format_info = { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//person" + } + } + + records = parse_xml_data(mixed_xml, format_info) + + assert len(records) == 2 + + # Should capture both attributes and child elements + first_person = records[0] + assert first_person["id"] == "1" + assert first_person["company"] == "ACME Corp" + assert first_person["city"] == "NYC" + + # Integration with Transformation Tests + def test_xml_with_transformations(self): + """Test XML parsing with data transformations""" + records = parse_xml_data(self.un_trade_xml, { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "/ROOT/data/record", + "field_attribute": "name" + } + }) + + # Apply transformations + mappings = [ + { + "source_field": "country_or_area", + "target_field": "country", + "transforms": [{"type": "upper"}] + }, + { + "source_field": "trade_usd", + "target_field": "trade_value", + "transforms": [{"type": "to_float"}] + }, + { + "source_field": "year", + "target_field": "year", + "transforms": [{"type": "to_int"}] + } + ] + + transformed_records = [] + for record in records: + transformed = apply_transformations(record, mappings) + transformed_records.append(transformed) + + # Check transformations were applied + first_transformed = transformed_records[0] + assert first_transformed["country"] == "ALBANIA" + assert first_transformed["trade_value"] == "24445532.903" # Converted to string for ExtractedObject + assert first_transformed["year"] == "2024" + + # Real-world Complexity Tests + def test_complex_real_world_xml(self): + """Test with complex real-world XML structure""" + complex_xml = """ + + + 2024-01-15T10:30:00Z + Trade Statistics Database + + + + United States + China + 854232 + Integrated circuits + Import + 202401 + + 15000000.50 + 125000.75 + 120.00 + + + + United States + Germany + 870323 + Motor cars + Import + 202401 + + 5000000.00 + 250 + 20000.00 + + + +""" + + format_info = { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//trade_record" + } + } + + records = parse_xml_data(complex_xml, format_info) + + assert len(records) == 2 + + # Check first record structure + first_record = records[0] + assert first_record["reporting_country"] == "United States" + assert first_record["partner_country"] == "China" + assert first_record["commodity_code"] == "854232" + assert first_record["trade_flow"] == "Import" + + # Check second record + second_record = records[1] + assert second_record["partner_country"] == "Germany" + assert second_record["commodity_description"] == "Motor cars" \ No newline at end of file