diff --git a/docs/tech-specs/structured-data-descriptor.md b/docs/tech-specs/structured-data-descriptor.md new file mode 100644 index 00000000..e3a797ae --- /dev/null +++ b/docs/tech-specs/structured-data-descriptor.md @@ -0,0 +1,559 @@ +# Structured Data Descriptor Specification + +## Overview + +The Structured Data Descriptor is a JSON-based configuration language that describes how to parse, transform, and import structured data into TrustGraph. It provides a declarative approach to data ingestion, supporting multiple input formats and complex transformation pipelines without requiring custom code. + +## Core Concepts + +### 1. Format Definition +Describes the input file type and parsing options. Determines which parser to use and how to interpret the source data. + +### 2. Field Mappings +Maps source paths to target fields with transformations. Defines how data flows from input sources to output schema fields. + +### 3. Transform Pipeline +Chain of data transformations that can be applied to field values, including: +- Data cleaning (trim, normalize) +- Format conversion (date parsing, type casting) +- Calculations (arithmetic, string manipulation) +- Lookups (reference tables, substitutions) + +### 4. Validation Rules +Data quality checks applied to ensure data integrity: +- Type validation +- Range checks +- Pattern matching (regex) +- Required field validation +- Custom validation logic + +### 5. Global Settings +Configuration that applies across the entire import process: +- Lookup tables for data enrichment +- Global variables and constants +- Output format specifications +- Error handling policies + +## Implementation Strategy + +The importer implementation follows this pipeline: + +1. **Parse Configuration** - Load and validate the JSON descriptor +2. **Initialize Parser** - Load appropriate parser (CSV, XML, JSON, etc.) based on `format.type` +3. **Apply Preprocessing** - Execute global filters and transformations +4. **Process Records** - For each input record: + - Extract data using source paths (JSONPath, XPath, column names) + - Apply field-level transforms in sequence + - Validate results against defined rules + - Apply default values for missing data +5. **Apply Postprocessing** - Execute deduplication, aggregation, etc. +6. **Generate Output** - Produce data in specified target format + +## Path Expression Support + +Different input formats use appropriate path expression languages: + +- **CSV**: Column names or indices (`"column_name"` or `"[2]"`) +- **JSON**: JSONPath syntax (`"$.user.profile.email"`) +- **XML**: XPath expressions (`"//product[@id='123']/price"`) +- **Fixed-width**: Field names from field definitions + +## Benefits + +- **Single Codebase** - One importer handles multiple input formats +- **User-Friendly** - Non-technical users can create configurations +- **Reusable** - Configurations can be shared and versioned +- **Flexible** - Complex transformations without custom coding +- **Robust** - Built-in validation and comprehensive error handling +- **Maintainable** - Declarative approach reduces implementation complexity + +## Language Specification + +The Structured Data Descriptor uses a JSON configuration format with the following top-level structure: + +```json +{ + "version": "1.0", + "metadata": { + "name": "Configuration Name", + "description": "Description of what this config does", + "author": "Author Name", + "created": "2024-01-01T00:00:00Z" + }, + "format": { ... }, + "globals": { ... }, + "preprocessing": [ ... ], + "mappings": [ ... ], + "postprocessing": [ ... ], + "output": { ... } +} +``` + +### Format Definition + +Describes the input data format and parsing options: + +```json +{ + "format": { + "type": "csv|json|xml|fixed-width|excel|parquet", + "encoding": "utf-8", + "options": { + // Format-specific options + } + } +} +``` + +#### CSV Format Options +```json +{ + "format": { + "type": "csv", + "options": { + "delimiter": ",", + "quote_char": "\"", + "escape_char": "\\", + "skip_rows": 1, + "has_header": true, + "null_values": ["", "NULL", "null", "N/A"] + } + } +} +``` + +#### JSON Format Options +```json +{ + "format": { + "type": "json", + "options": { + "root_path": "$.data", + "array_mode": "records|single", + "flatten": false + } + } +} +``` + +#### XML Format Options +```json +{ + "format": { + "type": "xml", + "options": { + "root_element": "//records/record", + "namespaces": { + "ns": "http://example.com/namespace" + } + } + } +} +``` + +### Global Settings + +Define lookup tables, variables, and global configuration: + +```json +{ + "globals": { + "variables": { + "current_date": "2024-01-01", + "batch_id": "BATCH_001", + "default_confidence": 0.8 + }, + "lookup_tables": { + "country_codes": { + "US": "United States", + "UK": "United Kingdom", + "CA": "Canada" + }, + "status_mapping": { + "1": "active", + "0": "inactive" + } + }, + "constants": { + "source_system": "legacy_crm", + "import_type": "full" + } + } +} +``` + +### Field Mappings + +Define how source data maps to target fields with transformations: + +```json +{ + "mappings": [ + { + "target_field": "person_name", + "source": "$.name", + "transforms": [ + {"type": "trim"}, + {"type": "title_case"}, + {"type": "required"} + ], + "validation": [ + {"type": "min_length", "value": 2}, + {"type": "max_length", "value": 100}, + {"type": "pattern", "value": "^[A-Za-z\\s]+$"} + ] + }, + { + "target_field": "age", + "source": "$.age", + "transforms": [ + {"type": "to_int"}, + {"type": "default", "value": 0} + ], + "validation": [ + {"type": "range", "min": 0, "max": 150} + ] + }, + { + "target_field": "country", + "source": "$.country_code", + "transforms": [ + {"type": "lookup", "table": "country_codes"}, + {"type": "default", "value": "Unknown"} + ] + } + ] +} +``` + +### Transform Types + +Available transformation functions: + +#### String Transforms +```json +{"type": "trim"}, +{"type": "upper"}, +{"type": "lower"}, +{"type": "title_case"}, +{"type": "replace", "pattern": "old", "replacement": "new"}, +{"type": "regex_replace", "pattern": "\\d+", "replacement": "XXX"}, +{"type": "substring", "start": 0, "end": 10}, +{"type": "pad_left", "length": 10, "char": "0"} +``` + +#### Type Conversions +```json +{"type": "to_string"}, +{"type": "to_int"}, +{"type": "to_float"}, +{"type": "to_bool"}, +{"type": "to_date", "format": "YYYY-MM-DD"}, +{"type": "parse_json"} +``` + +#### Data Operations +```json +{"type": "default", "value": "default_value"}, +{"type": "lookup", "table": "table_name"}, +{"type": "concat", "values": ["field1", " - ", "field2"]}, +{"type": "calculate", "expression": "${field1} + ${field2}"}, +{"type": "conditional", "condition": "${age} > 18", "true_value": "adult", "false_value": "minor"} +``` + +### Validation Rules + +Data quality checks with configurable error handling: + +#### Basic Validations +```json +{"type": "required"}, +{"type": "not_null"}, +{"type": "min_length", "value": 5}, +{"type": "max_length", "value": 100}, +{"type": "range", "min": 0, "max": 1000}, +{"type": "pattern", "value": "^[A-Z]{2,3}$"}, +{"type": "in_list", "values": ["active", "inactive", "pending"]} +``` + +#### Custom Validations +```json +{ + "type": "custom", + "expression": "${age} >= 18 && ${country} == 'US'", + "message": "Must be 18+ and in US" +}, +{ + "type": "cross_field", + "fields": ["start_date", "end_date"], + "expression": "${start_date} < ${end_date}", + "message": "Start date must be before end date" +} +``` + +### Preprocessing and Postprocessing + +Global operations applied before/after field mapping: + +```json +{ + "preprocessing": [ + { + "type": "filter", + "condition": "${status} != 'deleted'" + }, + { + "type": "sort", + "field": "created_date", + "order": "asc" + } + ], + "postprocessing": [ + { + "type": "deduplicate", + "key_fields": ["email", "phone"] + }, + { + "type": "aggregate", + "group_by": ["country"], + "functions": { + "total_count": {"type": "count"}, + "avg_age": {"type": "avg", "field": "age"} + } + } + ] +} +``` + +### Output Configuration + +Define how processed data should be output: + +```json +{ + "output": { + "format": "trustgraph-objects", + "schema_name": "person", + "options": { + "batch_size": 1000, + "confidence": 0.9, + "source_span_field": "raw_text", + "metadata": { + "source": "crm_import", + "version": "1.0" + } + }, + "error_handling": { + "on_validation_error": "skip|fail|log", + "on_transform_error": "skip|fail|default", + "max_errors": 100, + "error_output": "errors.json" + } + } +} +``` + +## Complete Example + +```json +{ + "version": "1.0", + "metadata": { + "name": "Customer Import from CRM CSV", + "description": "Imports customer data from legacy CRM system", + "author": "Data Team", + "created": "2024-01-01T00:00:00Z" + }, + "format": { + "type": "csv", + "encoding": "utf-8", + "options": { + "delimiter": ",", + "has_header": true, + "skip_rows": 1 + } + }, + "globals": { + "variables": { + "import_date": "2024-01-01", + "default_confidence": 0.85 + }, + "lookup_tables": { + "country_codes": { + "US": "United States", + "CA": "Canada", + "UK": "United Kingdom" + } + } + }, + "preprocessing": [ + { + "type": "filter", + "condition": "${status} == 'active'" + } + ], + "mappings": [ + { + "target_field": "full_name", + "source": "customer_name", + "transforms": [ + {"type": "trim"}, + {"type": "title_case"} + ], + "validation": [ + {"type": "required"}, + {"type": "min_length", "value": 2} + ] + }, + { + "target_field": "email", + "source": "email_address", + "transforms": [ + {"type": "trim"}, + {"type": "lower"} + ], + "validation": [ + {"type": "pattern", "value": "^[\\w.-]+@[\\w.-]+\\.[a-zA-Z]{2,}$"} + ] + }, + { + "target_field": "age", + "source": "age", + "transforms": [ + {"type": "to_int"}, + {"type": "default", "value": 0} + ], + "validation": [ + {"type": "range", "min": 0, "max": 120} + ] + }, + { + "target_field": "country", + "source": "country_code", + "transforms": [ + {"type": "lookup", "table": "country_codes"}, + {"type": "default", "value": "Unknown"} + ] + } + ], + "output": { + "format": "trustgraph-objects", + "schema_name": "customer", + "options": { + "confidence": "${default_confidence}", + "batch_size": 500 + }, + "error_handling": { + "on_validation_error": "log", + "max_errors": 50 + } + } +} +``` + +## LLM Prompt for Descriptor Generation + +The following prompt can be used to have an LLM analyze sample data and generate a descriptor configuration: + +``` +I need you to analyze the provided data sample and create a Structured Data Descriptor configuration in JSON format. + +The descriptor should follow this specification: +- version: "1.0" +- metadata: Configuration name, description, author, and creation date +- format: Input format type and parsing options +- globals: Variables, lookup tables, and constants +- preprocessing: Filters and transformations applied before mapping +- mappings: Field-by-field mapping from source to target with transformations and validations +- postprocessing: Operations like deduplication or aggregation +- output: Target format and error handling configuration + +ANALYZE THE DATA: +1. Identify the format (CSV, JSON, XML, etc.) +2. Detect delimiters, encodings, and structure +3. Find data types for each field +4. Identify patterns and constraints +5. Look for fields that need cleaning or transformation +6. Find relationships between fields +7. Identify lookup opportunities (codes that map to values) +8. Detect required vs optional fields + +CREATE THE DESCRIPTOR: +For each field in the sample data: +- Map it to an appropriate target field name +- Add necessary transformations (trim, case conversion, type casting) +- Include appropriate validations (required, patterns, ranges) +- Set defaults for missing values + +Include preprocessing if needed: +- Filters to exclude invalid records +- Sorting requirements + +Include postprocessing if beneficial: +- Deduplication on key fields +- Aggregation for summary data + +Configure output for TrustGraph: +- format: "trustgraph-objects" +- schema_name: Based on the data entity type +- Appropriate error handling + +DATA SAMPLE: +[Insert data sample here] + +ADDITIONAL CONTEXT (optional): +- Target schema name: [if known] +- Business rules: [any specific requirements] +- Data quality issues to address: [known problems] + +Generate a complete, valid Structured Data Descriptor configuration that will properly import this data into TrustGraph. Include comments explaining key decisions. +``` + +### Example Usage Prompt + +``` +I need you to analyze the provided data sample and create a Structured Data Descriptor configuration in JSON format. + +[Standard instructions from above...] + +DATA SAMPLE: +```csv +CustomerID,Name,Email,Age,Country,Status,JoinDate,TotalPurchases +1001,"Smith, John",john.smith@email.com,35,US,1,2023-01-15,5420.50 +1002,"doe, jane",JANE.DOE@GMAIL.COM,28,CA,1,2023-03-22,3200.00 +1003,"Bob Johnson",bob@,62,UK,0,2022-11-01,0 +1004,"Alice Chen","alice.chen@company.org",41,US,1,2023-06-10,8900.25 +1005,,invalid-email,25,XX,1,2024-01-01,100 +``` + +ADDITIONAL CONTEXT: +- Target schema name: customer +- Business rules: Email should be valid and lowercase, names should be title case +- Data quality issues: Some emails are invalid, some names are missing, country codes need mapping +``` + +### Prompt for Analyzing Existing Data Without Sample + +``` +I need you to help me create a Structured Data Descriptor configuration for importing [data type] data. + +The source data has these characteristics: +- Format: [CSV/JSON/XML/etc] +- Fields: [list the fields] +- Data quality issues: [describe any known issues] +- Volume: [approximate number of records] + +Requirements: +- [List any specific transformation needs] +- [List any validation requirements] +- [List any business rules] + +Please generate a Structured Data Descriptor configuration that will: +1. Parse the input format correctly +2. Clean and standardize the data +3. Validate according to the requirements +4. Handle errors gracefully +5. Output in TrustGraph ExtractedObject format + +Focus on making the configuration robust and reusable. +``` \ No newline at end of file diff --git a/trustgraph-cli/pyproject.toml b/trustgraph-cli/pyproject.toml index aa2be8cb..5e9ec07f 100644 --- a/trustgraph-cli/pyproject.toml +++ b/trustgraph-cli/pyproject.toml @@ -54,6 +54,7 @@ tg-load-sample-documents = "trustgraph.cli.load_sample_documents:main" tg-load-text = "trustgraph.cli.load_text:main" tg-load-turtle = "trustgraph.cli.load_turtle:main" tg-load-knowledge = "trustgraph.cli.load_knowledge:main" +tg-load-structured-data = "trustgraph.cli.load_structured_data:main" tg-put-flow-class = "trustgraph.cli.put_flow_class:main" tg-put-kg-core = "trustgraph.cli.put_kg_core:main" tg-remove-library-document = "trustgraph.cli.remove_library_document:main" diff --git a/trustgraph-cli/trustgraph/cli/load_structured_data.py b/trustgraph-cli/trustgraph/cli/load_structured_data.py new file mode 100644 index 00000000..73be633c --- /dev/null +++ b/trustgraph-cli/trustgraph/cli/load_structured_data.py @@ -0,0 +1,282 @@ +""" +Load structured data into TrustGraph using a descriptor configuration. + +This utility can: +1. Analyze data samples to suggest appropriate schemas +2. Generate descriptor configurations from data samples +3. Parse and transform data using descriptor configurations +4. Import processed data into TrustGraph + +The tool supports running all steps automatically or individual steps for +validation and debugging. The descriptor language allows for complex +transformations, validations, and mappings without requiring custom code. +""" + +import argparse +import os +import sys +import json +import logging + +# Module logger +logger = logging.getLogger(__name__) + +default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/') + + +def load_structured_data( + api_url: str, + input_file: str, + descriptor_file: str = None, + suggest_schema: bool = False, + generate_descriptor: bool = False, + parse_only: bool = False, + output_file: str = None, + sample_size: int = 100, + sample_chars: int = 500, + schema_name: str = None, + dry_run: bool = False, + verbose: bool = False +): + """ + Load structured data using a descriptor configuration. + + Args: + api_url: TrustGraph API URL + input_file: Path to input data file + descriptor_file: Path to JSON descriptor configuration + suggest_schema: Analyze data and suggest matching schemas + generate_descriptor: Generate descriptor from data sample + parse_only: Parse data but don't import to TrustGraph + output_file: Path to write output (descriptor/parsed data) + sample_size: Number of records to sample for analysis + sample_chars: Maximum characters to read for sampling + schema_name: Target schema name for generation + dry_run: If True, validate but don't import data + verbose: Enable verbose logging + """ + if verbose: + logging.basicConfig(level=logging.DEBUG) + else: + logging.basicConfig(level=logging.INFO) + + # Determine operation mode + if suggest_schema: + logger.info(f"Analyzing {input_file} to suggest schemas...") + logger.info(f"Sample size: {sample_size} records") + logger.info(f"Sample chars: {sample_chars} characters") + # TODO: Implement schema suggestion + print(f"Would analyze {input_file} and suggest matching schemas") + print(f"Using sample of {sample_size} records, max {sample_chars} characters") + + elif generate_descriptor: + logger.info(f"Generating descriptor from {input_file}...") + logger.info(f"Sample size: {sample_size} records") + logger.info(f"Sample chars: {sample_chars} characters") + if schema_name: + logger.info(f"Target schema: {schema_name}") + # TODO: Implement descriptor generation + print(f"Would generate descriptor from {input_file}") + print(f"Using sample of {sample_size} records, max {sample_chars} characters") + if output_file: + print(f"Would save descriptor to {output_file}") + + elif parse_only: + if not descriptor_file: + raise ValueError("--descriptor is required when using --parse-only") + logger.info(f"Parsing {input_file} with descriptor {descriptor_file}...") + # TODO: Implement parsing + print(f"Would parse {input_file} using {descriptor_file}") + if output_file: + print(f"Would save parsed data to {output_file}") + + else: + # Full pipeline: parse and import + if not descriptor_file: + # Auto-generate descriptor if not provided + logger.info("No descriptor provided, auto-generating...") + # TODO: Generate descriptor + print(f"Would auto-generate descriptor from {input_file}") + + logger.info(f"Processing {input_file} for import...") + # TODO: Implement full pipeline + print(f"Would process and import data from {input_file}") + if dry_run: + print("Dry run mode - no data will be imported") + + +def main(): + """Main entry point for the CLI.""" + + parser = argparse.ArgumentParser( + prog='tg-load-structured-data', + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Step 1: Analyze data and suggest matching schemas + %(prog)s --input customers.csv --suggest-schema + + # Step 2: Generate descriptor configuration from data sample + %(prog)s --input customers.csv --generate-descriptor --schema-name customer --output descriptor.json + + # Generate descriptor with custom sampling (more data for better analysis) + %(prog)s --input large_dataset.csv --generate-descriptor --schema-name product --sample-chars 100000 --sample-size 500 + + # Step 3: Parse data and review output without importing + %(prog)s --input customers.csv --descriptor descriptor.json --parse-only --output parsed.json + + # Step 4: Import data to TrustGraph using descriptor + %(prog)s --input customers.csv --descriptor descriptor.json + + # All-in-one: Auto-generate descriptor and import (for simple cases) + %(prog)s --input customers.csv --schema-name customer + + # Dry run to validate without importing + %(prog)s --input customers.csv --descriptor descriptor.json --dry-run + +Use Cases: + --suggest-schema : Diagnose which TrustGraph schemas might match your data + --generate-descriptor: Create/review the structured data language configuration + --parse-only : Validate that parsed data looks correct before import + (no mode flags) : Full pipeline - parse and import to TrustGraph + +For more information on the descriptor format, see: + docs/tech-specs/structured-data-descriptor.md + """.strip() + ) + + parser.add_argument( + '-u', '--api-url', + default=default_url, + help=f'TrustGraph API URL (default: {default_url})' + ) + + parser.add_argument( + '-i', '--input', + required=True, + help='Path to input data file to process' + ) + + parser.add_argument( + '-d', '--descriptor', + help='Path to JSON descriptor configuration file (required for full import and parse-only)' + ) + + # Operation modes (mutually exclusive) + mode_group = parser.add_mutually_exclusive_group() + mode_group.add_argument( + '--suggest-schema', + action='store_true', + help='Analyze data sample and suggest matching TrustGraph schemas' + ) + mode_group.add_argument( + '--generate-descriptor', + action='store_true', + help='Generate descriptor configuration from data sample' + ) + mode_group.add_argument( + '--parse-only', + action='store_true', + help='Parse data using descriptor but don\'t import to TrustGraph' + ) + + parser.add_argument( + '-o', '--output', + help='Output file path (for generated descriptors or parsed data)' + ) + + parser.add_argument( + '--sample-size', + type=int, + default=100, + help='Number of records to sample for analysis/generation (default: 100)' + ) + + parser.add_argument( + '--sample-chars', + type=int, + default=500, + help='Maximum characters to read from data file for sampling (default: 500)' + ) + + parser.add_argument( + '--schema-name', + help='Target schema name for descriptor generation' + ) + + parser.add_argument( + '--dry-run', + action='store_true', + help='Validate configuration and data without importing (full pipeline only)' + ) + + parser.add_argument( + '-v', '--verbose', + action='store_true', + help='Enable verbose output for debugging' + ) + + parser.add_argument( + '--batch-size', + type=int, + default=1000, + help='Number of records to process in each batch (default: 1000)' + ) + + parser.add_argument( + '--max-errors', + type=int, + default=100, + help='Maximum number of errors before stopping (default: 100)' + ) + + parser.add_argument( + '--error-file', + help='Path to write error records (optional)' + ) + + args = parser.parse_args() + + # Validate argument combinations + if args.parse_only and not args.descriptor: + print("Error: --descriptor is required when using --parse-only", file=sys.stderr) + sys.exit(1) + + if not any([args.suggest_schema, args.generate_descriptor, args.parse_only]) and not args.descriptor: + # Full pipeline mode without descriptor - schema_name should be provided + if not args.schema_name: + print("Error: --descriptor or --schema-name is required for full import", file=sys.stderr) + sys.exit(1) + + try: + load_structured_data( + api_url=args.api_url, + input_file=args.input, + descriptor_file=args.descriptor, + suggest_schema=args.suggest_schema, + generate_descriptor=args.generate_descriptor, + parse_only=args.parse_only, + output_file=args.output, + sample_size=args.sample_size, + sample_chars=args.sample_chars, + schema_name=args.schema_name, + dry_run=args.dry_run, + verbose=args.verbose + ) + except FileNotFoundError as e: + print(f"Error: File not found - {e}", file=sys.stderr) + sys.exit(1) + except json.JSONDecodeError as e: + print(f"Error: Invalid JSON in descriptor - {e}", file=sys.stderr) + sys.exit(1) + except Exception as e: + print(f"Error: {e}", file=sys.stderr) + if args.verbose: + import traceback + traceback.print_exc() + sys.exit(1) + + +if __name__ == "__main__": + main() \ No newline at end of file