Make --auto make sense

This commit is contained in:
Cyber MacGeddon 2025-09-05 17:05:45 +01:00
parent 6bd7b87aec
commit bcbcc72d5a

View file

@ -115,14 +115,37 @@ def load_structured_data(
logger.info("Step 4: Importing data to TrustGraph...") logger.info("Step 4: Importing data to TrustGraph...")
print("🚀 Importing data to TrustGraph...") print("🚀 Importing data to TrustGraph...")
# Use the existing full pipeline logic with our auto-generated descriptor # Recursively call ourselves with the auto-generated descriptor
# We'll fall through to the main import logic by setting descriptor_file to None # This reuses all the existing import logic
# and schema_name to our discovered schema, then let existing logic handle import import tempfile
schema_name = discovered_schema import os
descriptor = auto_descriptor
# Continue to import logic below... # Save auto-generated descriptor to temp file
logger.info("Proceeding with data import...") temp_descriptor = tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False)
json.dump(auto_descriptor, temp_descriptor, indent=2)
temp_descriptor.close()
try:
# Call the full pipeline mode with our auto-generated descriptor
result = load_structured_data(
api_url=api_url,
input_file=input_file,
descriptor_file=temp_descriptor.name,
flow=flow,
dry_run=False, # We already handled dry_run above
verbose=verbose
)
print("✅ Auto-import completed successfully!")
logger.info("Auto-import pipeline completed successfully")
return result
finally:
# Clean up temp descriptor file
try:
os.unlink(temp_descriptor.name)
except:
pass
elif suggest_schema: elif suggest_schema:
logger.info(f"Analyzing {input_file} to suggest schemas...") logger.info(f"Analyzing {input_file} to suggest schemas...")
@ -970,9 +993,6 @@ Examples:
%(prog)s --input customers.csv --descriptor descriptor.json %(prog)s --input customers.csv --descriptor descriptor.json
%(prog)s --input products.xml --descriptor xml_descriptor.json %(prog)s --input products.xml --descriptor xml_descriptor.json
# All-in-one: Auto-generate descriptor and import (for simple cases)
%(prog)s --input customers.csv --schema-name customer
# FULLY AUTOMATIC: Discover schema + generate descriptor + import (zero manual steps!) # FULLY AUTOMATIC: Discover schema + generate descriptor + import (zero manual steps!)
%(prog)s --input customers.csv --auto %(prog)s --input customers.csv --auto
%(prog)s --input products.xml --auto --dry-run # Preview before importing %(prog)s --input products.xml --auto --dry-run # Preview before importing
@ -1111,11 +1131,15 @@ For more information on the descriptor format, see:
if (args.suggest_schema or args.generate_descriptor) and args.sample_size != 100: # 100 is default if (args.suggest_schema or args.generate_descriptor) and args.sample_size != 100: # 100 is default
print("Warning: --sample-size is ignored in analysis modes, use --sample-chars instead", file=sys.stderr) print("Warning: --sample-size is ignored in analysis modes, use --sample-chars instead", file=sys.stderr)
if not any([args.suggest_schema, args.generate_descriptor, args.parse_only]) and not args.descriptor: # Require explicit mode selection - no implicit behavior
# Full pipeline mode without descriptor - schema_name should be provided if not any([args.suggest_schema, args.generate_descriptor, args.parse_only, args.auto]):
if not args.schema_name: print("Error: Must specify an operation mode", file=sys.stderr)
print("Error: --descriptor or --schema-name is required for full import", file=sys.stderr) print("Available modes:", file=sys.stderr)
sys.exit(1) print(" --auto : Discover schema + generate descriptor + import", file=sys.stderr)
print(" --suggest-schema : Analyze data and suggest schemas", file=sys.stderr)
print(" --generate-descriptor : Generate descriptor from data", file=sys.stderr)
print(" --parse-only : Parse data without importing", file=sys.stderr)
sys.exit(1)
try: try:
load_structured_data( load_structured_data(
@ -1125,6 +1149,7 @@ For more information on the descriptor format, see:
suggest_schema=args.suggest_schema, suggest_schema=args.suggest_schema,
generate_descriptor=args.generate_descriptor, generate_descriptor=args.generate_descriptor,
parse_only=args.parse_only, parse_only=args.parse_only,
auto=args.auto,
output_file=args.output, output_file=args.output,
sample_size=args.sample_size, sample_size=args.sample_size,
sample_chars=args.sample_chars, sample_chars=args.sample_chars,