mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-07-24 12:41:02 +02:00
Make --auto make sense
This commit is contained in:
parent
6bd7b87aec
commit
bcbcc72d5a
1 changed files with 40 additions and 15 deletions
|
|
@ -115,14 +115,37 @@ def load_structured_data(
|
||||||
logger.info("Step 4: Importing data to TrustGraph...")
|
logger.info("Step 4: Importing data to TrustGraph...")
|
||||||
print("🚀 Importing data to TrustGraph...")
|
print("🚀 Importing data to TrustGraph...")
|
||||||
|
|
||||||
# Use the existing full pipeline logic with our auto-generated descriptor
|
# Recursively call ourselves with the auto-generated descriptor
|
||||||
# We'll fall through to the main import logic by setting descriptor_file to None
|
# This reuses all the existing import logic
|
||||||
# and schema_name to our discovered schema, then let existing logic handle import
|
import tempfile
|
||||||
schema_name = discovered_schema
|
import os
|
||||||
descriptor = auto_descriptor
|
|
||||||
|
|
||||||
# Continue to import logic below...
|
# Save auto-generated descriptor to temp file
|
||||||
logger.info("Proceeding with data import...")
|
temp_descriptor = tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False)
|
||||||
|
json.dump(auto_descriptor, temp_descriptor, indent=2)
|
||||||
|
temp_descriptor.close()
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Call the full pipeline mode with our auto-generated descriptor
|
||||||
|
result = load_structured_data(
|
||||||
|
api_url=api_url,
|
||||||
|
input_file=input_file,
|
||||||
|
descriptor_file=temp_descriptor.name,
|
||||||
|
flow=flow,
|
||||||
|
dry_run=False, # We already handled dry_run above
|
||||||
|
verbose=verbose
|
||||||
|
)
|
||||||
|
|
||||||
|
print("✅ Auto-import completed successfully!")
|
||||||
|
logger.info("Auto-import pipeline completed successfully")
|
||||||
|
return result
|
||||||
|
|
||||||
|
finally:
|
||||||
|
# Clean up temp descriptor file
|
||||||
|
try:
|
||||||
|
os.unlink(temp_descriptor.name)
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
|
||||||
elif suggest_schema:
|
elif suggest_schema:
|
||||||
logger.info(f"Analyzing {input_file} to suggest schemas...")
|
logger.info(f"Analyzing {input_file} to suggest schemas...")
|
||||||
|
|
@ -970,9 +993,6 @@ Examples:
|
||||||
%(prog)s --input customers.csv --descriptor descriptor.json
|
%(prog)s --input customers.csv --descriptor descriptor.json
|
||||||
%(prog)s --input products.xml --descriptor xml_descriptor.json
|
%(prog)s --input products.xml --descriptor xml_descriptor.json
|
||||||
|
|
||||||
# All-in-one: Auto-generate descriptor and import (for simple cases)
|
|
||||||
%(prog)s --input customers.csv --schema-name customer
|
|
||||||
|
|
||||||
# FULLY AUTOMATIC: Discover schema + generate descriptor + import (zero manual steps!)
|
# FULLY AUTOMATIC: Discover schema + generate descriptor + import (zero manual steps!)
|
||||||
%(prog)s --input customers.csv --auto
|
%(prog)s --input customers.csv --auto
|
||||||
%(prog)s --input products.xml --auto --dry-run # Preview before importing
|
%(prog)s --input products.xml --auto --dry-run # Preview before importing
|
||||||
|
|
@ -1111,11 +1131,15 @@ For more information on the descriptor format, see:
|
||||||
if (args.suggest_schema or args.generate_descriptor) and args.sample_size != 100: # 100 is default
|
if (args.suggest_schema or args.generate_descriptor) and args.sample_size != 100: # 100 is default
|
||||||
print("Warning: --sample-size is ignored in analysis modes, use --sample-chars instead", file=sys.stderr)
|
print("Warning: --sample-size is ignored in analysis modes, use --sample-chars instead", file=sys.stderr)
|
||||||
|
|
||||||
if not any([args.suggest_schema, args.generate_descriptor, args.parse_only]) and not args.descriptor:
|
# Require explicit mode selection - no implicit behavior
|
||||||
# Full pipeline mode without descriptor - schema_name should be provided
|
if not any([args.suggest_schema, args.generate_descriptor, args.parse_only, args.auto]):
|
||||||
if not args.schema_name:
|
print("Error: Must specify an operation mode", file=sys.stderr)
|
||||||
print("Error: --descriptor or --schema-name is required for full import", file=sys.stderr)
|
print("Available modes:", file=sys.stderr)
|
||||||
sys.exit(1)
|
print(" --auto : Discover schema + generate descriptor + import", file=sys.stderr)
|
||||||
|
print(" --suggest-schema : Analyze data and suggest schemas", file=sys.stderr)
|
||||||
|
print(" --generate-descriptor : Generate descriptor from data", file=sys.stderr)
|
||||||
|
print(" --parse-only : Parse data without importing", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
load_structured_data(
|
load_structured_data(
|
||||||
|
|
@ -1125,6 +1149,7 @@ For more information on the descriptor format, see:
|
||||||
suggest_schema=args.suggest_schema,
|
suggest_schema=args.suggest_schema,
|
||||||
generate_descriptor=args.generate_descriptor,
|
generate_descriptor=args.generate_descriptor,
|
||||||
parse_only=args.parse_only,
|
parse_only=args.parse_only,
|
||||||
|
auto=args.auto,
|
||||||
output_file=args.output,
|
output_file=args.output,
|
||||||
sample_size=args.sample_size,
|
sample_size=args.sample_size,
|
||||||
sample_chars=args.sample_chars,
|
sample_chars=args.sample_chars,
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue