apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-07-24 12:41:02 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Discover schema is working

Browse source
This commit is contained in:
Cyber MacGeddon 2025-09-06 11:47:14 +01:00
parent b571da61fb
commit 96665b552f
1 changed files with 34 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

48
trustgraph-cli/trustgraph/cli/load_structured_data.py
View file

@ -160,13 +160,18 @@ def load_structured_data(
logger.info(f"Sample chars: {sample_chars} characters") logger.info(f"Sample chars: {sample_chars} characters")
# Use the helper function to discover schema (get raw response for display) # Use the helper function to discover schema (get raw response for display)
response = _auto_discover_schema(api_url, input_file, sample_chars, logger, return_raw_response=True) response = _auto_discover_schema(api_url, input_file, sample_chars, flow, logger, return_raw_response=True)
if response: if response:
print("Schema Discoverion Results:") if isinstance(response, list) and len(response) == 1:
print("=" * 50) # Just print the schema name for clean output
print(response) print(response[0])
print("=" * 50) else:
# For multiple schemas or complex responses, show full details
print("Schema Discovery Results:")
print("=" * 50)
print(response)
print("=" * 50)
print("\nUse --generate-descriptor with --schema-name to create a descriptor configuration.") print("\nUse --generate-descriptor with --schema-name to create a descriptor configuration.")
else: else:
print("Could not determine the best matching schema for your data.") print("Could not determine the best matching schema for your data.")
@ -180,7 +185,7 @@ def load_structured_data(
# If no schema specified, discover it first # If no schema specified, discover it first
if not schema_name: if not schema_name:
logger.info("No schema specified, auto-discovering...") logger.info("No schema specified, auto-discovering...")
schema_name = _auto_discover_schema(api_url, input_file, sample_chars, logger) schema_name = _auto_discover_schema(api_url, input_file, sample_chars, flow, logger)
if not schema_name: if not schema_name:
print("Error: Could not determine schema automatically.") print("Error: Could not determine schema automatically.")
print("Please specify a schema using --schema-name or run --discover-schema first.") print("Please specify a schema using --schema-name or run --discover-schema first.")
@ -190,7 +195,7 @@ def load_structured_data(
logger.info(f"Target schema: {schema_name}") logger.info(f"Target schema: {schema_name}")
# Generate descriptor using helper function # Generate descriptor using helper function
descriptor = _auto_generate_descriptor(api_url, input_file, schema_name, sample_chars, logger) descriptor = _auto_generate_descriptor(api_url, input_file, schema_name, sample_chars, flow, logger)
if descriptor: if descriptor:
# Output the generated descriptor # Output the generated descriptor
@ -436,8 +441,8 @@ def load_structured_data(
"metadata": { "metadata": {
"id": f"parsed-{len(output_records)+1}", "id": f"parsed-{len(output_records)+1}",
"metadata": [], # Empty metadata triples "metadata": [], # Empty metadata triples
"user": "trustgraph", "user": user,
"collection": "default" "collection": collection
}, },
"schema_name": schema_name, "schema_name": schema_name,
"values": record, "values": record,
@ -478,13 +483,14 @@ def load_structured_data(
# Helper functions for auto mode # Helper functions for auto mode
def _auto_discover_schema(api_url, input_file, sample_chars, logger, return_raw_response=False): def _auto_discover_schema(api_url, input_file, sample_chars, flow, logger, return_raw_response=False):
"""Auto-discover the best matching schema for the input data """Auto-discover the best matching schema for the input data
Args: Args:
api_url: TrustGraph API URL api_url: TrustGraph API URL
input_file: Path to input data file input_file: Path to input data file
sample_chars: Number of characters to sample from file sample_chars: Number of characters to sample from file
flow: TrustGraph flow name to use for prompts
logger: Logger instance logger: Logger instance
return_raw_response: If True, return raw prompt response; if False, parse to extract schema name return_raw_response: If True, return raw prompt response; if False, parse to extract schema name
@ -532,7 +538,7 @@ def _auto_discover_schema(api_url, input_file, sample_chars, logger, return_raw_
logger.info(f"Successfully loaded {len(schemas)} schema definitions") logger.info(f"Successfully loaded {len(schemas)} schema definitions")
# Use prompt service for schema selection # Use prompt service for schema selection
flow_api = api.flow().id("default") flow_api = api.flow().id(flow)
# Call schema-selection prompt with actual schemas and data sample # Call schema-selection prompt with actual schemas and data sample
logger.info("Calling TrustGraph schema-selection prompt...") logger.info("Calling TrustGraph schema-selection prompt...")
@ -581,7 +587,7 @@ def _auto_discover_schema(api_url, input_file, sample_chars, logger, return_raw_
return None return None
def _auto_generate_descriptor(api_url, input_file, schema_name, sample_chars, logger): def _auto_generate_descriptor(api_url, input_file, schema_name, sample_chars, flow, logger):
"""Auto-generate descriptor configuration for the discovered schema""" """Auto-generate descriptor configuration for the discovered schema"""
try: try:
# Read sample data # Read sample data
@ -603,7 +609,7 @@ def _auto_generate_descriptor(api_url, input_file, schema_name, sample_chars, lo
schema_def = json.loads(schema_values[0].value) if isinstance(schema_values[0].value, str) else schema_values[0].value schema_def = json.loads(schema_values[0].value) if isinstance(schema_values[0].value, str) else schema_values[0].value
# Use prompt service for descriptor generation # Use prompt service for descriptor generation
flow_api = api.flow().id("default") flow_api = api.flow().id(flow)
# Call diagnose-structured-data prompt with schema and data sample # Call diagnose-structured-data prompt with schema and data sample
response = flow_api.prompt( response = flow_api.prompt(
@ -755,7 +761,19 @@ For more information on the descriptor format, see:
parser.add_argument( parser.add_argument(
'-f', '--flow', '-f', '--flow',
default='default', default='default',
help='TrustGraph flow name to use for import (default: default)' help='TrustGraph flow name to use for prompts and import (default: default)'
)
parser.add_argument(
'--user',
default='trustgraph',
help='User name for metadata (default: trustgraph)'
)
parser.add_argument(
'--collection',
default='default',
help='Collection name for metadata (default: default)'
) )
parser.add_argument( parser.add_argument(
@ -890,6 +908,8 @@ For more information on the descriptor format, see:
sample_chars=args.sample_chars, sample_chars=args.sample_chars,
schema_name=args.schema_name, schema_name=args.schema_name,
flow=args.flow, flow=args.flow,
user=args.user,
collection=args.collection,
dry_run=args.dry_run, dry_run=args.dry_run,
verbose=args.verbose verbose=args.verbose
) )