From 4ac304e4b41551009659a19f25de2190c1431717 Mon Sep 17 00:00:00 2001 From: Cyber MacGeddon Date: Thu, 3 Jul 2025 11:33:43 +0100 Subject: [PATCH] CLI docs --- docs/cli/tg-remove-library-document.md | 530 ++++++++++++++++++++++ docs/cli/tg-show-library-documents.md | 481 ++++++++++++++++++++ docs/cli/tg-start-library-processing.md | 563 ++++++++++++++++++++++++ docs/cli/tg-stop-library-processing.md | 507 +++++++++++++++++++++ 4 files changed, 2081 insertions(+) create mode 100644 docs/cli/tg-remove-library-document.md create mode 100644 docs/cli/tg-show-library-documents.md create mode 100644 docs/cli/tg-start-library-processing.md create mode 100644 docs/cli/tg-stop-library-processing.md diff --git a/docs/cli/tg-remove-library-document.md b/docs/cli/tg-remove-library-document.md new file mode 100644 index 00000000..f3095e85 --- /dev/null +++ b/docs/cli/tg-remove-library-document.md @@ -0,0 +1,530 @@ +# tg-remove-library-document + +Removes a document from the TrustGraph document library. + +## Synopsis + +```bash +tg-remove-library-document --id DOCUMENT_ID [options] +``` + +## Description + +The `tg-remove-library-document` command permanently removes a document from TrustGraph's document library. This operation deletes the document metadata, content, and any associated processing records. + +**⚠️ Warning**: This operation is permanent and cannot be undone. Ensure you have backups if the document data is important. + +## Options + +### Required Arguments + +- `--identifier, --id ID`: Document ID to remove + +### Optional Arguments + +- `-u, --url URL`: TrustGraph API URL (default: `$TRUSTGRAPH_URL` or `http://localhost:8088/`) +- `-U, --user USER`: User ID (default: `trustgraph`) + +## Examples + +### Remove Single Document +```bash +tg-remove-library-document --id "doc_123456789" +``` + +### Remove with Custom User +```bash +tg-remove-library-document --id "doc_987654321" -U "research-team" +``` + +### Remove with Custom API URL +```bash +tg-remove-library-document --id "doc_555" -u http://staging:8088/ +``` + +## Prerequisites + +### Document Must Exist +Verify the document exists before attempting removal: + +```bash +# List documents to find the ID +tg-show-library-documents + +# Search for specific document +tg-show-library-documents | grep "doc_123456789" +``` + +### Check for Active Processing +Before removing a document, check if it's currently being processed: + +```bash +# Check for active processing jobs +tg-show-flows | grep "processing" + +# Stop any active processing first +# tg-stop-library-processing --id "processing_id" +``` + +## Use Cases + +### Cleanup Old Documents +```bash +# Remove outdated documents +old_docs=("doc_old1" "doc_old2" "doc_deprecated") +for doc_id in "${old_docs[@]}"; do + echo "Removing $doc_id..." + tg-remove-library-document --id "$doc_id" +done +``` + +### Remove Test Documents +```bash +# Remove test documents after development +tg-show-library-documents | \ + grep "test\|demo\|sample" | \ + grep "| id" | \ + awk '{print $3}' | \ + while read doc_id; do + echo "Removing test document: $doc_id" + tg-remove-library-document --id "$doc_id" + done +``` + +### User-Specific Cleanup +```bash +# Remove all documents for a specific user +cleanup_user_documents() { + local user="$1" + + echo "Removing all documents for user: $user" + + # Get document IDs for the user + tg-show-library-documents -U "$user" | \ + grep "| id" | \ + awk '{print $3}' | \ + while read doc_id; do + echo "Removing document: $doc_id" + tg-remove-library-document --id "$doc_id" -U "$user" + done +} + +# Usage +cleanup_user_documents "temp-user" +``` + +### Conditional Removal +```bash +# Remove documents based on criteria +remove_by_criteria() { + local criteria="$1" + + echo "Removing documents matching criteria: $criteria" + + tg-show-library-documents | \ + grep -B5 -A5 "$criteria" | \ + grep "| id" | \ + awk '{print $3}' | \ + while read doc_id; do + # Confirm before removal + echo -n "Remove document $doc_id? (y/N): " + read confirm + if [[ "$confirm" =~ ^[Yy]$ ]]; then + tg-remove-library-document --id "$doc_id" + echo "Removed: $doc_id" + else + echo "Skipped: $doc_id" + fi + done +} + +# Remove documents containing "draft" in title +remove_by_criteria "draft" +``` + +## Safety Procedures + +### Backup Before Removal +```bash +# Create backup of document metadata before removal +backup_document() { + local doc_id="$1" + local backup_dir="document_backups/$(date +%Y%m%d)" + + mkdir -p "$backup_dir" + + echo "Backing up document: $doc_id" + + # Get document metadata + tg-show-library-documents | \ + grep -A10 -B2 "$doc_id" > "$backup_dir/$doc_id.metadata" + + # Note: Actual document content backup would require additional API + echo "Backup saved: $backup_dir/$doc_id.metadata" +} + +# Backup then remove +safe_remove() { + local doc_id="$1" + + backup_document "$doc_id" + + echo "Removing document: $doc_id" + tg-remove-library-document --id "$doc_id" + + echo "Document removed: $doc_id" +} + +# Usage +safe_remove "doc_123456789" +``` + +### Verification Script +```bash +#!/bin/bash +# safe-remove-document.sh +doc_id="$1" +user="${2:-trustgraph}" + +if [ -z "$doc_id" ]; then + echo "Usage: $0 [user]" + exit 1 +fi + +echo "Safety checks for removing document: $doc_id" + +# Check if document exists +if ! tg-show-library-documents -U "$user" | grep -q "$doc_id"; then + echo "ERROR: Document '$doc_id' not found for user '$user'" + exit 1 +fi + +# Show document details +echo "Document details:" +tg-show-library-documents -U "$user" | grep -A10 -B2 "$doc_id" + +# Check for active processing +echo "Checking for active processing..." +active_processing=$(tg-show-flows | grep -c "processing.*$doc_id" || echo "0") +if [ "$active_processing" -gt 0 ]; then + echo "WARNING: Document has $active_processing active processing jobs" + echo "Consider stopping processing first" +fi + +# Confirm removal +echo "" +read -p "Are you sure you want to remove this document? (y/N): " confirm + +if [ "$confirm" = "y" ] || [ "$confirm" = "Y" ]; then + echo "Removing document..." + tg-remove-library-document --id "$doc_id" -U "$user" + + # Verify removal + if ! tg-show-library-documents -U "$user" | grep -q "$doc_id"; then + echo "Document removed successfully" + else + echo "ERROR: Document still exists after removal" + exit 1 + fi +else + echo "Removal cancelled" +fi +``` + +### Bulk Removal with Confirmation +```bash +# Remove multiple documents with individual confirmation +bulk_remove_with_confirmation() { + local doc_list="$1" + + if [ ! -f "$doc_list" ]; then + echo "Usage: $0 " + return 1 + fi + + echo "Bulk removal with confirmation" + echo "Document list: $doc_list" + echo "==============================" + + while IFS= read -r doc_id; do + if [ -n "$doc_id" ]; then + # Show document info + echo -e "\nDocument ID: $doc_id" + tg-show-library-documents | grep -A5 -B1 "$doc_id" | grep -E "title|note|tags" + + # Confirm removal + echo -n "Remove this document? (y/N/q): " + read confirm + + case "$confirm" in + y|Y) + tg-remove-library-document --id "$doc_id" + echo "Removed: $doc_id" + ;; + q|Q) + echo "Quitting bulk removal" + break + ;; + *) + echo "Skipped: $doc_id" + ;; + esac + fi + done < "$doc_list" +} + +# Create list of documents to remove +echo -e "doc_123\ndoc_456\ndoc_789" > remove_list.txt +bulk_remove_with_confirmation "remove_list.txt" +``` + +## Advanced Usage + +### Age-Based Removal +```bash +# Remove documents older than specified days +remove_old_documents() { + local days_old="$1" + local dry_run="${2:-false}" + + if [ -z "$days_old" ]; then + echo "Usage: remove_old_documents [dry_run]" + return 1 + fi + + cutoff_date=$(date -d "$days_old days ago" +"%Y-%m-%d") + echo "Removing documents older than $cutoff_date" + + tg-show-library-documents | \ + awk -v cutoff="$cutoff_date" -v dry="$dry_run" ' + /^\| id/ { id = $3 } + /^\| time/ { + if ($3 < cutoff) { + if (dry == "true") { + print "Would remove: " id " (date: " $3 ")" + } else { + system("tg-remove-library-document --id " id) + print "Removed: " id " (date: " $3 ")" + } + } + }' +} + +# Dry run first +remove_old_documents 90 true + +# Actually remove +remove_old_documents 90 false +``` + +### Size-Based Cleanup +```bash +# Remove documents based on collection size limits +cleanup_by_collection_size() { + local max_docs="$1" + + echo "Maintaining maximum $max_docs documents per user" + + # Get unique users + users=$(tg-show-library-documents | grep "| id" | awk '{print $3}' | sort | uniq) + + for user in $users; do + echo "Checking user: $user" + + # Count documents for user + doc_count=$(tg-show-library-documents -U "$user" | grep -c "| id") + + if [ "$doc_count" -gt "$max_docs" ]; then + excess=$((doc_count - max_docs)) + echo "User $user has $doc_count documents (removing $excess oldest)" + + # Get oldest documents (by time) + tg-show-library-documents -U "$user" | \ + awk ' + /^\| id/ { id = $3 } + /^\| time/ { print $3 " " id } + ' | \ + sort | \ + head -n "$excess" | \ + while read date doc_id; do + echo "Removing old document: $doc_id ($date)" + tg-remove-library-document --id "$doc_id" -U "$user" + done + else + echo "User $user has $doc_count documents (within limit)" + fi + done +} + +# Maintain maximum 100 documents per user +cleanup_by_collection_size 100 +``` + +### Pattern-Based Removal +```bash +# Remove documents matching specific patterns +remove_by_pattern() { + local pattern="$1" + local field="${2:-title}" + + echo "Removing documents with '$pattern' in $field" + + tg-show-library-documents | \ + awk -v pattern="$pattern" -v field="$field" ' + /^\| id/ { id = $3 } + /^\| title/ && field=="title" { if ($0 ~ pattern) print id } + /^\| note/ && field=="note" { if ($0 ~ pattern) print id } + /^\| tags/ && field=="tags" { if ($0 ~ pattern) print id } + ' | \ + while read doc_id; do + echo "Removing document: $doc_id" + tg-remove-library-document --id "$doc_id" + done +} + +# Remove all test documents +remove_by_pattern "test" "title" +remove_by_pattern "temp" "tags" +``` + +## Error Handling + +### Document Not Found +```bash +Exception: Document not found +``` +**Solution**: Verify document ID exists with `tg-show-library-documents`. + +### Permission Errors +```bash +Exception: Access denied +``` +**Solution**: Check user permissions and document ownership. + +### Active Processing +```bash +Exception: Cannot remove document with active processing +``` +**Solution**: Stop processing with `tg-stop-library-processing` before removal. + +### API Connection Issues +```bash +Exception: Connection refused +``` +**Solution**: Check API URL and ensure TrustGraph is running. + +## Monitoring and Logging + +### Removal Logging +```bash +# Log all removals +logged_remove() { + local doc_id="$1" + local log_file="document_removals.log" + + timestamp=$(date) + echo "[$timestamp] Removing document: $doc_id" >> "$log_file" + + # Get document info before removal + tg-show-library-documents | \ + grep -A5 -B1 "$doc_id" >> "$log_file" + + # Remove document + if tg-remove-library-document --id "$doc_id"; then + echo "[$timestamp] Successfully removed: $doc_id" >> "$log_file" + else + echo "[$timestamp] Failed to remove: $doc_id" >> "$log_file" + fi + + echo "---" >> "$log_file" +} + +# Usage +logged_remove "doc_123456789" +``` + +### Audit Trail +```bash +# Create audit trail for removals +create_removal_audit() { + local doc_id="$1" + local reason="$2" + local audit_file="removal_audit.csv" + + # Create header if file doesn't exist + if [ ! -f "$audit_file" ]; then + echo "timestamp,document_id,user,reason,status" > "$audit_file" + fi + + timestamp=$(date '+%Y-%m-%d %H:%M:%S') + user=$(whoami) + + # Attempt removal + if tg-remove-library-document --id "$doc_id"; then + status="success" + else + status="failed" + fi + + # Log to audit file + echo "$timestamp,$doc_id,$user,$reason,$status" >> "$audit_file" +} + +# Usage +create_removal_audit "doc_123" "Outdated content" +``` + +## Environment Variables + +- `TRUSTGRAPH_URL`: Default API URL + +## Related Commands + +- [`tg-show-library-documents`](tg-show-library-documents.md) - List library documents +- [`tg-add-library-document`](tg-add-library-document.md) - Add documents to library +- [`tg-start-library-processing`](tg-start-library-processing.md) - Start document processing +- [`tg-stop-library-processing`](tg-stop-library-processing.md) - Stop document processing + +## API Integration + +This command uses the [Library API](../apis/api-librarian.md) to remove documents from the document repository. + +## Best Practices + +1. **Always Backup**: Create backups before removing important documents +2. **Verification**: Verify document existence before removal attempts +3. **Processing Check**: Ensure no active processing before removal +4. **Audit Trail**: Maintain logs of all removal operations +5. **Confirmation**: Use interactive confirmation for bulk operations +6. **Testing**: Test removal procedures in non-production environments +7. **Access Control**: Ensure appropriate permissions for removal operations + +## Troubleshooting + +### Document Still Exists After Removal +```bash +# Verify removal +tg-show-library-documents | grep "document-id" + +# Check for caching issues +# Wait a moment and try again + +# Verify API connectivity +curl -s "$TRUSTGRAPH_URL/api/v1/library/documents" > /dev/null +``` + +### Permission Issues +```bash +# Check user permissions +tg-show-library-documents -U "your-user" | grep "document-id" + +# Verify user ownership of document +``` + +### Cannot Remove Due to References +```bash +# Check for document references in processing jobs +tg-show-flows | grep "document-id" + +# Stop any referencing processes first +``` \ No newline at end of file diff --git a/docs/cli/tg-show-library-documents.md b/docs/cli/tg-show-library-documents.md new file mode 100644 index 00000000..ea5118a9 --- /dev/null +++ b/docs/cli/tg-show-library-documents.md @@ -0,0 +1,481 @@ +# tg-show-library-documents + +Lists all documents stored in the TrustGraph document library with their metadata. + +## Synopsis + +```bash +tg-show-library-documents [options] +``` + +## Description + +The `tg-show-library-documents` command displays all documents currently stored in TrustGraph's document library. For each document, it shows comprehensive metadata including ID, timestamp, title, document type, comments, and associated tags. + +The document library serves as a centralized repository for managing documents before and after processing through TrustGraph workflows. + +## Options + +### Optional Arguments + +- `-u, --api-url URL`: TrustGraph API URL (default: `$TRUSTGRAPH_URL` or `http://localhost:8088/`) +- `-U, --user USER`: User ID to filter documents (default: `trustgraph`) + +## Examples + +### List All Documents +```bash +tg-show-library-documents +``` + +### List Documents for Specific User +```bash +tg-show-library-documents -U "research-team" +``` + +### Using Custom API URL +```bash +tg-show-library-documents -u http://production:8088/ +``` + +## Output Format + +The command displays each document in a formatted table: + +``` ++-------+----------------------------------+ +| id | doc_123456789 | +| time | 2023-12-15 10:30:45 | +| title | Technical Manual v2.1 | +| kind | PDF | +| note | Updated installation procedures | +| tags | technical, manual, v2.1 | ++-------+----------------------------------+ + ++-------+----------------------------------+ +| id | doc_987654321 | +| time | 2023-12-14 15:22:10 | +| title | Q4 Financial Report | +| kind | PDF | +| note | Quarterly analysis and metrics | +| tags | finance, quarterly, 2023 | ++-------+----------------------------------+ +``` + +### Document Properties + +- **id**: Unique document identifier +- **time**: Upload/creation timestamp +- **title**: Document title or name +- **kind**: Document type (PDF, DOCX, TXT, etc.) +- **note**: Comments or description +- **tags**: Comma-separated list of tags + +### Empty Results + +If no documents exist: +``` +No documents. +``` + +## Use Cases + +### Document Inventory +```bash +# Get complete document inventory +tg-show-library-documents > document-inventory.txt + +# Count total documents +tg-show-library-documents | grep -c "| id" +``` + +### Document Discovery +```bash +# Find documents by title pattern +tg-show-library-documents | grep -i "manual" + +# Find documents by type +tg-show-library-documents | grep "| kind.*PDF" + +# Find recent documents +tg-show-library-documents | grep "2023-12" +``` + +### User-Specific Queries +```bash +# List documents by different users +users=("research-team" "finance-dept" "legal-team") +for user in "${users[@]}"; do + echo "Documents for $user:" + tg-show-library-documents -U "$user" + echo "---" +done +``` + +### Document Management +```bash +# Extract document IDs for processing +tg-show-library-documents | \ + grep "| id" | \ + awk '{print $3}' > document-ids.txt + +# Find documents by tags +tg-show-library-documents | \ + grep -A5 -B5 "research" | \ + grep "| id" | \ + awk '{print $3}' +``` + +## Advanced Usage + +### Document Analysis +```bash +# Analyze document distribution by type +analyze_document_types() { + echo "Document Type Distribution:" + echo "==========================" + + tg-show-library-documents | \ + grep "| kind" | \ + awk '{print $3}' | \ + sort | uniq -c | sort -nr +} + +analyze_document_types +``` + +### Document Age Analysis +```bash +# Find old documents +find_old_documents() { + local days_old="$1" + + echo "Documents older than $days_old days:" + echo "====================================" + + cutoff_date=$(date -d "$days_old days ago" +"%Y-%m-%d") + + tg-show-library-documents | \ + grep "| time" | \ + while read -r line; do + doc_date=$(echo "$line" | awk '{print $3}') + if [[ "$doc_date" < "$cutoff_date" ]]; then + echo "$line" + fi + done +} + +# Find documents older than 30 days +find_old_documents 30 +``` + +### Tag Analysis +```bash +# Analyze tag usage +analyze_tags() { + echo "Tag Usage Analysis:" + echo "==================" + + tg-show-library-documents | \ + grep "| tags" | \ + sed 's/| tags.*| \(.*\) |/\1/' | \ + tr ',' '\n' | \ + sed 's/^ *//;s/ *$//' | \ + sort | uniq -c | sort -nr +} + +analyze_tags +``` + +### Document Search +```bash +# Search documents by multiple criteria +search_documents() { + local query="$1" + + echo "Searching for: $query" + echo "====================" + + tg-show-library-documents | \ + grep -i -A6 -B6 "$query" | \ + grep -E "^\+|^\|" +} + +# Search for specific terms +search_documents "financial" +search_documents "manual" +``` + +### User Document Summary +```bash +# Generate user document summary +user_summary() { + local user="$1" + + echo "Document Summary for User: $user" + echo "================================" + + docs=$(tg-show-library-documents -U "$user") + + if [[ "$docs" == "No documents." ]]; then + echo "No documents found for user: $user" + return + fi + + # Count documents + doc_count=$(echo "$docs" | grep -c "| id") + echo "Total documents: $doc_count" + + # Count by type + echo -e "\nBy type:" + echo "$docs" | \ + grep "| kind" | \ + awk '{print $3}' | \ + sort | uniq -c | sort -nr + + # Recent documents + echo -e "\nRecent documents (last 7 days):" + recent_date=$(date -d "7 days ago" +"%Y-%m-%d") + echo "$docs" | \ + grep "| time" | \ + awk -v cutoff="$recent_date" '$3 >= cutoff {print $0}' +} + +# Generate summary for specific user +user_summary "research-team" +``` + +### Document Export +```bash +# Export document metadata to CSV +export_to_csv() { + local output_file="$1" + + echo "id,time,title,kind,note,tags" > "$output_file" + + tg-show-library-documents | \ + awk ' + BEGIN { record="" } + /^\+/ { + if (record != "") { + print record + record="" + } + } + /^\| id/ { gsub(/^\| id *\| /, ""); gsub(/ *\|$/, ""); record=$0"," } + /^\| time/ { gsub(/^\| time *\| /, ""); gsub(/ *\|$/, ""); record=record$0"," } + /^\| title/ { gsub(/^\| title *\| /, ""); gsub(/ *\|$/, ""); record=record$0"," } + /^\| kind/ { gsub(/^\| kind *\| /, ""); gsub(/ *\|$/, ""); record=record$0"," } + /^\| note/ { gsub(/^\| note *\| /, ""); gsub(/ *\|$/, ""); record=record$0"," } + /^\| tags/ { gsub(/^\| tags *\| /, ""); gsub(/ *\|$/, ""); record=record$0 } + END { if (record != "") print record } + ' >> "$output_file" + + echo "Exported to: $output_file" +} + +# Export to CSV +export_to_csv "documents.csv" +``` + +### Document Monitoring +```bash +# Monitor document library changes +monitor_documents() { + local interval="$1" + local log_file="document_changes.log" + + echo "Monitoring document library (interval: ${interval}s)" + echo "Log file: $log_file" + + # Get initial state + tg-show-library-documents > last_state.tmp + + while true; do + sleep "$interval" + + # Get current state + tg-show-library-documents > current_state.tmp + + # Compare states + if ! diff -q last_state.tmp current_state.tmp > /dev/null; then + timestamp=$(date) + echo "[$timestamp] Document library changed" >> "$log_file" + + # Log differences + diff last_state.tmp current_state.tmp >> "$log_file" + echo "---" >> "$log_file" + + # Update last state + mv current_state.tmp last_state.tmp + + echo "[$timestamp] Changes detected and logged" + else + rm current_state.tmp + fi + done +} + +# Monitor every 60 seconds +monitor_documents 60 +``` + +### Bulk Operations Helper +```bash +# Generate commands for bulk operations +generate_bulk_commands() { + local operation="$1" + + case "$operation" in + "remove-old") + echo "# Commands to remove old documents:" + cutoff_date=$(date -d "90 days ago" +"%Y-%m-%d") + tg-show-library-documents | \ + grep -B1 "| time.*$cutoff_date" | \ + grep "| id" | \ + awk '{print "tg-remove-library-document --id " $3}' + ;; + "process-unprocessed") + echo "# Commands to process documents:" + tg-show-library-documents | \ + grep "| id" | \ + awk '{print "tg-start-library-processing -d " $3 " --id proc-" $3}' + ;; + *) + echo "Unknown operation: $operation" + echo "Available: remove-old, process-unprocessed" + ;; + esac +} + +# Generate removal commands for old documents +generate_bulk_commands "remove-old" +``` + +## Integration with Other Commands + +### Document Processing Workflow +```bash +# Complete document workflow +process_document_workflow() { + echo "Document Library Workflow" + echo "========================" + + # 1. List current documents + echo "Current documents:" + tg-show-library-documents + + # 2. Add new document (example) + # tg-add-library-document --file new-doc.pdf --title "New Document" + + # 3. Start processing + # tg-start-library-processing -d doc_id --id proc_id + + # 4. Monitor processing + # tg-show-flows | grep processing + + # 5. Verify completion + echo "Documents after processing:" + tg-show-library-documents +} +``` + +### Document Lifecycle Management +```bash +# Manage document lifecycle +lifecycle_management() { + echo "Document Lifecycle Management" + echo "============================" + + # Get all documents + tg-show-library-documents | \ + grep "| id" | \ + awk '{print $3}' | \ + while read doc_id; do + echo "Processing document: $doc_id" + + # Check if already processed + if tg-invoke-document-rag -q "test" 2>/dev/null | grep -q "$doc_id"; then + echo " Already processed" + else + echo " Starting processing..." + # tg-start-library-processing -d "$doc_id" --id "proc-$doc_id" + fi + done +} +``` + +## Error Handling + +### Connection Issues +```bash +Exception: Connection refused +``` +**Solution**: Check API URL and ensure TrustGraph is running. + +### Permission Errors +```bash +Exception: Access denied +``` +**Solution**: Verify user permissions for library access. + +### User Not Found +```bash +Exception: User not found +``` +**Solution**: Check user ID spelling and ensure user exists. + +## Environment Variables + +- `TRUSTGRAPH_URL`: Default API URL + +## Related Commands + +- [`tg-add-library-document`](tg-add-library-document.md) - Add documents to library +- [`tg-remove-library-document`](tg-remove-library-document.md) - Remove documents from library +- [`tg-start-library-processing`](tg-start-library-processing.md) - Start document processing +- [`tg-stop-library-processing`](tg-stop-library-processing.md) - Stop document processing +- [`tg-invoke-document-rag`](tg-invoke-document-rag.md) - Query processed documents + +## API Integration + +This command uses the [Library API](../apis/api-librarian.md) to retrieve document metadata and listings. + +## Best Practices + +1. **Regular Monitoring**: Check library contents regularly +2. **User Organization**: Use different users for different document categories +3. **Tag Consistency**: Maintain consistent tagging schemes +4. **Cleanup**: Regularly remove outdated documents +5. **Backup**: Export document metadata for backup purposes +6. **Access Control**: Use appropriate user permissions +7. **Documentation**: Maintain good document titles and descriptions + +## Troubleshooting + +### No Documents Shown +```bash +# Check if documents exist for different users +tg-show-library-documents -U "different-user" + +# Verify API connectivity +curl -s "$TRUSTGRAPH_URL/api/v1/library/documents" > /dev/null +echo "API response: $?" +``` + +### Formatting Issues +```bash +# If output is garbled, check terminal width +export COLUMNS=120 +tg-show-library-documents +``` + +### Slow Response +```bash +# For large document libraries, consider filtering by user +tg-show-library-documents -U "specific-user" + +# Check system resources +free -h +ps aux | grep trustgraph +``` \ No newline at end of file diff --git a/docs/cli/tg-start-library-processing.md b/docs/cli/tg-start-library-processing.md new file mode 100644 index 00000000..534cedac --- /dev/null +++ b/docs/cli/tg-start-library-processing.md @@ -0,0 +1,563 @@ +# tg-start-library-processing + +Submits a library document for processing through TrustGraph workflows. + +## Synopsis + +```bash +tg-start-library-processing -d DOCUMENT_ID --id PROCESSING_ID [options] +``` + +## Description + +The `tg-start-library-processing` command initiates processing of a document stored in TrustGraph's document library. This triggers workflows that can extract text, generate embeddings, create knowledge graphs, and enable document search and analysis. + +Each processing job is assigned a unique processing ID for tracking and management purposes. + +## Options + +### Required Arguments + +- `-d, --document-id ID`: Document ID from the library to process +- `--id, --processing-id ID`: Unique identifier for this processing job + +### Optional Arguments + +- `-u, --api-url URL`: TrustGraph API URL (default: `$TRUSTGRAPH_URL` or `http://localhost:8088/`) +- `-U, --user USER`: User ID for processing context (default: `trustgraph`) +- `-i, --flow-id ID`: Flow instance to use for processing (default: `default`) +- `--collection COLLECTION`: Collection to assign processed data (default: `default`) +- `--tags TAGS`: Comma-separated tags for the processing job + +## Examples + +### Basic Document Processing +```bash +tg-start-library-processing -d "doc_123456789" --id "proc_001" +``` + +### Processing with Custom Collection +```bash +tg-start-library-processing \ + -d "research_paper_456" \ + --id "research_proc_001" \ + --collection "research-papers" \ + --tags "nlp,research,2023" +``` + +### Processing with Specific Flow +```bash +tg-start-library-processing \ + -d "technical_manual" \ + --id "manual_proc_001" \ + -i "document-analysis-flow" \ + -U "technical-team" \ + --collection "technical-docs" +``` + +### Processing Multiple Documents +```bash +# Process several documents in sequence +documents=("doc_001" "doc_002" "doc_003") +for i in "${!documents[@]}"; do + doc_id="${documents[$i]}" + proc_id="batch_proc_$(printf %03d $((i+1)))" + + echo "Processing document: $doc_id" + tg-start-library-processing \ + -d "$doc_id" \ + --id "$proc_id" \ + --collection "batch-processing" \ + --tags "batch,automated" +done +``` + +## Processing Workflow + +### Document Processing Steps +1. **Document Retrieval**: Fetch document from library +2. **Content Extraction**: Extract text and metadata +3. **Text Processing**: Clean and normalize content +4. **Embedding Generation**: Create vector embeddings +5. **Knowledge Extraction**: Generate triples and entities +6. **Index Creation**: Make content searchable + +### Processing Types +Different document types may trigger different processing workflows: +- **PDF Documents**: Text extraction, OCR if needed +- **Text Files**: Direct text processing +- **Images**: OCR and image analysis +- **Structured Data**: Schema extraction and mapping + +## Use Cases + +### Batch Document Processing +```bash +# Process all unprocessed documents +process_all_documents() { + local collection="$1" + local batch_id="batch_$(date +%Y%m%d_%H%M%S)" + + echo "Starting batch processing for collection: $collection" + + # Get all document IDs + tg-show-library-documents | \ + grep "| id" | \ + awk '{print $3}' | \ + while read -r doc_id; do + proc_id="${batch_id}_${doc_id}" + + echo "Processing document: $doc_id" + tg-start-library-processing \ + -d "$doc_id" \ + --id "$proc_id" \ + --collection "$collection" \ + --tags "batch,automated,$(date +%Y%m%d)" + + # Add delay to avoid overwhelming the system + sleep 2 + done +} + +# Process all documents +process_all_documents "processed-docs" +``` + +### Department-Specific Processing +```bash +# Process documents by department +process_by_department() { + local dept="$1" + local flow="$2" + + echo "Processing documents for department: $dept" + + # Find documents with department tag + tg-show-library-documents -U "$dept" | \ + grep "| id" | \ + awk '{print $3}' | \ + while read -r doc_id; do + proc_id="${dept}_proc_$(date +%s)_${doc_id}" + + echo "Processing $dept document: $doc_id" + tg-start-library-processing \ + -d "$doc_id" \ + --id "$proc_id" \ + -i "$flow" \ + -U "$dept" \ + --collection "${dept}-processed" \ + --tags "$dept,departmental" + done +} + +# Process documents for different departments +process_by_department "research" "research-flow" +process_by_department "finance" "document-flow" +process_by_department "legal" "compliance-flow" +``` + +### Priority Processing +```bash +# Process high-priority documents first +priority_processing() { + local priority_tags=("urgent" "high-priority" "critical") + + for tag in "${priority_tags[@]}"; do + echo "Processing $tag documents..." + + tg-show-library-documents | \ + grep -B5 -A5 "$tag" | \ + grep "| id" | \ + awk '{print $3}' | \ + while read -r doc_id; do + proc_id="priority_$(date +%s)_${doc_id}" + + echo "Processing priority document: $doc_id" + tg-start-library-processing \ + -d "$doc_id" \ + --id "$proc_id" \ + --collection "priority-processed" \ + --tags "priority,$tag" + done + done +} + +priority_processing +``` + +### Conditional Processing +```bash +# Process documents based on criteria +conditional_processing() { + local criteria="$1" + local flow="$2" + + echo "Processing documents matching criteria: $criteria" + + tg-show-library-documents | \ + grep -B10 -A10 "$criteria" | \ + grep "| id" | \ + awk '{print $3}' | \ + while read -r doc_id; do + # Check if already processed + if tg-invoke-document-rag -q "test" 2>/dev/null | grep -q "$doc_id"; then + echo "Document $doc_id already processed, skipping" + continue + fi + + proc_id="conditional_$(date +%s)_${doc_id}" + + echo "Processing document: $doc_id" + tg-start-library-processing \ + -d "$doc_id" \ + --id "$proc_id" \ + -i "$flow" \ + --collection "conditional-processed" \ + --tags "conditional,$criteria" + done +} + +# Process technical documents +conditional_processing "technical" "technical-flow" +``` + +## Advanced Usage + +### Processing with Validation +```bash +# Process with pre and post validation +validated_processing() { + local doc_id="$1" + local proc_id="$2" + local collection="$3" + + echo "Starting validated processing for: $doc_id" + + # Pre-processing validation + if ! tg-show-library-documents | grep -q "$doc_id"; then + echo "ERROR: Document $doc_id not found" + return 1 + fi + + # Check if processing ID is unique + if tg-show-flows | grep -q "$proc_id"; then + echo "ERROR: Processing ID $proc_id already in use" + return 1 + fi + + # Start processing + echo "Starting processing..." + tg-start-library-processing \ + -d "$doc_id" \ + --id "$proc_id" \ + --collection "$collection" \ + --tags "validated,$(date +%Y%m%d)" + + # Monitor processing + echo "Monitoring processing progress..." + timeout=300 # 5 minutes + elapsed=0 + interval=10 + + while [ $elapsed -lt $timeout ]; do + if tg-invoke-document-rag -q "test" -C "$collection" 2>/dev/null | grep -q "$doc_id"; then + echo "✓ Processing completed successfully" + return 0 + fi + + echo "Processing in progress... (${elapsed}s elapsed)" + sleep $interval + elapsed=$((elapsed + interval)) + done + + echo "⚠ Processing timeout reached" + return 1 +} + +# Usage +validated_processing "doc_123" "validated_proc_001" "validated-docs" +``` + +### Parallel Processing with Limits +```bash +# Process multiple documents in parallel with concurrency limits +parallel_processing() { + local doc_list=("$@") + local max_concurrent=5 + local current_jobs=0 + + echo "Processing ${#doc_list[@]} documents with max $max_concurrent concurrent jobs" + + for doc_id in "${doc_list[@]}"; do + # Wait if max concurrent jobs reached + while [ $current_jobs -ge $max_concurrent ]; do + wait -n # Wait for any job to complete + current_jobs=$((current_jobs - 1)) + done + + # Start processing in background + ( + proc_id="parallel_$(date +%s)_${doc_id}" + echo "Starting processing: $doc_id" + + tg-start-library-processing \ + -d "$doc_id" \ + --id "$proc_id" \ + --collection "parallel-processed" \ + --tags "parallel,batch" + + echo "Completed processing: $doc_id" + ) & + + current_jobs=$((current_jobs + 1)) + done + + # Wait for all remaining jobs + wait + echo "All processing jobs completed" +} + +# Get document list and process in parallel +doc_list=($(tg-show-library-documents | grep "| id" | awk '{print $3}')) +parallel_processing "${doc_list[@]}" +``` + +### Processing with Retry Logic +```bash +# Process with automatic retry on failure +processing_with_retry() { + local doc_id="$1" + local proc_id="$2" + local max_retries=3 + local retry_delay=30 + + for attempt in $(seq 1 $max_retries); do + echo "Processing attempt $attempt/$max_retries for document: $doc_id" + + if tg-start-library-processing \ + -d "$doc_id" \ + --id "${proc_id}_attempt_${attempt}" \ + --collection "retry-processed" \ + --tags "retry,attempt_$attempt"; then + + # Wait and check if processing succeeded + sleep $retry_delay + + if tg-invoke-document-rag -q "test" 2>/dev/null | grep -q "$doc_id"; then + echo "✓ Processing succeeded on attempt $attempt" + return 0 + else + echo "Processing started but content not yet accessible" + fi + else + echo "✗ Processing failed on attempt $attempt" + fi + + if [ $attempt -lt $max_retries ]; then + echo "Retrying in ${retry_delay}s..." + sleep $retry_delay + fi + done + + echo "✗ Processing failed after $max_retries attempts" + return 1 +} + +# Usage +processing_with_retry "doc_123" "retry_proc_001" +``` + +### Configuration-Driven Processing +```bash +# Process documents based on configuration file +config_driven_processing() { + local config_file="$1" + + if [ ! -f "$config_file" ]; then + echo "Configuration file not found: $config_file" + return 1 + fi + + echo "Processing documents based on configuration: $config_file" + + # Example configuration format: + # doc_id,flow_id,collection,tags + # doc_123,research-flow,research-docs,nlp research + + while IFS=',' read -r doc_id flow_id collection tags; do + # Skip header line + if [ "$doc_id" = "doc_id" ]; then + continue + fi + + proc_id="config_$(date +%s)_${doc_id}" + + echo "Processing: $doc_id -> $collection (flow: $flow_id)" + + tg-start-library-processing \ + -d "$doc_id" \ + --id "$proc_id" \ + -i "$flow_id" \ + --collection "$collection" \ + --tags "$tags" + + done < "$config_file" +} + +# Create example configuration +cat > processing_config.csv << EOF +doc_id,flow_id,collection,tags +doc_123,research-flow,research-docs,nlp research +doc_456,finance-flow,finance-docs,financial quarterly +doc_789,general-flow,general-docs,general processing +EOF + +# Process based on configuration +config_driven_processing "processing_config.csv" +``` + +## Error Handling + +### Document Not Found +```bash +Exception: Document not found +``` +**Solution**: Verify document exists with `tg-show-library-documents`. + +### Processing ID Conflict +```bash +Exception: Processing ID already exists +``` +**Solution**: Use a unique processing ID or check existing jobs with `tg-show-flows`. + +### Flow Not Found +```bash +Exception: Flow instance not found +``` +**Solution**: Verify flow exists with `tg-show-flows` or `tg-show-flow-classes`. + +### Insufficient Resources +```bash +Exception: Processing queue full +``` +**Solution**: Wait for current jobs to complete or scale processing resources. + +## Monitoring and Management + +### Processing Status +```bash +# Monitor processing progress +monitor_processing() { + local proc_id="$1" + local timeout="${2:-300}" # 5 minutes default + + echo "Monitoring processing: $proc_id" + + elapsed=0 + interval=10 + + while [ $elapsed -lt $timeout ]; do + # Check if processing is active + if tg-show-flows | grep -q "$proc_id"; then + echo "Processing active... (${elapsed}s elapsed)" + else + echo "Processing completed or stopped" + break + fi + + sleep $interval + elapsed=$((elapsed + interval)) + done + + if [ $elapsed -ge $timeout ]; then + echo "Monitoring timeout reached" + fi +} + +# Monitor specific processing job +monitor_processing "proc_001" 600 +``` + +### Batch Monitoring +```bash +# Monitor multiple processing jobs +monitor_batch() { + local proc_pattern="$1" + + echo "Monitoring batch processing: $proc_pattern" + + while true; do + active_jobs=$(tg-show-flows | grep -c "$proc_pattern" || echo "0") + + if [ "$active_jobs" -eq 0 ]; then + echo "All batch processing jobs completed" + break + fi + + echo "Active jobs: $active_jobs" + sleep 30 + done +} + +# Monitor batch processing +monitor_batch "batch_proc_" +``` + +## Environment Variables + +- `TRUSTGRAPH_URL`: Default API URL + +## Related Commands + +- [`tg-show-library-documents`](tg-show-library-documents.md) - List available documents +- [`tg-stop-library-processing`](tg-stop-library-processing.md) - Stop processing jobs +- [`tg-show-flows`](tg-show-flows.md) - Monitor processing flows +- [`tg-invoke-document-rag`](tg-invoke-document-rag.md) - Query processed documents + +## API Integration + +This command uses the [Library API](../apis/api-librarian.md) to initiate document processing workflows. + +## Best Practices + +1. **Unique IDs**: Always use unique processing IDs to avoid conflicts +2. **Resource Management**: Monitor system resources during batch processing +3. **Error Handling**: Implement retry logic for robust processing +4. **Monitoring**: Track processing progress and completion +5. **Collection Organization**: Use meaningful collection names +6. **Tagging**: Apply consistent tagging for better organization +7. **Documentation**: Document processing procedures and configurations + +## Troubleshooting + +### Processing Not Starting +```bash +# Check document exists +tg-show-library-documents | grep "document-id" + +# Check flow is available +tg-show-flows | grep "flow-id" + +# Check system resources +free -h +df -h +``` + +### Slow Processing +```bash +# Check processing queue +tg-show-flows | grep processing | wc -l + +# Monitor system load +top +htop +``` + +### Processing Failures +```bash +# Check processing logs +# (Log location depends on TrustGraph configuration) + +# Retry with different flow +tg-start-library-processing -d "doc-id" --id "retry-proc" -i "alternative-flow" +``` \ No newline at end of file diff --git a/docs/cli/tg-stop-library-processing.md b/docs/cli/tg-stop-library-processing.md new file mode 100644 index 00000000..053ea011 --- /dev/null +++ b/docs/cli/tg-stop-library-processing.md @@ -0,0 +1,507 @@ +# tg-stop-library-processing + +Removes a library document processing record from TrustGraph. + +## Synopsis + +```bash +tg-stop-library-processing --id PROCESSING_ID [options] +``` + +## Description + +The `tg-stop-library-processing` command removes a document processing record from TrustGraph's library processing system. This command removes the processing record but **does not stop in-flight processing** that may already be running. + +This is primarily used for cleaning up processing records, managing processing queues, and maintaining processing history. + +## Options + +### Required Arguments + +- `--id, --processing-id ID`: Processing ID to remove + +### Optional Arguments + +- `-u, --api-url URL`: TrustGraph API URL (default: `$TRUSTGRAPH_URL` or `http://localhost:8088/`) +- `-U, --user USER`: User ID (default: `trustgraph`) + +## Examples + +### Remove Single Processing Record +```bash +tg-stop-library-processing --id "proc_123456789" +``` + +### Remove with Custom User +```bash +tg-stop-library-processing --id "research_proc_001" -U "research-team" +``` + +### Remove with Custom API URL +```bash +tg-stop-library-processing --id "proc_555" -u http://staging:8088/ +``` + +## Important Limitations + +### Processing Record vs Active Processing +This command only removes the **processing record** and does not: +- Stop currently running processing jobs +- Cancel in-flight document analysis +- Interrupt active workflows + +### What It Does +- Removes processing metadata from library +- Cleans up processing history +- Allows reuse of processing IDs +- Maintains processing queue hygiene + +### What It Doesn't Do +- Stop active processing threads +- Cancel running analysis jobs +- Interrupt flow execution +- Free up computational resources immediately + +## Use Cases + +### Cleanup Failed Processing Records +```bash +# Remove failed processing records +failed_processes=("proc_failed_001" "proc_error_002" "proc_timeout_003") +for proc_id in "${failed_processes[@]}"; do + echo "Removing failed processing record: $proc_id" + tg-stop-library-processing --id "$proc_id" +done +``` + +### Batch Cleanup +```bash +# Clean up all processing records for a specific pattern +cleanup_batch_processing() { + local pattern="$1" + + echo "Cleaning up processing records matching: $pattern" + + # This would require a way to list processing records + # For now, use known processing IDs + tg-show-flows | \ + grep "$pattern" | \ + awk '{print $1}' | \ + while read proc_id; do + echo "Removing processing record: $proc_id" + tg-stop-library-processing --id "$proc_id" + done +} + +# Clean up old batch processing records +cleanup_batch_processing "batch_proc_" +``` + +### User-Specific Cleanup +```bash +# Clean up processing records for specific user +cleanup_user_processing() { + local user="$1" + + echo "Cleaning up processing records for user: $user" + + # Note: This assumes you have a way to list processing records by user + # Implementation would depend on available APIs + + # Example with known processing IDs + user_processes=("${user}_proc_001" "${user}_proc_002" "${user}_proc_003") + + for proc_id in "${user_processes[@]}"; do + echo "Removing processing record: $proc_id" + tg-stop-library-processing --id "$proc_id" -U "$user" + done +} + +# Clean up for specific user +cleanup_user_processing "temp-user" +``` + +### Age-Based Cleanup +```bash +# Clean up old processing records +cleanup_old_processing() { + local days_old="$1" + + echo "Cleaning up processing records older than $days_old days" + + # This would require timestamp information from processing records + # Implementation depends on available metadata + + cutoff_date=$(date -d "$days_old days ago" +"%Y%m%d") + + # Example with date-pattern processing IDs + # proc_20231215_001, proc_20231214_002, etc. + + for proc_id in proc_*; do + if [[ "$proc_id" =~ proc_([0-9]{8})_ ]]; then + proc_date="${BASH_REMATCH[1]}" + + if [[ "$proc_date" < "$cutoff_date" ]]; then + echo "Removing old processing record: $proc_id" + tg-stop-library-processing --id "$proc_id" + fi + fi + done +} + +# Clean up processing records older than 30 days +cleanup_old_processing 30 +``` + +## Safe Processing Management + +### Before Removing Processing Records +```bash +# Check if processing is actually complete before cleanup +safe_processing_cleanup() { + local proc_id="$1" + local doc_id="$2" + + echo "Safe cleanup for processing: $proc_id" + + # Check if document is accessible (processing likely complete) + if tg-invoke-document-rag -q "test" 2>/dev/null | grep -q "$doc_id"; then + echo "Document $doc_id is accessible, safe to remove processing record" + tg-stop-library-processing --id "$proc_id" + echo "Processing record removed: $proc_id" + else + echo "Document $doc_id not yet accessible, processing may still be active" + echo "Skipping removal of processing record: $proc_id" + fi +} + +# Usage +safe_processing_cleanup "proc_001" "doc_123" +``` + +### Verification Before Cleanup +```bash +# Verify processing completion before removing records +verify_and_cleanup() { + local proc_id="$1" + local collection="$2" + + echo "Verifying processing completion for: $proc_id" + + # Check if processing is still active in flows + if tg-show-flows | grep -q "$proc_id"; then + echo "Processing $proc_id is still active, not removing record" + return 1 + fi + + # Additional verification could include: + # - Checking if document content is available + # - Verifying embeddings are generated + # - Confirming knowledge graph updates + + echo "Processing appears complete, removing record" + tg-stop-library-processing --id "$proc_id" + + echo "Processing record removed: $proc_id" +} + +# Usage +verify_and_cleanup "proc_001" "research-docs" +``` + +## Advanced Usage + +### Conditional Cleanup +```bash +# Clean up processing records based on success criteria +conditional_cleanup() { + local proc_id="$1" + local doc_id="$2" + local collection="$3" + + echo "Conditional cleanup for: $proc_id" + + # Test if document is queryable (indicates successful processing) + test_query="What is this document about?" + + if result=$(tg-invoke-document-rag -q "$test_query" -C "$collection" 2>/dev/null); then + if echo "$result" | grep -q "answer"; then + echo "✓ Document is queryable, processing successful" + tg-stop-library-processing --id "$proc_id" + echo "Processing record cleaned up: $proc_id" + else + echo "⚠ Document query returned no answer, processing may be incomplete" + echo "Keeping processing record: $proc_id" + fi + else + echo "✗ Document query failed, processing incomplete or failed" + echo "Keeping processing record: $proc_id" + fi +} + +# Usage +conditional_cleanup "proc_001" "doc_123" "research-docs" +``` + +### Bulk Cleanup with Verification +```bash +# Bulk cleanup with individual verification +bulk_verified_cleanup() { + local proc_pattern="$1" + local collection="$2" + + echo "Bulk cleanup with verification for pattern: $proc_pattern" + + # Get list of processing IDs (this would need appropriate API) + # For now, use example pattern + + for proc_id in proc_batch_*; do + if [[ "$proc_id" =~ $proc_pattern ]]; then + echo "Checking processing: $proc_id" + + # Extract document ID from processing ID (example pattern) + if [[ "$proc_id" =~ _([^_]+)$ ]]; then + doc_id="${BASH_REMATCH[1]}" + + # Verify document is accessible + if tg-invoke-document-rag -q "test" -C "$collection" 2>/dev/null | grep -q "$doc_id"; then + echo "✓ Verified: $proc_id" + tg-stop-library-processing --id "$proc_id" + else + echo "⚠ Unverified: $proc_id" + fi + else + echo "? Unknown pattern: $proc_id" + fi + fi + done +} + +# Usage +bulk_verified_cleanup "batch_" "processed-docs" +``` + +### Processing Record Maintenance +```bash +# Maintain processing record hygiene +maintain_processing_records() { + local max_records="$1" + + echo "Maintaining processing records (max: $max_records)" + + # This would require an API to list and count processing records + # For now, demonstrate the concept + + # Count current processing records (placeholder) + current_count=150 # Would get this from API + + if [ "$current_count" -gt "$max_records" ]; then + excess=$((current_count - max_records)) + echo "Found $current_count records, removing $excess oldest" + + # Remove oldest processing records + # This would require timestamp information + echo "Would remove $excess oldest processing records" + + # Example implementation: + # oldest_records=($(get_oldest_processing_records $excess)) + # for proc_id in "${oldest_records[@]}"; do + # tg-stop-library-processing --id "$proc_id" + # done + else + echo "Processing record count within limits: $current_count" + fi +} + +# Maintain maximum 100 processing records +maintain_processing_records 100 +``` + +## Error Handling + +### Processing ID Not Found +```bash +Exception: Processing ID not found +``` +**Solution**: Verify processing ID exists and check spelling. + +### Processing Still Active +```bash +Exception: Cannot remove active processing record +``` +**Solution**: Wait for processing to complete or verify if processing is actually active. + +### Permission Errors +```bash +Exception: Access denied +``` +**Solution**: Check user permissions and processing record ownership. + +### API Connection Issues +```bash +Exception: Connection refused +``` +**Solution**: Check API URL and ensure TrustGraph is running. + +## Monitoring and Verification + +### Processing Record Status +```bash +# Check processing record status before removal +check_processing_status() { + local proc_id="$1" + + echo "Checking status of processing: $proc_id" + + # Check if processing is in active flows + if tg-show-flows | grep -q "$proc_id"; then + echo "Status: ACTIVE - Processing is currently running" + return 1 + else + echo "Status: INACTIVE - Processing not found in active flows" + return 0 + fi +} + +# Usage +if check_processing_status "proc_001"; then + echo "Safe to remove processing record" + tg-stop-library-processing --id "proc_001" +else + echo "Processing still active, not removing record" +fi +``` + +### Cleanup Verification +```bash +# Verify successful removal +verify_removal() { + local proc_id="$1" + + echo "Verifying removal of processing record: $proc_id" + + # Check if processing record still exists + # This would require an API to query processing records + + if tg-show-flows | grep -q "$proc_id"; then + echo "✗ Processing record still exists" + return 1 + else + echo "✓ Processing record successfully removed" + return 0 + fi +} + +# Usage +tg-stop-library-processing --id "proc_001" +verify_removal "proc_001" +``` + +## Integration with Processing Workflow + +### Complete Processing Lifecycle +```bash +# Complete processing lifecycle management +processing_lifecycle() { + local doc_id="$1" + local proc_id="$2" + local collection="$3" + + echo "Managing complete processing lifecycle" + echo "Document: $doc_id" + echo "Processing: $proc_id" + echo "Collection: $collection" + + # 1. Start processing + echo "1. Starting processing..." + tg-start-library-processing \ + -d "$doc_id" \ + --id "$proc_id" \ + --collection "$collection" + + # 2. Monitor processing + echo "2. Monitoring processing..." + timeout=300 + elapsed=0 + + while [ $elapsed -lt $timeout ]; do + if tg-invoke-document-rag -q "test" -C "$collection" 2>/dev/null | grep -q "$doc_id"; then + echo "✓ Processing completed" + break + fi + + sleep 10 + elapsed=$((elapsed + 10)) + done + + # 3. Verify completion + echo "3. Verifying completion..." + if tg-invoke-document-rag -q "What is this document?" -C "$collection" 2>/dev/null; then + echo "✓ Document is queryable" + + # 4. Clean up processing record + echo "4. Cleaning up processing record..." + tg-stop-library-processing --id "$proc_id" + echo "✓ Processing record removed" + else + echo "✗ Processing verification failed" + echo "Keeping processing record for investigation" + fi +} + +# Usage +processing_lifecycle "doc_123" "proc_test_001" "test-collection" +``` + +## Environment Variables + +- `TRUSTGRAPH_URL`: Default API URL + +## Related Commands + +- [`tg-start-library-processing`](tg-start-library-processing.md) - Start document processing +- [`tg-show-library-documents`](tg-show-library-documents.md) - List library documents +- [`tg-show-flows`](tg-show-flows.md) - Monitor active processing flows +- [`tg-invoke-document-rag`](tg-invoke-document-rag.md) - Verify processed documents + +## API Integration + +This command uses the [Library API](../apis/api-librarian.md) to remove processing records from the document processing system. + +## Best Practices + +1. **Verify Completion**: Ensure processing is complete before removing records +2. **Check Dependencies**: Verify no other processes depend on the processing record +3. **Gradual Cleanup**: Remove processing records gradually to avoid system impact +4. **Monitor Impact**: Watch for any effects of record removal on system performance +5. **Documentation**: Log processing record removals for audit purposes +6. **Backup**: Consider backing up processing metadata before removal +7. **Testing**: Test cleanup procedures in non-production environments + +## Troubleshooting + +### Record Won't Remove +```bash +# Check if processing is actually complete +tg-show-flows | grep "processing-id" + +# Verify API connectivity +curl -s "$TRUSTGRAPH_URL/api/v1/library/processing" > /dev/null +``` + +### Unexpected Behavior After Removal +```bash +# Check if document is still accessible +tg-invoke-document-rag -q "test" -C "collection" + +# Verify document processing status +tg-show-library-documents | grep "document-id" +``` + +### Permission Issues +```bash +# Check user permissions +tg-show-library-documents -U "your-user" + +# Verify processing record ownership +``` \ No newline at end of file