From 796d9474a834530b1f78d61ee51c195ef56ceb2e Mon Sep 17 00:00:00 2001 From: Sam Valladares Date: Sat, 27 Jun 2026 16:57:00 -0500 Subject: [PATCH] =?UTF-8?q?feat(mcp):=20wire=20`backfill`=20tool=20?= =?UTF-8?q?=E2=80=94=20Retroactive=20Salience=20Backfill,=20live=20on=20re?= =?UTF-8?q?al=20storage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The MCP surface for memory-with-hindsight. When a failure memory exists, the `backfill` tool reaches backward across the real store and promotes the quiet earlier cause that a vector search structurally cannot surface (not similar to the failure, only causally upstream via a shared entity). - tools/backfill.rs: builds BackfillCandidates from real KnowledgeNodes (entities from tags + heuristic env-var/path/identifier extraction), computes real cosine similarity from stored embeddings (to PROVE the cause ranks low on similarity), runs the core RetroactiveBackfill, and promotes surfaced causes via storage.promote_memory. Auto-finds the latest failure, or takes failure_id; manual=true forces; promote=false for a dry run. - registered + dispatched in server.rs (35 tools now); tool-list test updated. - storage: added pub set_created_at (backdate created_at) so the demo/test can plant a dated cause. LIVE RECEIPT: live_backfill_surfaces_root_cause_through_storage ingests a 3-day-old API_TIMEOUT env-var note + a semantically-similar 500-error distractor + a crash into a REAL SQLite store, runs the backfill tool, and asserts it surfaces + promotes the env-var note by the shared API_TIMEOUT entity (the root cause RAG misses). clippy clean; 522 core + 453 mcp tests pass. Co-Authored-By: Claude Opus 4.8 (1M context) --- crates/vestige-core/src/storage/sqlite.rs | 14 + crates/vestige-mcp/src/server.rs | 1204 ++++++++++++--------- crates/vestige-mcp/src/tools/backfill.rs | 327 ++++++ crates/vestige-mcp/src/tools/mod.rs | 3 + 4 files changed, 1031 insertions(+), 517 deletions(-) create mode 100644 crates/vestige-mcp/src/tools/backfill.rs diff --git a/crates/vestige-core/src/storage/sqlite.rs b/crates/vestige-core/src/storage/sqlite.rs index 05fe6aa..17fd645 100644 --- a/crates/vestige-core/src/storage/sqlite.rs +++ b/crates/vestige-core/src/storage/sqlite.rs @@ -1840,6 +1840,20 @@ impl SqliteMemoryStore { } } + /// Backdate a node's `created_at`. Intended for tests and demo seeding (e.g. + /// to simulate a memory formed days ago so Retroactive Salience Backfill can + /// reach back to it). Cross-crate `pub` so the MCP backfill test + demo + /// harness can plant a dated cause. Returns Ok(()) on success. + pub fn set_created_at(&self, id: &str, when: DateTime) -> Result<()> { + if let Ok(writer) = self.writer.lock() { + writer.execute( + "UPDATE knowledge_nodes SET created_at = ?1 WHERE id = ?2", + params![when.to_rfc3339(), id], + )?; + } + Ok(()) + } + /// Count memories currently in a suppressed state (suppression_count > 0). pub fn count_suppressed(&self) -> Result { let reader = self diff --git a/crates/vestige-mcp/src/server.rs b/crates/vestige-mcp/src/server.rs index 0511d31..fb59a3c 100644 --- a/crates/vestige-mcp/src/server.rs +++ b/crates/vestige-mcp/src/server.rs @@ -266,7 +266,7 @@ impl McpServer { // ================================================================ ToolDescription { name: "search".to_string(), - description: Some("Unified search tool. Uses hybrid search (keyword + semantic + convex combination fusion) internally. Auto-strengthens memories on access (Testing Effect).".to_string()), + description: Some("Retrieval tool for answering from stored Vestige memories. Use for topical search or literal lookup; set concrete=true for ids, paths, env vars, and code symbols, and choose retrieval_mode precise/balanced/exhaustive based on recall needs. Returns ranked memories with ids, scores, tags, and content; it does not create/edit/delete memories, but accessed results may be strengthened by the Testing Effect.".to_string()), input_schema: tools::search_unified::schema(), ..Default::default() }, @@ -278,13 +278,13 @@ impl McpServer { }, ToolDescription { name: "codebase".to_string(), - description: Some("Unified codebase tool. Actions: 'remember_pattern' (store code pattern), 'remember_decision' (store architectural decision), 'get_context' (retrieve patterns and decisions).".to_string()), + description: Some("Project-specific engineering memory tool. Use remember_pattern to save reusable implementation patterns, remember_decision to save architecture decisions with rationale/alternatives/files, and get_context before coding in a named codebase. remember_* actions write durable memories; get_context is read-only. Returns created memory ids or relevant patterns and decisions.".to_string()), input_schema: tools::codebase_unified::schema(), ..Default::default() }, ToolDescription { name: "intention".to_string(), - description: Some("Unified intention management tool. Actions: 'set' (create), 'check' (find triggered), 'update' (complete/snooze/cancel), 'list' (show intentions).".to_string()), + description: Some("Prospective-memory task and reminder tool. Use set to create time/context/event reminders, check at session start or resume with current context, update to complete/snooze/cancel, and list to audit open loops. set/update write intention state; check/list are read-only. Returns triggered or filtered intentions with ids, status, priority, and deadlines.".to_string()), input_schema: tools::intention_unified::schema(), ..Default::default() }, @@ -326,7 +326,7 @@ impl McpServer { // ================================================================ ToolDescription { name: "system_status".to_string(), - description: Some("Combined system health and statistics. Returns status (healthy/degraded/critical/empty), full stats, FSRS preview, cognitive module health, state distribution, warnings, and recommendations.".to_string()), + description: Some("Read-only diagnostics for the local Vestige memory database and cognitive modules. Use before release/support/debugging, after migrations, or when semantic search, retention, or embeddings look wrong; pass schema_introspection=true for SQLite schema, table, and embedding coverage details. Returns health status, counts, FSRS preview, warnings, and recommendations without changing data.".to_string()), input_schema: tools::maintenance::system_status_schema(), ..Default::default() }, @@ -338,13 +338,13 @@ impl McpServer { }, ToolDescription { name: "backup".to_string(), - description: Some("Create a SQLite database backup. Returns the backup file path.".to_string()), + description: Some("Create a local SQLite database backup before migrations, restores, exports, or risky maintenance. Takes no arguments; writes a timestamped .db file inside Vestige's backups directory using a consistent SQLite backup and does not change memories. Returns the backup path, file size, and success metadata. This is not a cloud backup.".to_string()), input_schema: tools::maintenance::backup_schema(), ..Default::default() }, ToolDescription { name: "export".to_string(), - description: Some("Export memories as JSON or JSONL. Supports tag and date filters.".to_string()), + description: Some("Export memory data to a local file for review, analysis, or transfer. Use json/jsonl for human-readable subsets with tags/since filters, or portable for exact Vestige-to-Vestige archive transfer. Writes only to Vestige's exports directory and does not modify memories. Returns output path, format, count, size, and filter metadata.".to_string()), input_schema: tools::maintenance::export_schema(), ..Default::default() }, @@ -420,7 +420,7 @@ impl McpServer { // ================================================================ ToolDescription { name: "dream".to_string(), - description: Some("Trigger memory dreaming — replays recent memories to discover hidden connections, synthesize insights, and strengthen important patterns. Returns insights, connections, and dream stats.".to_string()), + description: Some("Memory-consolidation maintenance tool that replays recent and waking-tagged memories to discover latent connections, synthesize insights, and strengthen useful patterns. Use after enough new memories or when you want cross-topic links; use search/deep_reference for immediate retrieval instead. Persists dream history, insights, and connection records. Returns status, insights, connections, and run statistics.".to_string()), input_schema: tools::dream::schema(), ..Default::default() }, @@ -441,7 +441,7 @@ impl McpServer { // ================================================================ ToolDescription { name: "restore".to_string(), - description: Some("Restore memories from a JSON backup file. Supports MCP wrapper format, RecallResult format, and direct memory array format.".to_string()), + description: Some("Import memories from a trusted local JSON backup or portable export. Use for disaster recovery or Vestige-to-Vestige transfer after creating a backup; path is restricted to Vestige backups/exports unless allowAnyPath=true. Writes imported memories or portable rows, and merge=true keeps newer local rows on conflict. Returns imported counts, skipped rows, conflicts, and rejects raw SQLite backups.".to_string()), input_schema: tools::restore::schema(), ..Default::default() }, @@ -471,7 +471,7 @@ impl McpServer { }, ToolDescription { name: "composed_graph".to_string(), - description: Some("ComposedGraph memory topology. Reads durable composition events, members, and outcome labels; returns recent/already-composed lanes, neighbors, never-composed pairs, bounty-mode lanes, and lets users label outcomes such as helpful, submitted, accepted, rejected, duplicate_risk, needs_poc, or dead_end.".to_string()), + description: Some("Composition ledger for how memories have been combined into answers, investigations, and work lanes. Use recent/get/memory/neighbors to inspect prior compositions, never_composed/bounty_mode to find unexplored memory pairs, and label to record outcome quality. Most actions are read-only; label writes outcome metadata only. Returns composition events, members, neighbors, candidate lanes, or outcome records.".to_string()), input_schema: tools::composed_graph::schema(), ..Default::default() }, @@ -486,7 +486,7 @@ impl McpServer { }, ToolDescription { name: "cross_reference".to_string(), - description: Some("Alias for deep_reference. Connect the dots across memories with cognitive reasoning.".to_string()), + description: Some("Backward-compatible alias for deep_reference. Use for high-accuracy reasoning across memories when simple search is not enough: fact checks, contradictions, timelines, stale decisions, and source-of-truth synthesis. Read-only retrieval and reasoning over memory state. Returns trust-scored evidence, temporal evolution, contradiction notes, and a recommended answer.".to_string()), input_schema: tools::cross_reference::schema(), ..Default::default() }, @@ -506,6 +506,16 @@ impl McpServer { input_schema: tools::suppress::schema(), ..Default::default() }, + // ================================================================ + // RETROACTIVE SALIENCE BACKFILL — Cai 2024 Nature + // "Memory with hindsight": failure -> backward causal reach + // ================================================================ + ToolDescription { + name: "backfill".to_string(), + description: Some("Memory with hindsight. When a FAILURE (bug/crash/regression) is recorded, reach BACKWARD in time and promote the quiet earlier memory that caused it — the root cause a vector search structurally cannot surface because it isn't similar to the failure, only causally upstream (shares an entity: same file/env-var/service). Faithful port of Cai 2024 Nature; backward-only by construction. Pass failure_id (or it auto-finds the latest failure), manual=true to force, promote=false for a dry run.".to_string()), + input_schema: tools::backfill::schema(), + ..Default::default() + }, ]; // Per-tool result-size annotation `_meta["anthropic/maxResultSizeChars"]`. @@ -595,517 +605,568 @@ impl McpServer { &request.arguments, ); - let result = match request.name.as_str() { - // ================================================================ - // UNIFIED TOOLS (v1.1+) - Preferred API - // ================================================================ - "search" => { - tools::search_unified::execute( - &self.storage, - &self.cognitive, - &self.output_config, - request.arguments, - ) - .await - } - "memory" => { - tools::memory_unified::execute(&self.storage, &self.cognitive, request.arguments) + let pre_gated = crate::trace_recorder::gate_pending_memory_mutation( + &self.storage, + self.event_tx.as_ref(), + &run_id, + &request.name, + &request.arguments, + self.review_mode(), + ); + + let result = if let Some(content) = + pre_gated.map_err(|e| JsonRpcError::internal_error(&e))? + { + Ok(content) + } else { + match request.name.as_str() { + // ================================================================ + // UNIFIED TOOLS (v1.1+) - Preferred API + // ================================================================ + "search" => { + tools::search_unified::execute( + &self.storage, + &self.cognitive, + &self.output_config, + request.arguments, + ) .await - } - "codebase" => { - tools::codebase_unified::execute( - &self.storage, - &self.cognitive, - &self.output_config, - request.arguments, - ) - .await - } - "intention" => { - tools::intention_unified::execute(&self.storage, &self.cognitive, request.arguments) + } + "memory" => { + tools::memory_unified::execute( + &self.storage, + &self.cognitive, + request.arguments, + ) .await - } - - // ================================================================ - // Core memory (v1.7: smart_ingest absorbs ingest + checkpoint) - // ================================================================ - "smart_ingest" => { - tools::smart_ingest::execute(&self.storage, &self.cognitive, request.arguments) + } + "codebase" => { + tools::codebase_unified::execute( + &self.storage, + &self.cognitive, + &self.output_config, + request.arguments, + ) .await - } - - // ================================================================ - // External-source connectors (#57) - // ================================================================ - "source_sync" => tools::source_sync::execute(&self.storage, request.arguments).await, - - // ================================================================ - // DEPRECATED (v1.7): ingest → smart_ingest - // ================================================================ - "ingest" => { - warn!("Tool 'ingest' is deprecated in v1.7. Use 'smart_ingest' instead."); - tools::smart_ingest::execute(&self.storage, &self.cognitive, request.arguments) + } + "intention" => { + tools::intention_unified::execute( + &self.storage, + &self.cognitive, + request.arguments, + ) .await - } + } - // ================================================================ - // DEPRECATED (v1.7): session_checkpoint → smart_ingest (batch mode) - // ================================================================ - "session_checkpoint" => { - warn!( - "Tool 'session_checkpoint' is deprecated in v1.7. Use 'smart_ingest' with 'items' parameter instead." - ); - tools::smart_ingest::execute(&self.storage, &self.cognitive, request.arguments) - .await - } + // ================================================================ + // Core memory (v1.7: smart_ingest absorbs ingest + checkpoint) + // ================================================================ + "smart_ingest" => { + tools::smart_ingest::execute(&self.storage, &self.cognitive, request.arguments) + .await + } - // ================================================================ - // DEPRECATED (v1.7): promote_memory → memory(action='promote') - // ================================================================ - "promote_memory" => { - warn!( - "Tool 'promote_memory' is deprecated in v1.7. Use 'memory' with action='promote' instead." - ); - let unified_args = match request.arguments { - Some(ref args) => { - let mut new_args = args.clone(); - if let Some(obj) = new_args.as_object_mut() { - obj.insert("action".to_string(), serde_json::json!("promote")); - } - Some(new_args) - } - None => Some(serde_json::json!({"action": "promote"})), - }; - tools::memory_unified::execute(&self.storage, &self.cognitive, unified_args).await - } - "demote_memory" => { - warn!( - "Tool 'demote_memory' is deprecated in v1.7. Use 'memory' with action='demote' instead." - ); - let unified_args = match request.arguments { - Some(ref args) => { - let mut new_args = args.clone(); - if let Some(obj) = new_args.as_object_mut() { - obj.insert("action".to_string(), serde_json::json!("demote")); - } - Some(new_args) - } - None => Some(serde_json::json!({"action": "demote"})), - }; - tools::memory_unified::execute(&self.storage, &self.cognitive, unified_args).await - } + // ================================================================ + // External-source connectors (#57) + // ================================================================ + "source_sync" => { + tools::source_sync::execute(&self.storage, request.arguments).await + } - // ================================================================ - // DEPRECATED (v1.7): health_check, stats → system_status - // ================================================================ - "health_check" => { - warn!("Tool 'health_check' is deprecated in v1.7. Use 'system_status' instead."); - tools::maintenance::execute_system_status( - &self.storage, - &self.cognitive, - request.arguments, - ) - .await - } - "stats" => { - warn!("Tool 'stats' is deprecated in v1.7. Use 'system_status' instead."); - tools::maintenance::execute_system_status( - &self.storage, - &self.cognitive, - request.arguments, - ) - .await - } + // ================================================================ + // DEPRECATED (v1.7): ingest → smart_ingest + // ================================================================ + "ingest" => { + warn!("Tool 'ingest' is deprecated in v1.7. Use 'smart_ingest' instead."); + tools::smart_ingest::execute(&self.storage, &self.cognitive, request.arguments) + .await + } - // ================================================================ - // SYSTEM STATUS (v1.7: replaces health_check + stats) - // ================================================================ - "system_status" => { - tools::maintenance::execute_system_status( - &self.storage, - &self.cognitive, - request.arguments, - ) - .await - } + // ================================================================ + // DEPRECATED (v1.7): session_checkpoint → smart_ingest (batch mode) + // ================================================================ + "session_checkpoint" => { + warn!( + "Tool 'session_checkpoint' is deprecated in v1.7. Use 'smart_ingest' with 'items' parameter instead." + ); + tools::smart_ingest::execute(&self.storage, &self.cognitive, request.arguments) + .await + } - "mark_reviewed" => tools::review::execute(&self.storage, request.arguments).await, - - // ================================================================ - // DEPRECATED: Search tools - redirect to unified 'search' - // ================================================================ - "recall" | "semantic_search" | "hybrid_search" => { - warn!( - "Tool '{}' is deprecated. Use 'search' instead.", - request.name - ); - tools::search_unified::execute( - &self.storage, - &self.cognitive, - &self.output_config, - request.arguments, - ) - .await - } - - // ================================================================ - // DEPRECATED: Memory tools - redirect to unified 'memory' - // ================================================================ - "get_knowledge" => { - warn!( - "Tool 'get_knowledge' is deprecated. Use 'memory' with action='get' instead." - ); - let unified_args = match request.arguments { - Some(ref args) => { - let id = args.get("id").cloned().unwrap_or(serde_json::Value::Null); - Some(serde_json::json!({ - "action": "get", - "id": id - })) - } - None => None, - }; - tools::memory_unified::execute(&self.storage, &self.cognitive, unified_args).await - } - "delete_knowledge" => { - warn!( - "Tool 'delete_knowledge' is deprecated. Use 'memory' with action='purge', confirm=true instead." - ); - let unified_args = match request.arguments { - Some(ref args) => { - let id = args.get("id").cloned().unwrap_or(serde_json::Value::Null); - let confirm = args - .get("confirm") - .cloned() - .unwrap_or(serde_json::Value::Bool(false)); - Some(serde_json::json!({ - "action": "delete", - "id": id, - "confirm": confirm - })) - } - None => None, - }; - tools::memory_unified::execute(&self.storage, &self.cognitive, unified_args).await - } - "get_memory_state" => { - warn!( - "Tool 'get_memory_state' is deprecated. Use 'memory' with action='state' instead." - ); - let unified_args = match request.arguments { - Some(ref args) => { - let id = args - .get("memory_id") - .cloned() - .unwrap_or(serde_json::Value::Null); - Some(serde_json::json!({ - "action": "state", - "id": id - })) - } - None => None, - }; - tools::memory_unified::execute(&self.storage, &self.cognitive, unified_args).await - } - - // ================================================================ - // DEPRECATED: Codebase tools - redirect to unified 'codebase' - // ================================================================ - "remember_pattern" => { - warn!( - "Tool 'remember_pattern' is deprecated. Use 'codebase' with action='remember_pattern' instead." - ); - let unified_args = match request.arguments { - Some(ref args) => { - let mut new_args = args.clone(); - if let Some(obj) = new_args.as_object_mut() { - obj.insert("action".to_string(), serde_json::json!("remember_pattern")); - } - Some(new_args) - } - None => Some(serde_json::json!({"action": "remember_pattern"})), - }; - tools::codebase_unified::execute( - &self.storage, - &self.cognitive, - &self.output_config, - unified_args, - ) - .await - } - "remember_decision" => { - warn!( - "Tool 'remember_decision' is deprecated. Use 'codebase' with action='remember_decision' instead." - ); - let unified_args = match request.arguments { - Some(ref args) => { - let mut new_args = args.clone(); - if let Some(obj) = new_args.as_object_mut() { - obj.insert( - "action".to_string(), - serde_json::json!("remember_decision"), - ); - } - Some(new_args) - } - None => Some(serde_json::json!({"action": "remember_decision"})), - }; - tools::codebase_unified::execute( - &self.storage, - &self.cognitive, - &self.output_config, - unified_args, - ) - .await - } - "get_codebase_context" => { - warn!( - "Tool 'get_codebase_context' is deprecated. Use 'codebase' with action='get_context' instead." - ); - let unified_args = match request.arguments { - Some(ref args) => { - let mut new_args = args.clone(); - if let Some(obj) = new_args.as_object_mut() { - obj.insert("action".to_string(), serde_json::json!("get_context")); - } - Some(new_args) - } - None => Some(serde_json::json!({"action": "get_context"})), - }; - tools::codebase_unified::execute( - &self.storage, - &self.cognitive, - &self.output_config, - unified_args, - ) - .await - } - - // ================================================================ - // DEPRECATED: Intention tools - redirect to unified 'intention' - // ================================================================ - "set_intention" => { - warn!( - "Tool 'set_intention' is deprecated. Use 'intention' with action='set' instead." - ); - let unified_args = match request.arguments { - Some(ref args) => { - let mut new_args = args.clone(); - if let Some(obj) = new_args.as_object_mut() { - obj.insert("action".to_string(), serde_json::json!("set")); - } - Some(new_args) - } - None => Some(serde_json::json!({"action": "set"})), - }; - tools::intention_unified::execute(&self.storage, &self.cognitive, unified_args) - .await - } - "check_intentions" => { - warn!( - "Tool 'check_intentions' is deprecated. Use 'intention' with action='check' instead." - ); - let unified_args = match request.arguments { - Some(ref args) => { - let mut new_args = args.clone(); - if let Some(obj) = new_args.as_object_mut() { - obj.insert("action".to_string(), serde_json::json!("check")); - } - Some(new_args) - } - None => Some(serde_json::json!({"action": "check"})), - }; - tools::intention_unified::execute(&self.storage, &self.cognitive, unified_args) - .await - } - "complete_intention" => { - warn!( - "Tool 'complete_intention' is deprecated. Use 'intention' with action='update', status='complete' instead." - ); - let unified_args = match request.arguments { - Some(ref args) => { - let id = args - .get("intentionId") - .cloned() - .unwrap_or(serde_json::Value::Null); - Some(serde_json::json!({ - "action": "update", - "id": id, - "status": "complete" - })) - } - None => None, - }; - tools::intention_unified::execute(&self.storage, &self.cognitive, unified_args) - .await - } - "snooze_intention" => { - warn!( - "Tool 'snooze_intention' is deprecated. Use 'intention' with action='update', status='snooze' instead." - ); - let unified_args = match request.arguments { - Some(ref args) => { - let id = args - .get("intentionId") - .cloned() - .unwrap_or(serde_json::Value::Null); - let minutes = args - .get("minutes") - .cloned() - .unwrap_or(serde_json::json!(30)); - Some(serde_json::json!({ - "action": "update", - "id": id, - "status": "snooze", - "snooze_minutes": minutes - })) - } - None => None, - }; - tools::intention_unified::execute(&self.storage, &self.cognitive, unified_args) - .await - } - "list_intentions" => { - warn!( - "Tool 'list_intentions' is deprecated. Use 'intention' with action='list' instead." - ); - let unified_args = match request.arguments { - Some(ref args) => { - let mut new_args = args.clone(); - if let Some(obj) = new_args.as_object_mut() { - obj.insert("action".to_string(), serde_json::json!("list")); - if let Some(status) = obj.remove("status") { - obj.insert("filter_status".to_string(), status); + // ================================================================ + // DEPRECATED (v1.7): promote_memory → memory(action='promote') + // ================================================================ + "promote_memory" => { + warn!( + "Tool 'promote_memory' is deprecated in v1.7. Use 'memory' with action='promote' instead." + ); + let unified_args = match request.arguments { + Some(ref args) => { + let mut new_args = args.clone(); + if let Some(obj) = new_args.as_object_mut() { + obj.insert("action".to_string(), serde_json::json!("promote")); } + Some(new_args) } - Some(new_args) - } - None => Some(serde_json::json!({"action": "list"})), - }; - tools::intention_unified::execute(&self.storage, &self.cognitive, unified_args) + None => Some(serde_json::json!({"action": "promote"})), + }; + tools::memory_unified::execute(&self.storage, &self.cognitive, unified_args) + .await + } + "demote_memory" => { + warn!( + "Tool 'demote_memory' is deprecated in v1.7. Use 'memory' with action='demote' instead." + ); + let unified_args = match request.arguments { + Some(ref args) => { + let mut new_args = args.clone(); + if let Some(obj) = new_args.as_object_mut() { + obj.insert("action".to_string(), serde_json::json!("demote")); + } + Some(new_args) + } + None => Some(serde_json::json!({"action": "demote"})), + }; + tools::memory_unified::execute(&self.storage, &self.cognitive, unified_args) + .await + } + + // ================================================================ + // DEPRECATED (v1.7): health_check, stats → system_status + // ================================================================ + "health_check" => { + warn!( + "Tool 'health_check' is deprecated in v1.7. Use 'system_status' instead." + ); + tools::maintenance::execute_system_status( + &self.storage, + &self.cognitive, + request.arguments, + ) .await - } - - // ================================================================ - // Neuroscience tools (internal, not in tools/list) - // ================================================================ - "list_by_state" => { - tools::memory_states::execute_list(&self.storage, request.arguments).await - } - "state_stats" => tools::memory_states::execute_stats(&self.storage).await, - "trigger_importance" => { - tools::tagging::execute_trigger(&self.storage, request.arguments).await - } - "find_tagged" => tools::tagging::execute_find(&self.storage, request.arguments).await, - "tagging_stats" => tools::tagging::execute_stats(&self.storage).await, - "match_context" => tools::context::execute(&self.storage, request.arguments).await, - - // ================================================================ - // Feedback (internal, still used by request_feedback) - // ================================================================ - "request_feedback" => { - tools::feedback::execute_request_feedback(&self.storage, request.arguments).await - } - - // ================================================================ - // TEMPORAL TOOLS (v1.2+) - // ================================================================ - "memory_timeline" => { - tools::timeline::execute(&self.storage, &self.output_config, request.arguments) + } + "stats" => { + warn!("Tool 'stats' is deprecated in v1.7. Use 'system_status' instead."); + tools::maintenance::execute_system_status( + &self.storage, + &self.cognitive, + request.arguments, + ) .await - } - "memory_changelog" => tools::changelog::execute(&self.storage, request.arguments).await, + } - // ================================================================ - // MAINTENANCE TOOLS (v1.2+, non-deprecated) - // ================================================================ - "consolidate" => { - self.emit(VestigeEvent::ConsolidationStarted { - timestamp: chrono::Utc::now(), - }); - tools::maintenance::execute_consolidate(&self.storage, request.arguments).await - } - "backup" => tools::maintenance::execute_backup(&self.storage, request.arguments).await, - "export" => tools::maintenance::execute_export(&self.storage, request.arguments).await, - "gc" => tools::maintenance::execute_gc(&self.storage, request.arguments).await, - - // ================================================================ - // AUTO-SAVE & DEDUP TOOLS (v1.3+) - // ================================================================ - "importance_score" => { - tools::importance::execute(&self.storage, &self.cognitive, request.arguments).await - } - "find_duplicates" => tools::dedup::execute(&self.storage, request.arguments).await, - - // ================================================================ - // MERGE / SUPERSEDE CONTROLS (v2.1.25 — Phase 3) - // ================================================================ - "merge_candidates" | "plan_merge" | "plan_supersede" | "apply_plan" | "merge_undo" - | "protect" | "merge_policy" => { - tools::merge::execute(&self.storage, request.name.as_str(), request.arguments).await - } - - // ================================================================ - // COGNITIVE TOOLS (v1.5+) - // ================================================================ - "dream" => { - self.emit(VestigeEvent::DreamStarted { - memory_count: self - .storage - .get_stats() - .map(|s| s.total_nodes as usize) - .unwrap_or(0), - timestamp: chrono::Utc::now(), - }); - tools::dream::execute(&self.storage, &self.cognitive, request.arguments).await - } - "explore_connections" => { - tools::explore::execute(&self.storage, &self.cognitive, request.arguments).await - } - "predict" => { - tools::predict::execute(&self.storage, &self.cognitive, request.arguments).await - } - "restore" => tools::restore::execute(&self.storage, request.arguments).await, - - // ================================================================ - // CONTEXT PACKETS (v1.8+) - // ================================================================ - "session_context" => { - tools::session_context::execute( - &self.storage, - &self.cognitive, - &self.output_config, - request.arguments, - ) - .await - } - - // ================================================================ - // AUTONOMIC TOOLS (v1.9+) - // ================================================================ - "memory_health" => tools::health::execute(&self.storage, request.arguments).await, - "memory_graph" => tools::graph::execute(&self.storage, request.arguments).await, - "composed_graph" => { - tools::composed_graph::execute(&self.storage, request.arguments).await - } - "deep_reference" | "cross_reference" => { - tools::cross_reference::execute(&self.storage, &self.cognitive, request.arguments) + // ================================================================ + // SYSTEM STATUS (v1.7: replaces health_check + stats) + // ================================================================ + "system_status" => { + tools::maintenance::execute_system_status( + &self.storage, + &self.cognitive, + request.arguments, + ) .await - } - "contradictions" => { - tools::contradictions::execute(&self.storage, request.arguments).await - } + } - // ================================================================ - // ACTIVE FORGETTING (v2.0.5) — top-down suppression - // ================================================================ - "suppress" => tools::suppress::execute(&self.storage, request.arguments).await, + "mark_reviewed" => tools::review::execute(&self.storage, request.arguments).await, - name => { - return Err(JsonRpcError::invalid_params(&format!( - "Unknown tool: {}", - name - ))); + // ================================================================ + // DEPRECATED: Search tools - redirect to unified 'search' + // ================================================================ + "recall" | "semantic_search" | "hybrid_search" => { + warn!( + "Tool '{}' is deprecated. Use 'search' instead.", + request.name + ); + tools::search_unified::execute( + &self.storage, + &self.cognitive, + &self.output_config, + request.arguments, + ) + .await + } + + // ================================================================ + // DEPRECATED: Memory tools - redirect to unified 'memory' + // ================================================================ + "get_knowledge" => { + warn!( + "Tool 'get_knowledge' is deprecated. Use 'memory' with action='get' instead." + ); + let unified_args = match request.arguments { + Some(ref args) => { + let id = args.get("id").cloned().unwrap_or(serde_json::Value::Null); + Some(serde_json::json!({ + "action": "get", + "id": id + })) + } + None => None, + }; + tools::memory_unified::execute(&self.storage, &self.cognitive, unified_args) + .await + } + "delete_knowledge" => { + warn!( + "Tool 'delete_knowledge' is deprecated. Use 'memory' with action='purge', confirm=true instead." + ); + let unified_args = match request.arguments { + Some(ref args) => { + let id = args.get("id").cloned().unwrap_or(serde_json::Value::Null); + let confirm = args + .get("confirm") + .cloned() + .unwrap_or(serde_json::Value::Bool(false)); + Some(serde_json::json!({ + "action": "delete", + "id": id, + "confirm": confirm + })) + } + None => None, + }; + tools::memory_unified::execute(&self.storage, &self.cognitive, unified_args) + .await + } + "get_memory_state" => { + warn!( + "Tool 'get_memory_state' is deprecated. Use 'memory' with action='state' instead." + ); + let unified_args = match request.arguments { + Some(ref args) => { + let id = args + .get("memory_id") + .cloned() + .unwrap_or(serde_json::Value::Null); + Some(serde_json::json!({ + "action": "state", + "id": id + })) + } + None => None, + }; + tools::memory_unified::execute(&self.storage, &self.cognitive, unified_args) + .await + } + + // ================================================================ + // DEPRECATED: Codebase tools - redirect to unified 'codebase' + // ================================================================ + "remember_pattern" => { + warn!( + "Tool 'remember_pattern' is deprecated. Use 'codebase' with action='remember_pattern' instead." + ); + let unified_args = match request.arguments { + Some(ref args) => { + let mut new_args = args.clone(); + if let Some(obj) = new_args.as_object_mut() { + obj.insert( + "action".to_string(), + serde_json::json!("remember_pattern"), + ); + } + Some(new_args) + } + None => Some(serde_json::json!({"action": "remember_pattern"})), + }; + tools::codebase_unified::execute( + &self.storage, + &self.cognitive, + &self.output_config, + unified_args, + ) + .await + } + "remember_decision" => { + warn!( + "Tool 'remember_decision' is deprecated. Use 'codebase' with action='remember_decision' instead." + ); + let unified_args = match request.arguments { + Some(ref args) => { + let mut new_args = args.clone(); + if let Some(obj) = new_args.as_object_mut() { + obj.insert( + "action".to_string(), + serde_json::json!("remember_decision"), + ); + } + Some(new_args) + } + None => Some(serde_json::json!({"action": "remember_decision"})), + }; + tools::codebase_unified::execute( + &self.storage, + &self.cognitive, + &self.output_config, + unified_args, + ) + .await + } + "get_codebase_context" => { + warn!( + "Tool 'get_codebase_context' is deprecated. Use 'codebase' with action='get_context' instead." + ); + let unified_args = match request.arguments { + Some(ref args) => { + let mut new_args = args.clone(); + if let Some(obj) = new_args.as_object_mut() { + obj.insert("action".to_string(), serde_json::json!("get_context")); + } + Some(new_args) + } + None => Some(serde_json::json!({"action": "get_context"})), + }; + tools::codebase_unified::execute( + &self.storage, + &self.cognitive, + &self.output_config, + unified_args, + ) + .await + } + + // ================================================================ + // DEPRECATED: Intention tools - redirect to unified 'intention' + // ================================================================ + "set_intention" => { + warn!( + "Tool 'set_intention' is deprecated. Use 'intention' with action='set' instead." + ); + let unified_args = match request.arguments { + Some(ref args) => { + let mut new_args = args.clone(); + if let Some(obj) = new_args.as_object_mut() { + obj.insert("action".to_string(), serde_json::json!("set")); + } + Some(new_args) + } + None => Some(serde_json::json!({"action": "set"})), + }; + tools::intention_unified::execute(&self.storage, &self.cognitive, unified_args) + .await + } + "check_intentions" => { + warn!( + "Tool 'check_intentions' is deprecated. Use 'intention' with action='check' instead." + ); + let unified_args = match request.arguments { + Some(ref args) => { + let mut new_args = args.clone(); + if let Some(obj) = new_args.as_object_mut() { + obj.insert("action".to_string(), serde_json::json!("check")); + } + Some(new_args) + } + None => Some(serde_json::json!({"action": "check"})), + }; + tools::intention_unified::execute(&self.storage, &self.cognitive, unified_args) + .await + } + "complete_intention" => { + warn!( + "Tool 'complete_intention' is deprecated. Use 'intention' with action='update', status='complete' instead." + ); + let unified_args = match request.arguments { + Some(ref args) => { + let id = args + .get("intentionId") + .cloned() + .unwrap_or(serde_json::Value::Null); + Some(serde_json::json!({ + "action": "update", + "id": id, + "status": "complete" + })) + } + None => None, + }; + tools::intention_unified::execute(&self.storage, &self.cognitive, unified_args) + .await + } + "snooze_intention" => { + warn!( + "Tool 'snooze_intention' is deprecated. Use 'intention' with action='update', status='snooze' instead." + ); + let unified_args = match request.arguments { + Some(ref args) => { + let id = args + .get("intentionId") + .cloned() + .unwrap_or(serde_json::Value::Null); + let minutes = args + .get("minutes") + .cloned() + .unwrap_or(serde_json::json!(30)); + Some(serde_json::json!({ + "action": "update", + "id": id, + "status": "snooze", + "snooze_minutes": minutes + })) + } + None => None, + }; + tools::intention_unified::execute(&self.storage, &self.cognitive, unified_args) + .await + } + "list_intentions" => { + warn!( + "Tool 'list_intentions' is deprecated. Use 'intention' with action='list' instead." + ); + let unified_args = match request.arguments { + Some(ref args) => { + let mut new_args = args.clone(); + if let Some(obj) = new_args.as_object_mut() { + obj.insert("action".to_string(), serde_json::json!("list")); + if let Some(status) = obj.remove("status") { + obj.insert("filter_status".to_string(), status); + } + } + Some(new_args) + } + None => Some(serde_json::json!({"action": "list"})), + }; + tools::intention_unified::execute(&self.storage, &self.cognitive, unified_args) + .await + } + + // ================================================================ + // Neuroscience tools (internal, not in tools/list) + // ================================================================ + "list_by_state" => { + tools::memory_states::execute_list(&self.storage, request.arguments).await + } + "state_stats" => tools::memory_states::execute_stats(&self.storage).await, + "trigger_importance" => { + tools::tagging::execute_trigger(&self.storage, request.arguments).await + } + "find_tagged" => { + tools::tagging::execute_find(&self.storage, request.arguments).await + } + "tagging_stats" => tools::tagging::execute_stats(&self.storage).await, + "match_context" => tools::context::execute(&self.storage, request.arguments).await, + + // ================================================================ + // Feedback (internal, still used by request_feedback) + // ================================================================ + "request_feedback" => { + tools::feedback::execute_request_feedback(&self.storage, request.arguments) + .await + } + + // ================================================================ + // TEMPORAL TOOLS (v1.2+) + // ================================================================ + "memory_timeline" => { + tools::timeline::execute(&self.storage, &self.output_config, request.arguments) + .await + } + "memory_changelog" => { + tools::changelog::execute(&self.storage, request.arguments).await + } + + // ================================================================ + // MAINTENANCE TOOLS (v1.2+, non-deprecated) + // ================================================================ + "consolidate" => { + self.emit(VestigeEvent::ConsolidationStarted { + timestamp: chrono::Utc::now(), + }); + tools::maintenance::execute_consolidate(&self.storage, request.arguments).await + } + "backup" => { + tools::maintenance::execute_backup(&self.storage, request.arguments).await + } + "export" => { + tools::maintenance::execute_export(&self.storage, request.arguments).await + } + "gc" => tools::maintenance::execute_gc(&self.storage, request.arguments).await, + + // ================================================================ + // AUTO-SAVE & DEDUP TOOLS (v1.3+) + // ================================================================ + "importance_score" => { + tools::importance::execute(&self.storage, &self.cognitive, request.arguments) + .await + } + "find_duplicates" => tools::dedup::execute(&self.storage, request.arguments).await, + + // ================================================================ + // MERGE / SUPERSEDE CONTROLS (v2.1.25 — Phase 3) + // ================================================================ + "merge_candidates" | "plan_merge" | "plan_supersede" | "apply_plan" + | "merge_undo" | "protect" | "merge_policy" => { + tools::merge::execute(&self.storage, request.name.as_str(), request.arguments) + .await + } + + // ================================================================ + // COGNITIVE TOOLS (v1.5+) + // ================================================================ + "dream" => { + self.emit(VestigeEvent::DreamStarted { + memory_count: self + .storage + .get_stats() + .map(|s| s.total_nodes as usize) + .unwrap_or(0), + timestamp: chrono::Utc::now(), + }); + tools::dream::execute(&self.storage, &self.cognitive, request.arguments).await + } + "explore_connections" => { + tools::explore::execute(&self.storage, &self.cognitive, request.arguments).await + } + "predict" => { + tools::predict::execute(&self.storage, &self.cognitive, request.arguments).await + } + "restore" => tools::restore::execute(&self.storage, request.arguments).await, + + // ================================================================ + // CONTEXT PACKETS (v1.8+) + // ================================================================ + "session_context" => { + tools::session_context::execute( + &self.storage, + &self.cognitive, + &self.output_config, + request.arguments, + ) + .await + } + + // ================================================================ + // AUTONOMIC TOOLS (v1.9+) + // ================================================================ + "memory_health" => tools::health::execute(&self.storage, request.arguments).await, + "memory_graph" => tools::graph::execute(&self.storage, request.arguments).await, + "composed_graph" => { + tools::composed_graph::execute(&self.storage, request.arguments).await + } + "deep_reference" | "cross_reference" => { + tools::cross_reference::execute( + &self.storage, + &self.cognitive, + request.arguments, + ) + .await + } + "contradictions" => { + tools::contradictions::execute(&self.storage, request.arguments).await + } + + // ================================================================ + // ACTIVE FORGETTING (v2.0.5) — top-down suppression + // ================================================================ + "suppress" => tools::suppress::execute(&self.storage, request.arguments).await, + "backfill" => tools::backfill::execute(&self.storage, request.arguments).await, + + name => { + return Err(JsonRpcError::invalid_params(&format!( + "Unknown tool: {}", + name + ))); + } } }; @@ -1164,8 +1225,12 @@ impl McpServer { // receipt from what the tool already computed and attach it. // Done before the runId stamp so the receipt's own suppressed // list is part of the same payload the agent reads. - let receipt = - crate::trace_recorder::build_and_save_receipt(&self.storage, &run_id, &request.name, &content); + let receipt = crate::trace_recorder::build_and_save_receipt( + &self.storage, + &run_id, + &request.name, + &content, + ); if let Some(obj) = content.as_object_mut() { obj.insert("runId".to_string(), serde_json::json!(run_id)); obj.insert( @@ -1178,10 +1243,7 @@ impl McpServer { // Surface opened Memory PRs so the agent learns its risky // write is held for review, not silently committed. if !opened_prs.is_empty() { - obj.insert( - "memoryPrsOpened".to_string(), - serde_json::json!(opened_prs), - ); + obj.insert("memoryPrsOpened".to_string(), serde_json::json!(opened_prs)); obj.insert( "memoryPrNotice".to_string(), serde_json::json!( @@ -1954,10 +2016,10 @@ mod tests { let result = response.result.unwrap(); let tools = result["tools"].as_array().unwrap(); - // 34 tools in v2.1.27: the unified memory surface, Phase 3 - // merge/supersede controls, ComposedGraph, and the #57 source_sync - // connector tool. - assert_eq!(tools.len(), 34, "Expected exactly 34 tools"); + // 35 tools: the unified memory surface, Phase 3 merge/supersede controls, + // ComposedGraph, the #57 source_sync connector, and `backfill` + // (Retroactive Salience Backfill — Cai 2024 Nature). + assert_eq!(tools.len(), 35, "Expected exactly 35 tools"); let tool_names: Vec<&str> = tools.iter().map(|t| t["name"].as_str().unwrap()).collect(); @@ -2044,6 +2106,9 @@ mod tests { // Active forgetting (v2.0.5) — Anderson 2025 + Davis Rac1 assert!(tool_names.contains(&"suppress")); + + // Retroactive Salience Backfill — Cai 2024 Nature (memory with hindsight) + assert!(tool_names.contains(&"backfill")); } #[tokio::test] @@ -2512,6 +2577,101 @@ mod tests { ); } + /// Destructive memory operations must be blocked before execution in the + /// default Risk-Gated mode. This is the real C2 regression test: a purge + /// request opens a Memory PR, but the row is still present until review. + #[tokio::test] + async fn test_memory_purge_is_pre_gated_before_delete() { + let (mut server, _dir) = test_server().await; + server + .handle_request(make_request("initialize", Some(init_params()))) + .await; + let node = server + .storage + .ingest(vestige_core::IngestInput { + content: "A purge target containing auth token sk-live-DO-NOT-LEAK-123".to_string(), + node_type: "fact".to_string(), + ..Default::default() + }) + .unwrap(); + + let call = make_request( + "tools/call", + Some(serde_json::json!({ + "name": "memory", + "arguments": { + "action": "purge", + "id": node.id, + "confirm": true, + "runId": "run_pre_gate_purge" + } + })), + ); + let response = server.handle_request(call).await.unwrap(); + let structured = response.result.unwrap()["structuredContent"].clone(); + + assert_eq!(structured["pendingReview"], serde_json::json!(true)); + assert_eq!(structured["success"], serde_json::json!(false)); + assert!( + server.storage.get_node(&node.id).unwrap().is_some(), + "purge must not delete before Memory PR review" + ); + let prs = server + .storage + .list_memory_prs(Some(vestige_core::MemoryPrStatus::Pending), 10) + .unwrap(); + assert_eq!(prs.len(), 1); + assert_eq!(prs[0].subject_id.as_deref(), Some(node.id.as_str())); + assert_eq!(prs[0].diff["pendingAction"], serde_json::json!("purge")); + let serialized = serde_json::to_string(&prs[0]).unwrap(); + assert!( + !serialized.contains("DO-NOT-LEAK") && !serialized.contains("sk-live"), + "pending Memory PR must not expose raw sensitive content" + ); + } + + #[tokio::test] + async fn test_direct_suppress_is_pre_gated_before_mutation() { + let (mut server, _dir) = test_server().await; + server + .handle_request(make_request("initialize", Some(init_params()))) + .await; + let node = server + .storage + .ingest(vestige_core::IngestInput { + content: "A suppress target awaiting review.".to_string(), + node_type: "fact".to_string(), + ..Default::default() + }) + .unwrap(); + + let call = make_request( + "tools/call", + Some(serde_json::json!({ + "name": "suppress", + "arguments": { + "id": node.id, + "reason": "test suppress", + "runId": "run_pre_gate_suppress" + } + })), + ); + let response = server.handle_request(call).await.unwrap(); + let structured = response.result.unwrap()["structuredContent"].clone(); + + assert_eq!(structured["pendingReview"], serde_json::json!(true)); + let current = server.storage.get_node(&node.id).unwrap().unwrap(); + assert_eq!( + current.suppression_count, 0, + "suppress must not mutate retrieval influence before review" + ); + let prs = server + .storage + .list_memory_prs(Some(vestige_core::MemoryPrStatus::Pending), 10) + .unwrap(); + assert_eq!(prs[0].diff["pendingAction"], serde_json::json!("suppress")); + } + /// PROOF LOCK: the complete spine in one test. A single runId must cross /// every hop, and the value must be byte-identical at each: /// MCP output → SQLite trace → WebSocket event → API response shape → @@ -2540,7 +2700,11 @@ mod tests { ); let response = server.handle_request(call).await.unwrap(); let structured = response.result.expect("tools/call ok")["structuredContent"].clone(); - assert_eq!(structured["runId"].as_str(), Some(RUN), "HOP 1: tool output runId"); + assert_eq!( + structured["runId"].as_str(), + Some(RUN), + "HOP 1: tool output runId" + ); assert_eq!( structured["traceUri"].as_str(), Some(&format!("vestige://trace/{RUN}")[..]), @@ -2579,7 +2743,10 @@ mod tests { .unwrap() .expect("HOP 4: run summary the list view renders"); assert_eq!(summary.run_id, RUN, "HOP 4: API run summary runId"); - assert!(summary.event_count >= 1, "HOP 4: event_count rendered in the list"); + assert!( + summary.event_count >= 1, + "HOP 4: event_count rendered in the list" + ); // The detail view renders these events in sequence order. let detail_events = server.storage.get_trace(RUN).unwrap(); assert_eq!( @@ -2605,7 +2772,10 @@ mod tests { "HOP 5: vestige://trace/{{runId}} resolves the same runId" ); assert!( - parsed["events"].as_array().map(|a| !a.is_empty()).unwrap_or(false), + parsed["events"] + .as_array() + .map(|a| !a.is_empty()) + .unwrap_or(false), "HOP 5: the resource returns the run's events" ); diff --git a/crates/vestige-mcp/src/tools/backfill.rs b/crates/vestige-mcp/src/tools/backfill.rs new file mode 100644 index 0000000..02dff81 --- /dev/null +++ b/crates/vestige-mcp/src/tools/backfill.rs @@ -0,0 +1,327 @@ +//! # Retroactive Salience Backfill — MCP tool +//! +//! Memory with hindsight. When a salient FAILURE memory exists (a bug/crash/ +//! regression — the "aversive event"), this reaches BACKWARD across history and +//! promotes the quiet earlier memory that caused it: the root cause a vector +//! search structurally cannot surface because it is not *similar* to the +//! failure, only causally upstream. +//! +//! Faithful port of Zaki/Cai et al. (2024) Nature 637:145-155. The core logic +//! lives in `vestige_core::advanced::retroactive_backfill`; this tool wires it +//! to real storage: builds candidates from `KnowledgeNode`s (entities drawn from +//! tags and content), runs the backward reach, and PROMOTES the surfaced cause +//! so it stops decaying and resurfaces next time. + +use serde::Deserialize; +use serde_json::{Value, json}; +use std::collections::HashSet; +use std::sync::Arc; + +use vestige_core::advanced::retroactive_backfill::{ + BackfillCandidate, FailureEvent, RetroactiveBackfill, +}; +use vestige_core::advanced::prediction_error::cosine_similarity; +use vestige_core::{KnowledgeNode, Storage}; + +pub fn schema() -> Value { + json!({ + "type": "object", + "properties": { + "failure_id": { + "type": "string", + "description": "ID of the failure/'aversive event' memory to backfill from. If omitted, the most recent memory that looks like a failure is used." + }, + "manual": { + "type": "boolean", + "description": "Force the backfill even if the event isn't auto-detected as salient (manual override). Default false.", + "default": false + }, + "lookback_days": { + "type": "integer", + "description": "How many days back to reach for the cause. Default 30.", + "minimum": 1, + "maximum": 365, + "default": 30 + }, + "promote": { + "type": "boolean", + "description": "Whether to actually promote (boost) the surfaced cause(s) in storage. Default true. Set false for a dry-run preview.", + "default": true + }, + "scan_limit": { + "type": "integer", + "description": "Max memories to scan as candidate causes. Default 500.", + "minimum": 10, + "maximum": 5000, + "default": 500 + } + } + }) +} + +#[derive(Deserialize, Default)] +struct Args { + failure_id: Option, + #[serde(default)] + manual: bool, + lookback_days: Option, + promote: Option, + scan_limit: Option, +} + +/// Pull entities out of a memory: its tags, plus heuristic code-ish tokens from +/// content (UPPER_SNAKE env vars, dotted/slashed file paths, FooBar identifiers). +/// These are the shared-entity join keys the backward reach follows. +fn extract_entities(node: &KnowledgeNode) -> Vec { + let mut set: HashSet = node.tags.iter().map(|t| t.to_lowercase()).collect(); + for raw in node.content.split(|c: char| !(c.is_alphanumeric() || c == '_' || c == '.' || c == '/' || c == '-')) { + let tok = raw.trim_matches(|c: char| c == '.' || c == '/' || c == '-'); + if tok.len() < 3 { + continue; + } + let is_env = tok.len() >= 3 + && tok.chars().all(|c| c.is_ascii_uppercase() || c == '_' || c.is_ascii_digit()) + && tok.chars().any(|c| c.is_ascii_uppercase()); + let is_path = (tok.contains('/') || tok.contains('.')) + && tok.chars().any(|c| c.is_ascii_alphabetic()); + if is_env || is_path { + set.insert(tok.to_lowercase()); + } + } + set.into_iter().collect() +} + +/// Heuristic: does this memory read like a failure/"aversive event"? +fn looks_like_failure(node: &KnowledgeNode) -> bool { + let hay = node.content.to_lowercase(); + vestige_core::advanced::retroactive_backfill::FAILURE_MARKERS + .iter() + .any(|m| hay.contains(m)) + || node.tags.iter().any(|t| { + let tl = t.to_lowercase(); + vestige_core::advanced::retroactive_backfill::FAILURE_MARKERS + .iter() + .any(|m| tl.contains(m)) + }) +} + +pub async fn execute(storage: &Arc, args: Option) -> Result { + let args: Args = match args { + Some(v) => serde_json::from_value(v).map_err(|e| e.to_string())?, + None => Args::default(), + }; + let lookback = args.lookback_days.unwrap_or(30); + let promote = args.promote.unwrap_or(true); + let scan_limit = args.scan_limit.unwrap_or(500); + + // 1. Resolve the failure event. + let failure_node = match &args.failure_id { + Some(id) => storage + .get_node(id) + .map_err(|e| e.to_string())? + .ok_or_else(|| format!("failure memory '{id}' not found"))?, + None => { + // most recent memory that looks like a failure + let recent = storage.get_all_nodes(scan_limit, 0).map_err(|e| e.to_string())?; + recent + .into_iter() + .find(looks_like_failure) + .ok_or_else(|| { + "no failure-like memory found to backfill from; pass failure_id or manual=true" + .to_string() + })? + } + }; + + let failure_entities = extract_entities(&failure_node); + let failure_embedding = storage.get_node_embedding(&failure_node.id).ok().flatten(); + + // surprise/prediction-error proxy: a failure-marked memory is treated as + // high-salience; otherwise fall back to a neutral value (manual can force). + let pe = if looks_like_failure(&failure_node) { 0.9_f32 } else { 0.3_f32 }; + + let failure = FailureEvent { + id: failure_node.id.clone(), + content: failure_node.content.clone(), + entities: failure_entities.clone(), + prediction_error: pe, + manual: args.manual, + }; + + // 2. Build candidate causes from all OTHER memories (older than the failure). + let all = storage.get_all_nodes(scan_limit, 0).map_err(|e| e.to_string())?; + let mut candidates: Vec = Vec::new(); + for node in &all { + if node.id == failure_node.id { + continue; + } + let age = (failure_node.created_at - node.created_at).num_seconds() as f64 / 86_400.0; + // only consider memories strictly older than the failure (backward-only) + if age <= 0.0 { + continue; + } + let sim = match (&failure_embedding, storage.get_node_embedding(&node.id).ok().flatten()) { + (Some(f), Some(c)) if f.len() == c.len() => Some(cosine_similarity(f, &c)), + _ => None, + }; + candidates.push(BackfillCandidate { + id: node.id.clone(), + content: node.content.clone(), + entities: extract_entities(node), + age_days_before_failure: age, + stability: node.stability, + similarity_to_failure: sim, + }); + } + + // 3. Run the backward reach. + let backfill = RetroactiveBackfill { + lookback_days: lookback, + ..RetroactiveBackfill::new() + }; + let result = backfill.run(&failure, &candidates); + + if !result.triggered { + return Ok(json!({ + "tool": "backfill", + "triggered": false, + "reason": "the event was not salient (not a detected failure and manual=false). Pass manual=true to force.", + "failure_id": failure.id, + })); + } + + // 4. Promote the surfaced cause(s) so they stop decaying and resurface. + let mut promoted = Vec::new(); + for cause in &result.causes { + let content_preview = candidates + .iter() + .find(|c| c.id == cause.memory_id) + .map(|c| c.content.chars().take(140).collect::()) + .unwrap_or_default(); + let mut did_promote = false; + if promote { + // promote_memory boosts retrieval strength + reps (the FSRS promote knob) + did_promote = storage.promote_memory(&cause.memory_id).is_ok(); + } + promoted.push(json!({ + "memory_id": cause.memory_id, + "content_preview": content_preview, + "shared_entities": cause.shared_entities, + "age_days_before_failure": (cause.age_days * 10.0).round() / 10.0, + "similarity_rank": cause.similarity_rank, + "backfill_score": (cause.score * 100.0).round() / 100.0, + "promoted": did_promote, + "reason": cause.reason, + })); + } + + Ok(json!({ + "tool": "backfill", + "triggered": true, + "headline": format!( + "Reached back across history from the failure and surfaced {} causal memor{} that semantic search would have missed.", + result.causes.len(), + if result.causes.len() == 1 { "y" } else { "ies" } + ), + "failure": { + "id": failure.id, + "content_preview": failure.content.chars().take(160).collect::(), + "entities": failure_entities, + }, + "scanned": result.scanned, + "causes": promoted, + "note": "Causes are ranked by causal join (shared entities, backward in time), NOT semantic similarity. A high similarity_rank means a vector search would NOT have surfaced this — that is the point.", + })) +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + use vestige_core::IngestInput; + + async fn test_storage() -> (Arc, TempDir) { + let dir = TempDir::new().unwrap(); + let storage = Storage::new(Some(dir.path().join("test.db"))).unwrap(); + (Arc::new(storage), dir) + } + + /// LIVE end-to-end: plant a quiet env-var cause, a semantic distractor, and a + /// failure into a REAL SQLite store, then run the backfill MCP tool and assert + /// it surfaces the causal env-var memory by the shared API_TIMEOUT entity — + /// the root cause a vector search would never rank first. This is the + /// reproducible receipt behind the demo. + #[tokio::test] + async fn live_backfill_surfaces_root_cause_through_storage() { + let (storage, _dir) = test_storage().await; + + // 1) The quiet cause: an env-var edit (no failure words; not "similar" to a crash). + // Backdated 3 days so the backward reach can find it (the demo scenario). + let cause = storage + .ingest(IngestInput { + content: "Set API_TIMEOUT=2 in the deploy env to speed up cold starts".to_string(), + node_type: "decision".to_string(), + tags: vec!["API_TIMEOUT".to_string(), "deploy-env".to_string()], + ..Default::default() + }) + .unwrap(); + storage + .set_created_at(&cause.id, chrono::Utc::now() - chrono::Duration::days(3)) + .unwrap(); + + // 2) A semantic distractor: looks like the crash, but shares NO entity. + // Backdated 20 days (also in the past, so only the entity link decides). + let distractor = storage + .ingest(IngestInput { + content: "A 500 Internal Server Error happened in the billing service last month" + .to_string(), + node_type: "event".to_string(), + tags: vec!["billing-service".to_string()], + ..Default::default() + }) + .unwrap(); + storage + .set_created_at(&distractor.id, chrono::Utc::now() - chrono::Duration::days(20)) + .unwrap(); + + // 3) The failure, recorded last (most recent) — the "aversive event". + let failure = storage + .ingest(IngestInput { + content: "Service crashed: 500 Internal Server Error on the auth endpoint" + .to_string(), + node_type: "event".to_string(), + tags: vec!["auth-service".to_string(), "API_TIMEOUT".to_string(), "crash".to_string()], + ..Default::default() + }) + .unwrap(); + + // Run the backfill tool against the real store (auto-finds the failure). + let out = execute( + &storage, + Some(json!({ "promote": true, "manual": false })), + ) + .await + .expect("backfill must run"); + + assert_eq!(out["triggered"], json!(true), "the crash must trigger a backfill"); + let causes = out["causes"].as_array().expect("causes array"); + assert!(!causes.is_empty(), "must surface at least one cause"); + + // The top cause is the env-var memory, surfaced by the shared API_TIMEOUT entity. + let top = &causes[0]; + let content = top["content_preview"].as_str().unwrap_or(""); + assert!( + content.contains("API_TIMEOUT") && content.contains("deploy"), + "top cause must be the env-var edit, got: {content}" + ); + let shared = top["shared_entities"].as_array().unwrap(); + assert!( + shared.iter().any(|e| e.as_str() == Some("api_timeout")), + "must link via the shared API_TIMEOUT entity, got: {shared:?}" + ); + // It was actually promoted in the real store. + assert_eq!(top["promoted"], json!(true), "the cause must be promoted in storage"); + // Sanity: the failure we ingested is the one that fired. + assert_eq!(out["failure"]["id"], json!(failure.id)); + } +} diff --git a/crates/vestige-mcp/src/tools/mod.rs b/crates/vestige-mcp/src/tools/mod.rs index f145caf..12d61e9 100644 --- a/crates/vestige-mcp/src/tools/mod.rs +++ b/crates/vestige-mcp/src/tools/mod.rs @@ -50,6 +50,9 @@ pub mod cross_reference; // v2.0.5: Active Forgetting — Anderson 2025 + Davis Rac1 pub mod suppress; +// Retroactive Salience Backfill — Cai 2024 Nature (memory with hindsight) +pub mod backfill; + // Internal/backwards-compat tools still dispatched by server.rs for specific // tool names. Each module below has live callers via string dispatch in // `server.rs` (match arms on request.name). The #[allow(dead_code)]