feat: Vestige v1.5.0 — Cognitive Engine, memory dreaming, graph exploration, predictive retrieval

28-module CognitiveEngine with full neuroscience pipeline on every tool call.
FSRS-6 now fully automatic: periodic consolidation (6h timer + inline every
100 tool calls), real retrievability formula, episodic-to-semantic auto-merge,
cross-memory reinforcement, Park et al. triple retrieval scoring, ACT-R
base-level activation, personalized w20 optimization.

New tools (19 → 23):
- dream: memory consolidation via replay, discovers hidden connections
- explore_connections: graph traversal (chain, associations, bridges)
- predict: proactive retrieval based on context and activity patterns
- restore: memory restore from JSON backups

All existing tools upgraded with cognitive pre/post processing pipelines.
33 files changed, ~4,100 lines added.
This commit is contained in:
Sam Valladares 2026-02-18 23:34:15 -06:00
parent 3fce1f0b70
commit 927f41c3e4
34 changed files with 4302 additions and 266 deletions

View file

@ -3,13 +3,27 @@
//! Merges recall, semantic_search, and hybrid_search into a single `search` tool.
//! Always uses hybrid search internally (keyword + semantic + RRF fusion).
//! Implements Testing Effect (Roediger & Karpicke 2006) by auto-strengthening memories on access.
//!
//! v1.5.0: Enhanced 7-stage cognitive pipeline:
//! 1. Reranker (over-fetch 3x, rerank down)
//! 2. Temporal boosting (recency + validity)
//! 3. Memory state accessibility filtering
//! 4. Context matching (topic overlap)
//! 5. Spreading activation associations
//! 6. Predictive memory recording
//! 7. Reconsolidation (mark labile)
use chrono::Utc;
use serde::Deserialize;
use serde_json::Value;
use std::sync::Arc;
use tokio::sync::Mutex;
use vestige_core::Storage;
use crate::cognitive::CognitiveEngine;
use vestige_core::{
CompetitionCandidate, EncodingContext, MemoryLifecycle, MemorySnapshot, MemoryState, Storage,
TopicalContext,
};
/// Input schema for unified search tool
pub fn schema() -> Value {
@ -46,6 +60,11 @@ pub fn schema() -> Value {
"description": "Level of detail in results. 'brief' = id/type/tags/score only (saves tokens). 'summary' = default 8-field response. 'full' = all fields including FSRS state and timestamps.",
"enum": ["brief", "summary", "full"],
"default": "summary"
},
"context_topics": {
"type": "array",
"items": { "type": "string" },
"description": "Optional topics for context-dependent retrieval boosting"
}
},
"required": ["query"]
@ -61,14 +80,24 @@ struct SearchArgs {
min_similarity: Option<f32>,
#[serde(alias = "detail_level")]
detail_level: Option<String>,
context_topics: Option<Vec<String>>,
}
/// Execute unified search
/// Execute unified search with 7-stage cognitive pipeline.
///
/// Uses hybrid search (keyword + semantic + RRF fusion) internally.
/// Auto-strengthens memories on access (Testing Effect - Roediger & Karpicke 2006).
/// Pipeline:
/// 1. Hybrid search (keyword + semantic + RRF) with 3x over-fetch
/// 2. Reranker (BM25-like rescoring, trim to limit)
/// 3. Temporal boosting (recency + validity windows)
/// 4. Memory state accessibility filtering (Active/Dormant/Silent/Unavailable)
/// 5. Context matching (topic overlap boosting)
/// 6. Spreading activation (find associated memories)
/// 7. Side effects: predictive memory recording + reconsolidation labile marking
///
/// Also applies Testing Effect (Roediger & Karpicke 2006) by auto-strengthening on access.
pub async fn execute(
storage: &Arc<Mutex<Storage>>,
cognitive: &Arc<Mutex<CognitiveEngine>>,
args: Option<Value>,
) -> Result<Value, String> {
let args: SearchArgs = match args {
@ -102,22 +131,23 @@ pub async fn execute(
let keyword_weight = 0.5_f32;
let semantic_weight = 0.5_f32;
let storage = storage.lock().await;
// ====================================================================
// STAGE 1: Hybrid search with 3x over-fetch for reranking pool
// ====================================================================
let overfetch_limit = (limit * 3).min(100); // Cap at 100 to avoid excessive DB load
let storage_guard = storage.lock().await;
// Execute hybrid search
let results = storage
.hybrid_search(&args.query, limit, keyword_weight, semantic_weight)
let results = storage_guard
.hybrid_search(&args.query, overfetch_limit, keyword_weight, semantic_weight)
.map_err(|e| e.to_string())?;
// Filter results by min_retention and min_similarity
let filtered_results: Vec<_> = results
// Filter by min_retention and min_similarity first (cheap filters)
let mut filtered_results: Vec<_> = results
.into_iter()
.filter(|r| {
// Check retention strength
if r.node.retention_strength < min_retention {
return false;
}
// Check similarity if semantic score is available
if let Some(sem_score) = r.semantic_score
&& sem_score < min_similarity
{
@ -127,24 +157,254 @@ pub async fn execute(
})
.collect();
// Auto-strengthen memories on access (Testing Effect - Roediger & Karpicke 2006)
// This implements "use it or lose it" - accessed memories get stronger
let ids: Vec<&str> = filtered_results.iter().map(|r| r.node.id.as_str()).collect();
let _ = storage.strengthen_batch_on_access(&ids); // Ignore errors, don't fail search
// ====================================================================
// STAGE 2: Reranker (BM25-like rescoring, trim to requested limit)
// ====================================================================
if let Ok(cog) = cognitive.try_lock() {
let candidates: Vec<_> = filtered_results
.iter()
.map(|r| (r.clone(), r.node.content.clone()))
.collect();
// Format results based on detail_level
if let Ok(reranked) = cog.reranker.rerank(&args.query, candidates, Some(limit as usize)) {
// Replace filtered_results with reranked items (preserves original SearchResult)
filtered_results = reranked.into_iter().map(|rr| rr.item).collect();
} else {
// Reranker failed — fall back to original order, just truncate
filtered_results.truncate(limit as usize);
}
} else {
// Couldn't acquire cognitive lock — truncate to limit
filtered_results.truncate(limit as usize);
}
// ====================================================================
// STAGE 3: Temporal boosting (recency + validity windows)
// ====================================================================
if let Ok(cog) = cognitive.try_lock() {
for result in &mut filtered_results {
let recency = cog.temporal_searcher.recency_boost(result.node.created_at);
let validity = cog.temporal_searcher.validity_boost(
result.node.valid_from,
result.node.valid_until,
None,
);
// Blend: 85% relevance + 15% temporal signal
let temporal_factor = recency * validity;
result.combined_score =
result.combined_score * 0.85 + (result.combined_score * temporal_factor as f32) * 0.15;
}
}
// ====================================================================
// STAGE 4: Memory state accessibility filtering
// ====================================================================
if let Ok(cog) = cognitive.try_lock() {
for result in &mut filtered_results {
// Build a MemoryLifecycle from node data for the calculator
let mut lifecycle = MemoryLifecycle::new();
lifecycle.last_access = result.node.last_accessed;
lifecycle.access_count = result.node.reps as u32;
// Determine state from retention strength
lifecycle.state = if result.node.retention_strength > 0.7 {
MemoryState::Active
} else if result.node.retention_strength > 0.3 {
MemoryState::Dormant
} else if result.node.retention_strength > 0.1 {
MemoryState::Silent
} else {
MemoryState::Unavailable
};
let adjusted = cog
.accessibility_calc
.calculate(&lifecycle, result.combined_score as f64);
result.combined_score = adjusted as f32;
}
}
// ====================================================================
// STAGE 5: Context matching (Tulving 1973 encoding specificity)
// ====================================================================
if let Some(ref topics) = args.context_topics {
if !topics.is_empty() {
let retrieval_ctx = EncodingContext::new()
.with_topical(TopicalContext::with_topics(topics.clone()));
if let Ok(cog) = cognitive.try_lock() {
for result in &mut filtered_results {
// Build encoding context from memory's tags
let encoding_ctx = EncodingContext::new()
.with_topical(TopicalContext::with_topics(result.node.tags.clone()));
let context_score = cog.context_matcher.match_contexts(&encoding_ctx, &retrieval_ctx);
// Blend: context match boosts relevance up to +30%
result.combined_score *= 1.0 + (context_score as f32 * 0.3);
}
}
}
}
// Context reinstatement for top result (helps Claude understand WHY this memory matched)
let reinstatement_info: Option<Value> = if let Ok(cog) = cognitive.try_lock() {
if let Some(first) = filtered_results.first() {
let current_ctx = if let Some(ref topics) = args.context_topics {
EncodingContext::new().with_topical(TopicalContext::with_topics(topics.clone()))
} else {
EncodingContext::new()
};
let reinstatement = cog.context_matcher.reinstate_context(&first.node.id, &current_ctx);
Some(serde_json::json!({
"memoryId": reinstatement.memory_id,
"temporalHint": reinstatement.temporal_hint,
"topicalHint": reinstatement.topical_hint,
"sessionHint": reinstatement.session_hint,
"relatedMemories": reinstatement.related_memories,
}))
} else {
None
}
} else {
None
};
// ====================================================================
// STAGE 5B: Retrieval competition (Anderson et al. 1994)
// ====================================================================
let mut suppressed_count = 0_usize;
if filtered_results.len() > 1 {
if let Ok(mut cog) = cognitive.try_lock() {
let candidates: Vec<CompetitionCandidate> = filtered_results
.iter()
.map(|r| CompetitionCandidate {
memory_id: r.node.id.clone(),
relevance_score: r.combined_score as f64,
similarity_to_query: r.semantic_score.unwrap_or(0.0) as f64,
})
.collect();
if let Some(result) = cog.competition_mgr.run_competition(&candidates, 0.7) {
// Apply suppression: losers get penalized
for suppressed_id in &result.suppressed_ids {
if let Some(r) = filtered_results.iter_mut().find(|r| &r.node.id == suppressed_id) {
r.combined_score *= 0.85; // 15% suppression penalty
suppressed_count += 1;
}
}
}
}
}
// Re-sort by adjusted combined_score (descending) after all score modifications
filtered_results.sort_by(|a, b| {
b.combined_score
.partial_cmp(&a.combined_score)
.unwrap_or(std::cmp::Ordering::Equal)
});
// ====================================================================
// STAGE 6: Spreading activation (find associated memories)
// ====================================================================
let associations: Vec<Value> = if let Ok(mut cog) = cognitive.try_lock() {
if let Some(first) = filtered_results.first() {
let activated = cog.activation_network.activate(&first.node.id, 1.0);
activated
.iter()
.take(3)
.map(|a| {
serde_json::json!({
"memoryId": a.memory_id,
"activation": a.activation,
"distance": a.distance,
})
})
.collect()
} else {
vec![]
}
} else {
vec![]
};
// ====================================================================
// Auto-strengthen on access (Testing Effect)
// ====================================================================
let ids: Vec<&str> = filtered_results.iter().map(|r| r.node.id.as_str()).collect();
let _ = storage_guard.strengthen_batch_on_access(&ids);
// Drop storage lock before acquiring cognitive for side effects
drop(storage_guard);
// ====================================================================
// STAGE 7: Side effects — predictive memory + reconsolidation
// ====================================================================
if let Ok(mut cog) = cognitive.try_lock() {
// 7A. Record query for predictive memory
let _ = cog.predictive_memory.record_query(&args.query, &[]);
// 7B. Record each accessed memory for predictive/speculative models
for result in &filtered_results {
let _ = cog.predictive_memory.record_memory_access(
&result.node.id,
&result.node.content.chars().take(100).collect::<String>(),
&result.node.tags,
);
cog.speculative_retriever.record_access(
&result.node.id,
None, // file_context
Some(args.query.as_str()), // query_context
None, // was_helpful (unknown yet)
);
// 7C. Mark labile for reconsolidation window (5 min)
let snapshot = MemorySnapshot {
content: result.node.content.clone(),
tags: result.node.tags.clone(),
retention_strength: result.node.retention_strength,
storage_strength: result.node.storage_strength,
retrieval_strength: result.node.retrieval_strength,
connection_ids: vec![],
captured_at: Utc::now(),
};
cog.reconsolidation.mark_labile(&result.node.id, snapshot);
}
}
// ====================================================================
// Format and return
// ====================================================================
let formatted: Vec<Value> = filtered_results
.iter()
.map(|r| format_search_result(r, detail_level))
.collect();
Ok(serde_json::json!({
// Check learning mode via attention signal
let learning_mode = cognitive.try_lock().ok().map(|cog| cog.attention_signal.is_learning_mode()).unwrap_or(false);
let mut response = serde_json::json!({
"query": args.query,
"method": "hybrid",
"method": "hybrid+cognitive",
"detailLevel": detail_level,
"total": formatted.len(),
"results": formatted,
}))
});
// Include associations if any were found
if !associations.is_empty() {
response["associations"] = serde_json::json!(associations);
}
// Include context reinstatement if computed
if let Some(ri) = reinstatement_info {
response["contextReinstatement"] = ri;
}
// Include competition stats
if suppressed_count > 0 {
response["competitionSuppressed"] = serde_json::json!(suppressed_count);
}
// Include learning mode detection
if learning_mode {
response["learningModeDetected"] = serde_json::json!(true);
}
Ok(response)
}
/// Format a search result based on the requested detail level.
@ -247,9 +507,14 @@ pub fn format_node(node: &vestige_core::KnowledgeNode, detail_level: &str) -> Va
#[cfg(test)]
mod tests {
use super::*;
use crate::cognitive::CognitiveEngine;
use tempfile::TempDir;
use vestige_core::IngestInput;
fn test_cognitive() -> Arc<Mutex<CognitiveEngine>> {
Arc::new(Mutex::new(CognitiveEngine::new()))
}
/// Create a test storage instance with a temporary database
async fn test_storage() -> (Arc<Mutex<Storage>>, TempDir) {
let dir = TempDir::new().unwrap();
@ -282,7 +547,7 @@ mod tests {
async fn test_search_empty_query_fails() {
let (storage, _dir) = test_storage().await;
let args = serde_json::json!({ "query": "" });
let result = execute(&storage, Some(args)).await;
let result = execute(&storage, &test_cognitive(), Some(args)).await;
assert!(result.is_err());
assert!(result.unwrap_err().contains("empty"));
}
@ -291,7 +556,7 @@ mod tests {
async fn test_search_whitespace_only_query_fails() {
let (storage, _dir) = test_storage().await;
let args = serde_json::json!({ "query": " \t\n " });
let result = execute(&storage, Some(args)).await;
let result = execute(&storage, &test_cognitive(), Some(args)).await;
assert!(result.is_err());
assert!(result.unwrap_err().contains("empty"));
}
@ -299,7 +564,7 @@ mod tests {
#[tokio::test]
async fn test_search_missing_arguments_fails() {
let (storage, _dir) = test_storage().await;
let result = execute(&storage, None).await;
let result = execute(&storage, &test_cognitive(), None).await;
assert!(result.is_err());
assert!(result.unwrap_err().contains("Missing arguments"));
}
@ -308,7 +573,7 @@ mod tests {
async fn test_search_missing_query_field_fails() {
let (storage, _dir) = test_storage().await;
let args = serde_json::json!({ "limit": 10 });
let result = execute(&storage, Some(args)).await;
let result = execute(&storage, &test_cognitive(), Some(args)).await;
assert!(result.is_err());
assert!(result.unwrap_err().contains("Invalid arguments"));
}
@ -327,7 +592,7 @@ mod tests {
"query": "test",
"limit": 0
});
let result = execute(&storage, Some(args)).await;
let result = execute(&storage, &test_cognitive(), Some(args)).await;
assert!(result.is_ok());
}
@ -341,7 +606,7 @@ mod tests {
"query": "test",
"limit": 1000
});
let result = execute(&storage, Some(args)).await;
let result = execute(&storage, &test_cognitive(), Some(args)).await;
assert!(result.is_ok());
}
@ -354,7 +619,7 @@ mod tests {
"query": "test",
"limit": -5
});
let result = execute(&storage, Some(args)).await;
let result = execute(&storage, &test_cognitive(), Some(args)).await;
assert!(result.is_ok());
}
@ -371,7 +636,7 @@ mod tests {
"query": "test",
"min_retention": -0.5
});
let result = execute(&storage, Some(args)).await;
let result = execute(&storage, &test_cognitive(), Some(args)).await;
assert!(result.is_ok());
}
@ -384,7 +649,7 @@ mod tests {
"query": "test",
"min_retention": 1.5
});
let result = execute(&storage, Some(args)).await;
let result = execute(&storage, &test_cognitive(), Some(args)).await;
// Should succeed but may return no results (retention > 1.0 clamped to 1.0)
assert!(result.is_ok());
}
@ -402,7 +667,7 @@ mod tests {
"query": "test",
"min_similarity": -0.5
});
let result = execute(&storage, Some(args)).await;
let result = execute(&storage, &test_cognitive(), Some(args)).await;
assert!(result.is_ok());
}
@ -415,7 +680,7 @@ mod tests {
"query": "test",
"min_similarity": 1.5
});
let result = execute(&storage, Some(args)).await;
let result = execute(&storage, &test_cognitive(), Some(args)).await;
// Should succeed but may return no results
assert!(result.is_ok());
}
@ -430,12 +695,12 @@ mod tests {
ingest_test_content(&storage, "The Rust programming language is memory safe.").await;
let args = serde_json::json!({ "query": "rust" });
let result = execute(&storage, Some(args)).await;
let result = execute(&storage, &test_cognitive(), Some(args)).await;
assert!(result.is_ok());
let value = result.unwrap();
assert_eq!(value["query"], "rust");
assert_eq!(value["method"], "hybrid");
assert_eq!(value["method"], "hybrid+cognitive");
assert!(value["total"].is_number());
assert!(value["results"].is_array());
}
@ -450,7 +715,7 @@ mod tests {
"query": "python",
"min_similarity": 0.0
});
let result = execute(&storage, Some(args)).await;
let result = execute(&storage, &test_cognitive(), Some(args)).await;
assert!(result.is_ok());
let value = result.unwrap();
@ -472,7 +737,7 @@ mod tests {
"limit": 2,
"min_similarity": 0.0
});
let result = execute(&storage, Some(args)).await;
let result = execute(&storage, &test_cognitive(), Some(args)).await;
assert!(result.is_ok());
let value = result.unwrap();
@ -486,7 +751,7 @@ mod tests {
// Don't ingest anything - database is empty
let args = serde_json::json!({ "query": "anything" });
let result = execute(&storage, Some(args)).await;
let result = execute(&storage, &test_cognitive(), Some(args)).await;
assert!(result.is_ok());
let value = result.unwrap();
@ -503,7 +768,7 @@ mod tests {
"query": "testing",
"min_similarity": 0.0
});
let result = execute(&storage, Some(args)).await;
let result = execute(&storage, &test_cognitive(), Some(args)).await;
assert!(result.is_ok());
let value = result.unwrap();
@ -536,7 +801,7 @@ mod tests {
"query": "item",
"min_similarity": 0.0
});
let result = execute(&storage, Some(args)).await;
let result = execute(&storage, &test_cognitive(), Some(args)).await;
assert!(result.is_ok());
let value = result.unwrap();
@ -620,7 +885,7 @@ mod tests {
"detail_level": "brief",
"min_similarity": 0.0
});
let result = execute(&storage, Some(args)).await;
let result = execute(&storage, &test_cognitive(), Some(args)).await;
assert!(result.is_ok());
let value = result.unwrap();
@ -649,7 +914,7 @@ mod tests {
"detail_level": "full",
"min_similarity": 0.0
});
let result = execute(&storage, Some(args)).await;
let result = execute(&storage, &test_cognitive(), Some(args)).await;
assert!(result.is_ok());
let value = result.unwrap();
@ -676,7 +941,7 @@ mod tests {
"query": "default",
"min_similarity": 0.0
});
let result = execute(&storage, Some(args)).await;
let result = execute(&storage, &test_cognitive(), Some(args)).await;
assert!(result.is_ok());
let value = result.unwrap();
@ -698,7 +963,7 @@ mod tests {
"query": "test",
"detail_level": "invalid_level"
});
let result = execute(&storage, Some(args)).await;
let result = execute(&storage, &test_cognitive(), Some(args)).await;
assert!(result.is_err());
assert!(result.unwrap_err().contains("Invalid detail_level"));
}