Release v2.1.23 Receipt Lock hardening

Hardens Sanhedrin Receipt Lock for model-agnostic use, adds fail-open telemetry and receipt docs, fixes smart_ingest batch safety, wires opt-in CUDA Qwen3 device selection, and refreshes dashboard/release assets.

Fixes #58

Fixes #60
This commit is contained in:
Sam Valladares 2026-05-27 18:22:53 -05:00
parent a8550410b0
commit 5edb163157
161 changed files with 1775 additions and 262 deletions

View file

@ -1,6 +1,6 @@
[package]
name = "vestige-core"
version = "2.1.22"
version = "2.1.23"
edition = "2024"
rust-version = "1.91"
authors = ["Vestige Team"]
@ -60,6 +60,17 @@ qwen3-reranker = ["qwen3-embeddings"]
# Metal GPU acceleration on Apple Silicon (significantly faster inference)
metal = ["fastembed/metal"]
# CUDA GPU acceleration on NVIDIA hardware (Windows / Linux, x86_64 + aarch64).
# Propagates to `candle-core/cuda`, which pulls in `cudarc` and `candle-kernels`
# for a per-build nvcc compile pass. Pair with `qwen3-embeddings` so the Candle
# backend is present in the build graph.
#
# Build: cargo build --release -p vestige-mcp --features qwen3-embeddings,cuda
cuda = ["qwen3-embeddings", "candle-core/cuda"]
# cuDNN on top of CUDA — additional fused kernels and faster inference paths.
cudnn = ["cuda", "candle-core/cudnn"]
[dependencies]
# Serialization

View file

@ -209,6 +209,12 @@ fn get_backend() -> Result<std::sync::MutexGuard<'static, EmbeddingBackend>, Emb
#[cfg(feature = "qwen3-embeddings")]
fn qwen3_device() -> Device {
#[cfg(feature = "cuda")]
{
if let Ok(device) = Device::new_cuda(0) {
return device;
}
}
#[cfg(feature = "metal")]
{
if let Ok(device) = Device::new_metal(0) {

View file

@ -86,6 +86,15 @@ pub struct SmartIngestResult {
pub prediction_error: Option<f32>,
/// Human-readable explanation of the decision
pub reason: String,
/// Previous content when smart ingest mutated an existing memory.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub previous_content: Option<String>,
/// Existing memory id that received merged or appended content.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub merged_from: Option<String>,
/// Full updated content after a merge/append/context write.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub merge_preview: Option<String>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@ -637,6 +646,20 @@ impl Storage {
/// This solves the "bad vs good similar memory" problem.
#[cfg(all(feature = "embeddings", feature = "vector-search"))]
pub fn smart_ingest(&self, input: IngestInput) -> Result<SmartIngestResult> {
self.smart_ingest_excluding(input, &[])
}
/// Smart ingest with caller-provided candidate exclusions.
///
/// Batch callers use this to keep two new items from the same caller-curated
/// batch from merging into each other while still allowing smart updates
/// against memories that existed before the batch began.
#[cfg(all(feature = "embeddings", feature = "vector-search"))]
pub fn smart_ingest_excluding(
&self,
input: IngestInput,
excluded_node_ids: &[String],
) -> Result<SmartIngestResult> {
use crate::advanced::prediction_error::{
CandidateMemory, GateDecision, PredictionErrorGate, UpdateType,
};
@ -652,6 +675,9 @@ impl Storage {
similarity: None,
prediction_error: Some(1.0),
reason: "Embeddings not available, falling back to regular ingest".to_string(),
previous_content: None,
merged_from: None,
merge_preview: None,
});
}
@ -666,6 +692,9 @@ impl Storage {
// Build candidate memories
let mut candidates: Vec<CandidateMemory> = Vec::new();
for (node_id, _similarity) in similar.iter() {
if excluded_node_ids.iter().any(|id| id == node_id) {
continue;
}
if let Some(node) = self.get_node(node_id)? {
// Get embedding for this node
if let Some(emb) = self.get_node_embedding(node_id)? {
@ -715,6 +744,9 @@ impl Storage {
reason, related_memory_ids
)
},
previous_content: None,
merged_from: None,
merge_preview: None,
})
}
GateDecision::Update {
@ -738,6 +770,9 @@ impl Storage {
prediction_error: Some(prediction_error),
reason: "Content nearly identical - reinforced existing memory"
.to_string(),
previous_content: None,
merged_from: None,
merge_preview: None,
})
}
UpdateType::Merge | UpdateType::Append => {
@ -745,10 +780,11 @@ impl Storage {
let existing = self
.get_node(&target_id)?
.ok_or_else(|| StorageError::NotFound(target_id.clone()))?;
let previous_content = existing.content.clone();
let merged_content = format!(
"{}\n\n[Updated {}]\n{}",
existing.content,
previous_content,
chrono::Utc::now().format("%Y-%m-%d"),
input.content
);
@ -767,10 +803,18 @@ impl Storage {
similarity: Some(similarity),
prediction_error: Some(prediction_error),
reason: "Merged with existing similar memory".to_string(),
previous_content: Some(previous_content),
merged_from: Some(target_id),
merge_preview: Some(merged_content),
})
}
UpdateType::Replace => {
// Replace content entirely
let existing = self
.get_node(&target_id)?
.ok_or_else(|| StorageError::NotFound(target_id.clone()))?;
let previous_content = existing.content;
self.update_node_content(&target_id, &input.content)?;
let node = self
.get_node(&target_id)?
@ -783,6 +827,9 @@ impl Storage {
similarity: Some(similarity),
prediction_error: Some(prediction_error),
reason: "Replaced existing memory with new content".to_string(),
previous_content: Some(previous_content),
merged_from: Some(target_id),
merge_preview: Some(input.content),
})
}
UpdateType::AddContext => {
@ -790,9 +837,10 @@ impl Storage {
let existing = self
.get_node(&target_id)?
.ok_or_else(|| StorageError::NotFound(target_id.clone()))?;
let previous_content = existing.content.clone();
let merged_content =
format!("{}\n\n---\nContext: {}", existing.content, input.content);
format!("{}\n\n---\nContext: {}", previous_content, input.content);
self.update_node_content(&target_id, &merged_content)?;
let node = self
@ -806,6 +854,9 @@ impl Storage {
similarity: Some(similarity),
prediction_error: Some(prediction_error),
reason: "Added new content as context to existing memory".to_string(),
previous_content: Some(previous_content),
merged_from: Some(target_id),
merge_preview: Some(merged_content),
})
}
}
@ -829,6 +880,9 @@ impl Storage {
similarity: Some(similarity),
prediction_error: Some(prediction_error),
reason: format!("New memory supersedes old: {:?}", supersede_reason),
previous_content: None,
merged_from: None,
merge_preview: None,
})
}
GateDecision::Merge {
@ -850,6 +904,9 @@ impl Storage {
memory_ids.len(),
strategy
),
previous_content: None,
merged_from: None,
merge_preview: None,
})
}
}

View file

@ -1,6 +1,6 @@
[package]
name = "vestige-mcp"
version = "2.1.22"
version = "2.1.23"
edition = "2024"
description = "Cognitive memory MCP server for AI agents - FSRS-6, spreading activation, synaptic tagging, 3D dashboard, and 130 years of memory research"
authors = ["samvallad33"]
@ -24,6 +24,10 @@ ort-dynamic = ["embeddings", "vestige-core/ort-dynamic"]
qwen3-embeddings = ["embeddings", "vestige-core/qwen3-embeddings"]
qwen3-reranker = ["qwen3-embeddings"]
metal = ["embeddings", "vestige-core/metal"]
# CUDA GPU acceleration on NVIDIA hardware (Windows / Linux, x86_64 + aarch64).
# Pairs with `qwen3-embeddings`; see vestige-core Cargo.toml.
cuda = ["qwen3-embeddings", "vestige-core/cuda"]
cudnn = ["cuda", "vestige-core/cudnn"]
[[bin]]
name = "vestige-mcp"
@ -47,7 +51,7 @@ path = "src/bin/cli.rs"
# Only `bundled-sqlite` is always on. `embeddings` and `vector-search` are
# toggled via vestige-mcp's own feature flags below so `--no-default-features`
# actually works (previously hardcoded here, which silently defeated the flag).
vestige-core = { version = "2.1.22", path = "../vestige-core", default-features = false, features = ["bundled-sqlite"] }
vestige-core = { version = "2.1.23", path = "../vestige-core", default-features = false, features = ["bundled-sqlite"] }
# ============================================================================
# MCP Server Dependencies

View file

@ -823,6 +823,13 @@ fn install_sandwich_from_source(
0o755,
options,
)?;
let (json_copied, json_skipped) = copy_companion_files(
&source_root.join("hooks"),
&hooks_dir,
&["json"],
0o644,
options,
)?;
let (agents_copied, agents_skipped) = copy_companion_files(
&source_root.join("agents"),
&agents_dir,
@ -834,8 +841,8 @@ fn install_sandwich_from_source(
println!(
"{}: {} installed, {} skipped",
"Hooks".white().bold(),
hooks_copied,
hooks_skipped
hooks_copied + json_copied,
hooks_skipped + json_skipped
);
println!(
"{}: {} installed, {} skipped",
@ -849,22 +856,37 @@ fn install_sandwich_from_source(
}
let dashboard_port = env::var("VESTIGE_DASHBOARD_PORT").unwrap_or_else(|_| "3927".to_string());
let endpoint = options
let mut endpoint = options
.sanhedrin_endpoint
.clone()
.or_else(|| env::var("VESTIGE_SANHEDRIN_ENDPOINT").ok())
.or_else(|| env::var("MLX_ENDPOINT").ok())
.unwrap_or_else(|| "http://127.0.0.1:8080/v1/chat/completions".to_string())
.unwrap_or_default()
.trim_end_matches('/')
.to_string();
let model = options
let mut model = options
.sanhedrin_model
.clone()
.or_else(|| env::var("VESTIGE_SANHEDRIN_MODEL").ok())
.or_else(|| env::var("VESTIGE_SANDWICH_MODEL").ok())
.unwrap_or_else(|| "mlx-community/Qwen3.6-35B-A3B-4bit".to_string());
.unwrap_or_default();
if with_launchd {
if endpoint.is_empty() {
endpoint = "http://127.0.0.1:8080/v1/chat/completions".to_string();
}
if model.is_empty() {
model = "mlx-community/Qwen3.6-35B-A3B-4bit".to_string();
}
}
if enable_sanhedrin {
if endpoint.is_empty() || model.is_empty() {
println!(
"{}",
"Sanhedrin enabled without a verifier model; it will fail open until VESTIGE_SANHEDRIN_ENDPOINT and VESTIGE_SANHEDRIN_MODEL are set."
.yellow()
);
}
write_sanhedrin_env(&hooks_dir, &endpoint, &model, &dashboard_port)?;
}
if with_launchd {
@ -2442,6 +2464,8 @@ fn run_ingest(
/// Run the dashboard web server
fn run_dashboard(port: u16, open_browser: bool) -> anyhow::Result<()> {
use vestige_mcp::cognitive::CognitiveEngine;
println!("{}", "=== Vestige Dashboard ===".cyan().bold());
println!();
println!(
@ -2467,7 +2491,14 @@ fn run_dashboard(port: u16, open_browser: bool) -> anyhow::Result<()> {
let rt = tokio::runtime::Runtime::new()?;
rt.block_on(async move {
vestige_mcp::dashboard::start_dashboard(storage, None, port, open_browser)
// Initialize cognitive engine for dream and other cognitive features
let cognitive = Arc::new(tokio::sync::Mutex::new(CognitiveEngine::new()));
{
let mut cog = cognitive.lock().await;
cog.hydrate(&storage); // Load persisted connections
}
vestige_mcp::dashboard::start_dashboard(storage, Some(cognitive), port, open_browser)
.await
.map_err(|e| anyhow::anyhow!("Dashboard error: {}", e))
})

View file

@ -26,7 +26,20 @@ fn main() -> anyhow::Result<()> {
// Parse args
let args: Vec<String> = std::env::args().collect();
if args.len() < 2 {
eprintln!("Usage: vestige-restore <backup.json>");
print_usage_stderr();
std::process::exit(1);
}
if matches!(args[1].as_str(), "-h" | "--help") {
print_usage_stdout();
return Ok(());
}
if matches!(args[1].as_str(), "-V" | "--version") {
println!("vestige-restore {}", env!("CARGO_PKG_VERSION"));
return Ok(());
}
if args.len() > 2 {
eprintln!("Unexpected extra argument: {}", args[2]);
print_usage_stderr();
std::process::exit(1);
}
@ -91,6 +104,18 @@ fn main() -> anyhow::Result<()> {
Ok(())
}
fn print_usage_stdout() {
println!("{}", usage());
}
fn print_usage_stderr() {
eprintln!("{}", usage());
}
fn usage() -> &'static str {
"Vestige Restore\n\nUSAGE:\n vestige-restore <backup.json>\n\nOPTIONS:\n -h, --help Print help information\n -V, --version Print version information"
}
/// Truncate a string for display (UTF-8 safe)
fn truncate(s: &str, max_chars: usize) -> String {
let s = s.replace('\n', " ");

View file

@ -3,8 +3,9 @@
//! v2.0: Adds cognitive operation endpoints (dream, explore, predict, importance, consolidation)
use std::cmp::Reverse;
use std::collections::{BTreeMap, HashSet};
use std::fs::{self, OpenOptions};
use std::io::Write;
use std::io::{BufRead, BufReader, Write};
use std::path::{Path as FsPath, PathBuf};
use axum::extract::{Path, Query, State};
@ -355,6 +356,11 @@ pub struct SanhedrinAppealRequest {
pub claim_id: Option<String>,
}
#[derive(Debug, Deserialize)]
pub struct SanhedrinTelemetryParams {
pub days: Option<i64>,
}
/// Return the latest Sanhedrin receipt written by the Stop-hook bridge.
pub async fn get_sanhedrin_latest() -> Result<Json<Value>, StatusCode> {
let state_dir = sanhedrin_state_dir();
@ -369,15 +375,124 @@ pub async fn get_sanhedrin_latest() -> Result<Json<Value>, StatusCode> {
let raw = fs::read_to_string(&latest_path).map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
let receipt: Value =
serde_json::from_str(&raw).map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
let schema_warning = sanhedrin_schema_warning(&receipt);
Ok(Json(serde_json::json!({
"receipt": receipt,
"stateDir": state_dir,
"receiptPath": latest_path,
"htmlPath": state_dir.join("latest.html"),
"schemaWarning": schema_warning,
})))
}
/// Return rolling Sanhedrin receipts, appeals, and fail-open counters.
pub async fn get_sanhedrin_telemetry(
Query(params): Query<SanhedrinTelemetryParams>,
) -> Result<Json<Value>, StatusCode> {
let state_dir = sanhedrin_state_dir();
let days = params.days.unwrap_or(7).clamp(1, 90);
let telemetry = tokio::task::spawn_blocking(move || build_sanhedrin_telemetry(state_dir, days))
.await
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)??;
Ok(Json(telemetry))
}
fn build_sanhedrin_telemetry(state_dir: PathBuf, days: i64) -> Result<Value, StatusCode> {
let cutoff = Utc::now() - Duration::days(days);
let receipts_dir = state_dir.join("receipts");
let mut by_verdict = serde_json::Map::new();
for verdict in ["PASS", "NOTE", "CAUTION", "VETO", "APPEALED"] {
by_verdict.insert(verdict.to_string(), Value::from(0));
}
let mut by_class: BTreeMap<String, i64> = BTreeMap::new();
let mut daily: BTreeMap<String, serde_json::Map<String, Value>> = BTreeMap::new();
let mut total_runs = 0i64;
let mut last_run_at: Option<DateTime<Utc>> = None;
let mut truncated = false;
if let Ok(entries) = bounded_receipt_entries(&receipts_dir, cutoff) {
for path in entries {
let Ok(raw) = fs::read_to_string(&path) else {
continue;
};
let Ok(receipt) = serde_json::from_str::<Value>(&raw) else {
continue;
};
let Some(created_at) = parse_sanhedrin_timestamp(&receipt["createdAt"]) else {
continue;
};
if created_at < cutoff {
continue;
}
total_runs += 1;
if last_run_at.map(|last| created_at > last).unwrap_or(true) {
last_run_at = Some(created_at);
}
let verdict = receipt
.get("verdictBar")
.and_then(Value::as_str)
.unwrap_or("NOTE")
.to_ascii_uppercase();
increment_json_counter(&mut by_verdict, &verdict);
let day = created_at.date_naive().to_string();
let bucket = daily.entry(day).or_insert_with(|| {
let mut map = serde_json::Map::new();
map.insert("date".to_string(), Value::String(String::new()));
map.insert("total".to_string(), Value::from(0));
map.insert("pass".to_string(), Value::from(0));
map.insert("note".to_string(), Value::from(0));
map.insert("caution".to_string(), Value::from(0));
map.insert("veto".to_string(), Value::from(0));
map.insert("appealed".to_string(), Value::from(0));
map.insert("failOpen".to_string(), Value::from(0));
map
});
increment_json_counter(bucket, "total");
match verdict.as_str() {
"PASS" => increment_json_counter(bucket, "pass"),
"NOTE" => increment_json_counter(bucket, "note"),
"CAUTION" => increment_json_counter(bucket, "caution"),
"VETO" => increment_json_counter(bucket, "veto"),
"APPEALED" => increment_json_counter(bucket, "appealed"),
_ => increment_json_counter(bucket, "note"),
}
if let Some(claims) = receipt.get("claims").and_then(Value::as_array) {
for claim in claims {
let class = known_sanhedrin_class(
claim
.get("class")
.and_then(Value::as_str)
.unwrap_or("UNKNOWN"),
);
*by_class.entry(class).or_insert(0) += 1;
}
}
}
truncated = total_runs >= 5_000;
}
let appeals = count_jsonl_since(&state_dir.join("appeals.jsonl"), cutoff, false);
let fail_open = count_jsonl_since(&state_dir.join("fail-open.jsonl"), cutoff, true);
for (date, bucket) in daily.iter_mut() {
bucket.insert("date".to_string(), Value::String(date.clone()));
}
Ok(serde_json::json!({
"days": days,
"stateDir": state_dir,
"totalRuns": total_runs,
"byVerdict": by_verdict,
"byClass": by_class,
"appeals": appeals,
"failOpen": fail_open,
"truncated": truncated,
"lastRunAt": last_run_at.map(|dt| dt.to_rfc3339()),
"daily": daily.into_values().map(Value::Object).collect::<Vec<_>>(),
}))
}
/// Record feedback that a Sanhedrin veto was stale, wrong, or too strict.
///
/// This intentionally does not promote, demote, suppress, edit, or delete any
@ -468,6 +583,125 @@ fn sanhedrin_state_dir() -> PathBuf {
.unwrap_or_else(|| PathBuf::from(".vestige/sanhedrin"))
}
fn sanhedrin_schema_warning(receipt: &Value) -> Option<String> {
let schema = receipt.get("schema").and_then(Value::as_str).unwrap_or("");
if schema.is_empty() || schema == "vestige.sanhedrin.receipt.v1" {
None
} else {
Some(format!(
"Unsupported Sanhedrin receipt schema '{}'; dashboard expects vestige.sanhedrin.receipt.v1",
schema
))
}
}
fn parse_sanhedrin_timestamp(value: &Value) -> Option<DateTime<Utc>> {
let raw = value.as_str()?;
DateTime::parse_from_rfc3339(raw)
.map(|dt| dt.with_timezone(&Utc))
.ok()
}
fn bounded_receipt_entries(
receipts_dir: &FsPath,
cutoff: DateTime<Utc>,
) -> Result<Vec<PathBuf>, StatusCode> {
const MAX_RECEIPTS: usize = 5_000;
const MAX_RECEIPT_BYTES: u64 = 256 * 1024;
let mut entries: Vec<(Option<DateTime<Utc>>, PathBuf)> = Vec::new();
let Ok(read_dir) = fs::read_dir(receipts_dir) else {
return Ok(Vec::new());
};
for entry in read_dir.flatten() {
let path = entry.path();
if path.extension().and_then(|ext| ext.to_str()) != Some("json") {
continue;
}
let Ok(metadata) = fs::symlink_metadata(&path) else {
continue;
};
if metadata.file_type().is_symlink()
|| !metadata.is_file()
|| metadata.len() > MAX_RECEIPT_BYTES
{
continue;
}
let modified = metadata.modified().ok().map(DateTime::<Utc>::from);
if modified.map(|mtime| mtime < cutoff).unwrap_or(false) {
continue;
}
entries.push((modified, path));
}
entries.sort_by(|(left_time, _), (right_time, _)| right_time.cmp(left_time));
Ok(entries
.into_iter()
.take(MAX_RECEIPTS)
.map(|(_, path)| path)
.collect())
}
fn increment_json_counter(map: &mut serde_json::Map<String, Value>, key: &str) {
let current = map.get(key).and_then(Value::as_i64).unwrap_or(0);
map.insert(key.to_string(), Value::from(current + 1));
}
fn count_jsonl_since(path: &FsPath, cutoff: DateTime<Utc>, distinct_run: bool) -> i64 {
const MAX_LEDGER_BYTES: u64 = 2 * 1024 * 1024;
let Ok(metadata) = fs::symlink_metadata(path) else {
return 0;
};
if metadata.file_type().is_symlink() || !metadata.is_file() || metadata.len() > MAX_LEDGER_BYTES
{
return 0;
}
let Ok(file) = fs::File::open(path) else {
return 0;
};
let mut seen_runs = HashSet::new();
let mut count = 0i64;
for line in BufReader::new(file).lines().map_while(Result::ok) {
let Ok(item) = serde_json::from_str::<Value>(&line) else {
continue;
};
let Some(timestamp) = parse_sanhedrin_timestamp(&item["timestamp"]) else {
continue;
};
if timestamp < cutoff {
continue;
}
if distinct_run
&& let Some(run_id) = item.get("runId").and_then(Value::as_str)
&& !seen_runs.insert(run_id.to_string())
{
continue;
}
count += 1;
}
count
}
fn known_sanhedrin_class(class: &str) -> String {
match class {
"receipt_lock"
| "TECHNICAL"
| "BIOGRAPHICAL"
| "FINANCIAL"
| "ACHIEVEMENT"
| "TIMELINE"
| "QUANTITATIVE"
| "ATTRIBUTION"
| "CAUSAL"
| "COMPARATIVE"
| "EXISTENTIAL"
| "VAGUE-QUANTIFIER"
| "UNVERIFIED-POSITIVE" => class.to_string(),
_ => "OTHER".to_string(),
}
}
fn ensure_sanhedrin_dirs(state_dir: &FsPath) -> Result<(), StatusCode> {
fs::create_dir_all(state_dir.join("receipts")).map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)
}

View file

@ -176,8 +176,12 @@ fn build_router_inner(state: AppState, port: u16) -> (Router, AppState) {
// Wraps crate::tools::cross_reference::execute. Emits
// DeepReferenceCompleted so Graph3D can glide, pulse, and arc.
.route("/api/deep_reference", post(handlers::deep_reference_query))
// Sanhedrin receipts (v2.1.22): latest local hook verdict + appeal training.
// Sanhedrin receipts: latest local hook verdict + appeal training.
.route("/api/sanhedrin/latest", get(handlers::get_sanhedrin_latest))
.route(
"/api/sanhedrin/telemetry",
get(handlers::get_sanhedrin_telemetry),
)
.route("/api/sanhedrin/appeal", post(handlers::appeal_sanhedrin))
.layer(
ServiceBuilder::new()

View file

@ -243,7 +243,22 @@ fn prepare_storage_path(data_dir: Option<PathBuf>) -> io::Result<Option<PathBuf>
};
let data_dir = expand_tilde(data_dir);
fs::create_dir_all(&data_dir)?;
// Check if path exists and is a file (not a directory)
if data_dir.exists() && !data_dir.is_dir() {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
format!(
"Data directory path exists but is not a directory: {}",
data_dir.display()
),
));
}
// Only create if it doesn't exist (avoids "File exists" error on existing directories)
if !data_dir.exists() {
fs::create_dir_all(&data_dir)?;
}
#[cfg(unix)]
{

View file

@ -57,9 +57,15 @@ pub fn schema() -> Value {
"description": "Force creation of a new memory even if similar content exists",
"default": false
},
"batchMergePolicy": {
"type": "string",
"enum": ["force_create", "smart"],
"description": "Batch mode only. Defaults to 'force_create' so caller-separated items stay separate. Use 'smart' to allow Prediction Error Gating against existing memories.",
"default": "force_create"
},
"items": {
"type": "array",
"description": "Batch mode: array of items to save (max 20). Each runs through full cognitive pipeline with Prediction Error Gating. Use at session end or before context compaction.",
"description": "Batch mode: array of items to save (max 20). Defaults to force-creating each caller-separated item; set batchMergePolicy='smart' to allow Prediction Error Gating against existing memories. Use at session end or before context compaction.",
"maxItems": 20,
"items": {
"type": "object",
@ -104,6 +110,7 @@ struct SmartIngestArgs {
tags: Option<Vec<String>>,
source: Option<String>,
force_create: Option<bool>,
batch_merge_policy: Option<String>,
items: Option<Vec<BatchItem>>,
}
@ -131,8 +138,26 @@ pub async fn execute(
// Detect mode: batch (items present) vs single (content present)
if let Some(items) = args.items {
let global_force = args.force_create.unwrap_or(false);
return execute_batch(storage, cognitive, items, global_force).await;
let batch_merge_policy = args
.batch_merge_policy
.unwrap_or_else(|| "force_create".to_string());
let default_force_create = match batch_merge_policy.as_str() {
"force_create" => true,
"smart" => false,
other => {
return Err(format!(
"Invalid batchMergePolicy '{}'. Must be 'force_create' or 'smart'.",
other
));
}
};
let global_force = match args.force_create {
Some(true) => true,
Some(false) if batch_merge_policy == "smart" => false,
Some(false) => default_force_create,
None => default_force_create,
};
return execute_batch(storage, cognitive, items, global_force, &batch_merge_policy).await;
}
// Single mode: content is required
@ -252,6 +277,9 @@ pub async fn execute(
"similarity": result.similarity,
"predictionError": result.prediction_error,
"supersededId": result.superseded_id,
"previousContent": result.previous_content,
"mergedFrom": result.merged_from,
"mergePreview": result.merge_preview,
"importanceScore": importance_composite,
"reason": result.reason,
"explanation": match result.decision.as_str() {
@ -305,6 +333,7 @@ async fn execute_batch(
cognitive: &Arc<Mutex<CognitiveEngine>>,
items: Vec<BatchItem>,
global_force_create: bool,
batch_merge_policy: &str,
) -> Result<Value, String> {
if items.is_empty() {
return Err("Items array cannot be empty".to_string());
@ -321,6 +350,7 @@ async fn execute_batch(
let updated = 0u32;
let mut skipped = 0u32;
let mut errors = 0u32;
let mut batch_created_node_ids: Vec<String> = Vec::new();
for (i, item) in items.into_iter().enumerate() {
// Skip empty content
@ -400,6 +430,7 @@ async fn execute_batch(
let node_type = node.node_type.clone();
created += 1;
batch_created_node_ids.push(node_id.clone());
run_post_ingest(
cognitive,
&node_id,
@ -431,15 +462,18 @@ async fn execute_batch(
#[cfg(all(feature = "embeddings", feature = "vector-search"))]
{
match storage.smart_ingest(input) {
match storage.smart_ingest_excluding(input, &batch_created_node_ids) {
Ok(result) => {
let node_id = result.node.id.clone();
let node_content = result.node.content.clone();
let node_type = result.node.node_type.clone();
match result.decision.as_str() {
"create" | "supersede" | "replace" => created += 1,
"update" | "reinforce" | "merge" | "add_context" => updated += 1,
"create" | "supersede" | "merge" => {
created += 1;
batch_created_node_ids.push(node_id.clone());
}
"update" | "reinforce" | "replace" | "add_context" => updated += 1,
_ => created += 1,
}
@ -458,6 +492,11 @@ async fn execute_batch(
"decision": result.decision,
"nodeId": node_id,
"similarity": result.similarity,
"predictionError": result.prediction_error,
"supersededId": result.superseded_id,
"previousContent": result.previous_content,
"mergedFrom": result.merged_from,
"mergePreview": result.merge_preview,
"importanceScore": importance_composite,
"reason": result.reason
}));
@ -482,6 +521,7 @@ async fn execute_batch(
let node_type = node.node_type.clone();
created += 1;
batch_created_node_ids.push(node_id.clone());
run_post_ingest(
cognitive,
&node_id,
@ -514,6 +554,7 @@ async fn execute_batch(
Ok(serde_json::json!({
"success": errors == 0,
"mode": "batch",
"batchMergePolicy": batch_merge_policy,
"summary": {
"total": results.len(),
"created": created,
@ -637,6 +678,7 @@ mod tests {
assert_eq!(schema_value["type"], "object");
assert!(schema_value["properties"]["content"].is_object());
assert!(schema_value["properties"]["forceCreate"].is_object());
assert!(schema_value["properties"]["batchMergePolicy"].is_object());
assert!(schema_value["properties"]["items"].is_object());
// v1.7: no top-level required — content for single mode, items for batch mode
assert!(schema_value.get("required").is_none() || schema_value["required"].is_null());
@ -807,9 +849,53 @@ mod tests {
assert!(result.is_ok());
let value = result.unwrap();
assert_eq!(value["mode"], "batch");
assert_eq!(value["batchMergePolicy"], "force_create");
assert_eq!(value["summary"]["total"], 2);
}
#[tokio::test]
async fn test_batch_defaults_to_force_create_for_caller_separated_items() {
let (storage, _dir) = test_storage().await;
let result = execute(
&storage,
&test_cognitive(),
Some(serde_json::json!({
"forceCreate": false,
"items": [
{ "content": "Jira tickets should not auto-assign sprint fields." },
{ "content": "Sprint planning summaries should not append Jira status labels." }
]
})),
)
.await;
let value = result.unwrap();
assert_eq!(value["batchMergePolicy"], "force_create");
assert_eq!(value["summary"]["created"], 2);
assert_eq!(value["summary"]["updated"], 0);
for item in value["results"].as_array().unwrap() {
assert_eq!(item["decision"], "create");
assert!(item["reason"].as_str().unwrap().contains("Forced creation"));
}
}
#[tokio::test]
async fn test_batch_rejects_invalid_merge_policy() {
let (storage, _dir) = test_storage().await;
let result = execute(
&storage,
&test_cognitive(),
Some(serde_json::json!({
"batchMergePolicy": "merge_everything",
"items": [{ "content": "Invalid policy should fail." }]
})),
)
.await;
assert!(result.is_err());
assert!(result.unwrap_err().contains("batchMergePolicy"));
}
#[tokio::test]
async fn test_batch_skips_empty_content() {
let (storage, _dir) = test_storage().await;