mirror of
https://github.com/samvallad33/vestige.git
synced 2026-07-22 23:31:02 +02:00
Release v2.1.23 Receipt Lock hardening
Hardens Sanhedrin Receipt Lock for model-agnostic use, adds fail-open telemetry and receipt docs, fixes smart_ingest batch safety, wires opt-in CUDA Qwen3 device selection, and refreshes dashboard/release assets. Fixes #58 Fixes #60
This commit is contained in:
parent
a8550410b0
commit
5edb163157
161 changed files with 1775 additions and 262 deletions
|
|
@ -1,6 +1,6 @@
|
|||
[package]
|
||||
name = "vestige-core"
|
||||
version = "2.1.22"
|
||||
version = "2.1.23"
|
||||
edition = "2024"
|
||||
rust-version = "1.91"
|
||||
authors = ["Vestige Team"]
|
||||
|
|
@ -60,6 +60,17 @@ qwen3-reranker = ["qwen3-embeddings"]
|
|||
# Metal GPU acceleration on Apple Silicon (significantly faster inference)
|
||||
metal = ["fastembed/metal"]
|
||||
|
||||
# CUDA GPU acceleration on NVIDIA hardware (Windows / Linux, x86_64 + aarch64).
|
||||
# Propagates to `candle-core/cuda`, which pulls in `cudarc` and `candle-kernels`
|
||||
# for a per-build nvcc compile pass. Pair with `qwen3-embeddings` so the Candle
|
||||
# backend is present in the build graph.
|
||||
#
|
||||
# Build: cargo build --release -p vestige-mcp --features qwen3-embeddings,cuda
|
||||
cuda = ["qwen3-embeddings", "candle-core/cuda"]
|
||||
|
||||
# cuDNN on top of CUDA — additional fused kernels and faster inference paths.
|
||||
cudnn = ["cuda", "candle-core/cudnn"]
|
||||
|
||||
|
||||
[dependencies]
|
||||
# Serialization
|
||||
|
|
|
|||
|
|
@ -209,6 +209,12 @@ fn get_backend() -> Result<std::sync::MutexGuard<'static, EmbeddingBackend>, Emb
|
|||
|
||||
#[cfg(feature = "qwen3-embeddings")]
|
||||
fn qwen3_device() -> Device {
|
||||
#[cfg(feature = "cuda")]
|
||||
{
|
||||
if let Ok(device) = Device::new_cuda(0) {
|
||||
return device;
|
||||
}
|
||||
}
|
||||
#[cfg(feature = "metal")]
|
||||
{
|
||||
if let Ok(device) = Device::new_metal(0) {
|
||||
|
|
|
|||
|
|
@ -86,6 +86,15 @@ pub struct SmartIngestResult {
|
|||
pub prediction_error: Option<f32>,
|
||||
/// Human-readable explanation of the decision
|
||||
pub reason: String,
|
||||
/// Previous content when smart ingest mutated an existing memory.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub previous_content: Option<String>,
|
||||
/// Existing memory id that received merged or appended content.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub merged_from: Option<String>,
|
||||
/// Full updated content after a merge/append/context write.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub merge_preview: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
|
|
@ -637,6 +646,20 @@ impl Storage {
|
|||
/// This solves the "bad vs good similar memory" problem.
|
||||
#[cfg(all(feature = "embeddings", feature = "vector-search"))]
|
||||
pub fn smart_ingest(&self, input: IngestInput) -> Result<SmartIngestResult> {
|
||||
self.smart_ingest_excluding(input, &[])
|
||||
}
|
||||
|
||||
/// Smart ingest with caller-provided candidate exclusions.
|
||||
///
|
||||
/// Batch callers use this to keep two new items from the same caller-curated
|
||||
/// batch from merging into each other while still allowing smart updates
|
||||
/// against memories that existed before the batch began.
|
||||
#[cfg(all(feature = "embeddings", feature = "vector-search"))]
|
||||
pub fn smart_ingest_excluding(
|
||||
&self,
|
||||
input: IngestInput,
|
||||
excluded_node_ids: &[String],
|
||||
) -> Result<SmartIngestResult> {
|
||||
use crate::advanced::prediction_error::{
|
||||
CandidateMemory, GateDecision, PredictionErrorGate, UpdateType,
|
||||
};
|
||||
|
|
@ -652,6 +675,9 @@ impl Storage {
|
|||
similarity: None,
|
||||
prediction_error: Some(1.0),
|
||||
reason: "Embeddings not available, falling back to regular ingest".to_string(),
|
||||
previous_content: None,
|
||||
merged_from: None,
|
||||
merge_preview: None,
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -666,6 +692,9 @@ impl Storage {
|
|||
// Build candidate memories
|
||||
let mut candidates: Vec<CandidateMemory> = Vec::new();
|
||||
for (node_id, _similarity) in similar.iter() {
|
||||
if excluded_node_ids.iter().any(|id| id == node_id) {
|
||||
continue;
|
||||
}
|
||||
if let Some(node) = self.get_node(node_id)? {
|
||||
// Get embedding for this node
|
||||
if let Some(emb) = self.get_node_embedding(node_id)? {
|
||||
|
|
@ -715,6 +744,9 @@ impl Storage {
|
|||
reason, related_memory_ids
|
||||
)
|
||||
},
|
||||
previous_content: None,
|
||||
merged_from: None,
|
||||
merge_preview: None,
|
||||
})
|
||||
}
|
||||
GateDecision::Update {
|
||||
|
|
@ -738,6 +770,9 @@ impl Storage {
|
|||
prediction_error: Some(prediction_error),
|
||||
reason: "Content nearly identical - reinforced existing memory"
|
||||
.to_string(),
|
||||
previous_content: None,
|
||||
merged_from: None,
|
||||
merge_preview: None,
|
||||
})
|
||||
}
|
||||
UpdateType::Merge | UpdateType::Append => {
|
||||
|
|
@ -745,10 +780,11 @@ impl Storage {
|
|||
let existing = self
|
||||
.get_node(&target_id)?
|
||||
.ok_or_else(|| StorageError::NotFound(target_id.clone()))?;
|
||||
let previous_content = existing.content.clone();
|
||||
|
||||
let merged_content = format!(
|
||||
"{}\n\n[Updated {}]\n{}",
|
||||
existing.content,
|
||||
previous_content,
|
||||
chrono::Utc::now().format("%Y-%m-%d"),
|
||||
input.content
|
||||
);
|
||||
|
|
@ -767,10 +803,18 @@ impl Storage {
|
|||
similarity: Some(similarity),
|
||||
prediction_error: Some(prediction_error),
|
||||
reason: "Merged with existing similar memory".to_string(),
|
||||
previous_content: Some(previous_content),
|
||||
merged_from: Some(target_id),
|
||||
merge_preview: Some(merged_content),
|
||||
})
|
||||
}
|
||||
UpdateType::Replace => {
|
||||
// Replace content entirely
|
||||
let existing = self
|
||||
.get_node(&target_id)?
|
||||
.ok_or_else(|| StorageError::NotFound(target_id.clone()))?;
|
||||
let previous_content = existing.content;
|
||||
|
||||
self.update_node_content(&target_id, &input.content)?;
|
||||
let node = self
|
||||
.get_node(&target_id)?
|
||||
|
|
@ -783,6 +827,9 @@ impl Storage {
|
|||
similarity: Some(similarity),
|
||||
prediction_error: Some(prediction_error),
|
||||
reason: "Replaced existing memory with new content".to_string(),
|
||||
previous_content: Some(previous_content),
|
||||
merged_from: Some(target_id),
|
||||
merge_preview: Some(input.content),
|
||||
})
|
||||
}
|
||||
UpdateType::AddContext => {
|
||||
|
|
@ -790,9 +837,10 @@ impl Storage {
|
|||
let existing = self
|
||||
.get_node(&target_id)?
|
||||
.ok_or_else(|| StorageError::NotFound(target_id.clone()))?;
|
||||
let previous_content = existing.content.clone();
|
||||
|
||||
let merged_content =
|
||||
format!("{}\n\n---\nContext: {}", existing.content, input.content);
|
||||
format!("{}\n\n---\nContext: {}", previous_content, input.content);
|
||||
|
||||
self.update_node_content(&target_id, &merged_content)?;
|
||||
let node = self
|
||||
|
|
@ -806,6 +854,9 @@ impl Storage {
|
|||
similarity: Some(similarity),
|
||||
prediction_error: Some(prediction_error),
|
||||
reason: "Added new content as context to existing memory".to_string(),
|
||||
previous_content: Some(previous_content),
|
||||
merged_from: Some(target_id),
|
||||
merge_preview: Some(merged_content),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -829,6 +880,9 @@ impl Storage {
|
|||
similarity: Some(similarity),
|
||||
prediction_error: Some(prediction_error),
|
||||
reason: format!("New memory supersedes old: {:?}", supersede_reason),
|
||||
previous_content: None,
|
||||
merged_from: None,
|
||||
merge_preview: None,
|
||||
})
|
||||
}
|
||||
GateDecision::Merge {
|
||||
|
|
@ -850,6 +904,9 @@ impl Storage {
|
|||
memory_ids.len(),
|
||||
strategy
|
||||
),
|
||||
previous_content: None,
|
||||
merged_from: None,
|
||||
merge_preview: None,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[package]
|
||||
name = "vestige-mcp"
|
||||
version = "2.1.22"
|
||||
version = "2.1.23"
|
||||
edition = "2024"
|
||||
description = "Cognitive memory MCP server for AI agents - FSRS-6, spreading activation, synaptic tagging, 3D dashboard, and 130 years of memory research"
|
||||
authors = ["samvallad33"]
|
||||
|
|
@ -24,6 +24,10 @@ ort-dynamic = ["embeddings", "vestige-core/ort-dynamic"]
|
|||
qwen3-embeddings = ["embeddings", "vestige-core/qwen3-embeddings"]
|
||||
qwen3-reranker = ["qwen3-embeddings"]
|
||||
metal = ["embeddings", "vestige-core/metal"]
|
||||
# CUDA GPU acceleration on NVIDIA hardware (Windows / Linux, x86_64 + aarch64).
|
||||
# Pairs with `qwen3-embeddings`; see vestige-core Cargo.toml.
|
||||
cuda = ["qwen3-embeddings", "vestige-core/cuda"]
|
||||
cudnn = ["cuda", "vestige-core/cudnn"]
|
||||
|
||||
[[bin]]
|
||||
name = "vestige-mcp"
|
||||
|
|
@ -47,7 +51,7 @@ path = "src/bin/cli.rs"
|
|||
# Only `bundled-sqlite` is always on. `embeddings` and `vector-search` are
|
||||
# toggled via vestige-mcp's own feature flags below so `--no-default-features`
|
||||
# actually works (previously hardcoded here, which silently defeated the flag).
|
||||
vestige-core = { version = "2.1.22", path = "../vestige-core", default-features = false, features = ["bundled-sqlite"] }
|
||||
vestige-core = { version = "2.1.23", path = "../vestige-core", default-features = false, features = ["bundled-sqlite"] }
|
||||
|
||||
# ============================================================================
|
||||
# MCP Server Dependencies
|
||||
|
|
|
|||
|
|
@ -823,6 +823,13 @@ fn install_sandwich_from_source(
|
|||
0o755,
|
||||
options,
|
||||
)?;
|
||||
let (json_copied, json_skipped) = copy_companion_files(
|
||||
&source_root.join("hooks"),
|
||||
&hooks_dir,
|
||||
&["json"],
|
||||
0o644,
|
||||
options,
|
||||
)?;
|
||||
let (agents_copied, agents_skipped) = copy_companion_files(
|
||||
&source_root.join("agents"),
|
||||
&agents_dir,
|
||||
|
|
@ -834,8 +841,8 @@ fn install_sandwich_from_source(
|
|||
println!(
|
||||
"{}: {} installed, {} skipped",
|
||||
"Hooks".white().bold(),
|
||||
hooks_copied,
|
||||
hooks_skipped
|
||||
hooks_copied + json_copied,
|
||||
hooks_skipped + json_skipped
|
||||
);
|
||||
println!(
|
||||
"{}: {} installed, {} skipped",
|
||||
|
|
@ -849,22 +856,37 @@ fn install_sandwich_from_source(
|
|||
}
|
||||
|
||||
let dashboard_port = env::var("VESTIGE_DASHBOARD_PORT").unwrap_or_else(|_| "3927".to_string());
|
||||
let endpoint = options
|
||||
let mut endpoint = options
|
||||
.sanhedrin_endpoint
|
||||
.clone()
|
||||
.or_else(|| env::var("VESTIGE_SANHEDRIN_ENDPOINT").ok())
|
||||
.or_else(|| env::var("MLX_ENDPOINT").ok())
|
||||
.unwrap_or_else(|| "http://127.0.0.1:8080/v1/chat/completions".to_string())
|
||||
.unwrap_or_default()
|
||||
.trim_end_matches('/')
|
||||
.to_string();
|
||||
let model = options
|
||||
let mut model = options
|
||||
.sanhedrin_model
|
||||
.clone()
|
||||
.or_else(|| env::var("VESTIGE_SANHEDRIN_MODEL").ok())
|
||||
.or_else(|| env::var("VESTIGE_SANDWICH_MODEL").ok())
|
||||
.unwrap_or_else(|| "mlx-community/Qwen3.6-35B-A3B-4bit".to_string());
|
||||
.unwrap_or_default();
|
||||
if with_launchd {
|
||||
if endpoint.is_empty() {
|
||||
endpoint = "http://127.0.0.1:8080/v1/chat/completions".to_string();
|
||||
}
|
||||
if model.is_empty() {
|
||||
model = "mlx-community/Qwen3.6-35B-A3B-4bit".to_string();
|
||||
}
|
||||
}
|
||||
|
||||
if enable_sanhedrin {
|
||||
if endpoint.is_empty() || model.is_empty() {
|
||||
println!(
|
||||
"{}",
|
||||
"Sanhedrin enabled without a verifier model; it will fail open until VESTIGE_SANHEDRIN_ENDPOINT and VESTIGE_SANHEDRIN_MODEL are set."
|
||||
.yellow()
|
||||
);
|
||||
}
|
||||
write_sanhedrin_env(&hooks_dir, &endpoint, &model, &dashboard_port)?;
|
||||
}
|
||||
if with_launchd {
|
||||
|
|
@ -2442,6 +2464,8 @@ fn run_ingest(
|
|||
|
||||
/// Run the dashboard web server
|
||||
fn run_dashboard(port: u16, open_browser: bool) -> anyhow::Result<()> {
|
||||
use vestige_mcp::cognitive::CognitiveEngine;
|
||||
|
||||
println!("{}", "=== Vestige Dashboard ===".cyan().bold());
|
||||
println!();
|
||||
println!(
|
||||
|
|
@ -2467,7 +2491,14 @@ fn run_dashboard(port: u16, open_browser: bool) -> anyhow::Result<()> {
|
|||
|
||||
let rt = tokio::runtime::Runtime::new()?;
|
||||
rt.block_on(async move {
|
||||
vestige_mcp::dashboard::start_dashboard(storage, None, port, open_browser)
|
||||
// Initialize cognitive engine for dream and other cognitive features
|
||||
let cognitive = Arc::new(tokio::sync::Mutex::new(CognitiveEngine::new()));
|
||||
{
|
||||
let mut cog = cognitive.lock().await;
|
||||
cog.hydrate(&storage); // Load persisted connections
|
||||
}
|
||||
|
||||
vestige_mcp::dashboard::start_dashboard(storage, Some(cognitive), port, open_browser)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("Dashboard error: {}", e))
|
||||
})
|
||||
|
|
|
|||
|
|
@ -26,7 +26,20 @@ fn main() -> anyhow::Result<()> {
|
|||
// Parse args
|
||||
let args: Vec<String> = std::env::args().collect();
|
||||
if args.len() < 2 {
|
||||
eprintln!("Usage: vestige-restore <backup.json>");
|
||||
print_usage_stderr();
|
||||
std::process::exit(1);
|
||||
}
|
||||
if matches!(args[1].as_str(), "-h" | "--help") {
|
||||
print_usage_stdout();
|
||||
return Ok(());
|
||||
}
|
||||
if matches!(args[1].as_str(), "-V" | "--version") {
|
||||
println!("vestige-restore {}", env!("CARGO_PKG_VERSION"));
|
||||
return Ok(());
|
||||
}
|
||||
if args.len() > 2 {
|
||||
eprintln!("Unexpected extra argument: {}", args[2]);
|
||||
print_usage_stderr();
|
||||
std::process::exit(1);
|
||||
}
|
||||
|
||||
|
|
@ -91,6 +104,18 @@ fn main() -> anyhow::Result<()> {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
fn print_usage_stdout() {
|
||||
println!("{}", usage());
|
||||
}
|
||||
|
||||
fn print_usage_stderr() {
|
||||
eprintln!("{}", usage());
|
||||
}
|
||||
|
||||
fn usage() -> &'static str {
|
||||
"Vestige Restore\n\nUSAGE:\n vestige-restore <backup.json>\n\nOPTIONS:\n -h, --help Print help information\n -V, --version Print version information"
|
||||
}
|
||||
|
||||
/// Truncate a string for display (UTF-8 safe)
|
||||
fn truncate(s: &str, max_chars: usize) -> String {
|
||||
let s = s.replace('\n', " ");
|
||||
|
|
|
|||
|
|
@ -3,8 +3,9 @@
|
|||
//! v2.0: Adds cognitive operation endpoints (dream, explore, predict, importance, consolidation)
|
||||
|
||||
use std::cmp::Reverse;
|
||||
use std::collections::{BTreeMap, HashSet};
|
||||
use std::fs::{self, OpenOptions};
|
||||
use std::io::Write;
|
||||
use std::io::{BufRead, BufReader, Write};
|
||||
use std::path::{Path as FsPath, PathBuf};
|
||||
|
||||
use axum::extract::{Path, Query, State};
|
||||
|
|
@ -355,6 +356,11 @@ pub struct SanhedrinAppealRequest {
|
|||
pub claim_id: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct SanhedrinTelemetryParams {
|
||||
pub days: Option<i64>,
|
||||
}
|
||||
|
||||
/// Return the latest Sanhedrin receipt written by the Stop-hook bridge.
|
||||
pub async fn get_sanhedrin_latest() -> Result<Json<Value>, StatusCode> {
|
||||
let state_dir = sanhedrin_state_dir();
|
||||
|
|
@ -369,15 +375,124 @@ pub async fn get_sanhedrin_latest() -> Result<Json<Value>, StatusCode> {
|
|||
let raw = fs::read_to_string(&latest_path).map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
let receipt: Value =
|
||||
serde_json::from_str(&raw).map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||
let schema_warning = sanhedrin_schema_warning(&receipt);
|
||||
|
||||
Ok(Json(serde_json::json!({
|
||||
"receipt": receipt,
|
||||
"stateDir": state_dir,
|
||||
"receiptPath": latest_path,
|
||||
"htmlPath": state_dir.join("latest.html"),
|
||||
"schemaWarning": schema_warning,
|
||||
})))
|
||||
}
|
||||
|
||||
/// Return rolling Sanhedrin receipts, appeals, and fail-open counters.
|
||||
pub async fn get_sanhedrin_telemetry(
|
||||
Query(params): Query<SanhedrinTelemetryParams>,
|
||||
) -> Result<Json<Value>, StatusCode> {
|
||||
let state_dir = sanhedrin_state_dir();
|
||||
let days = params.days.unwrap_or(7).clamp(1, 90);
|
||||
let telemetry = tokio::task::spawn_blocking(move || build_sanhedrin_telemetry(state_dir, days))
|
||||
.await
|
||||
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)??;
|
||||
Ok(Json(telemetry))
|
||||
}
|
||||
|
||||
fn build_sanhedrin_telemetry(state_dir: PathBuf, days: i64) -> Result<Value, StatusCode> {
|
||||
let cutoff = Utc::now() - Duration::days(days);
|
||||
let receipts_dir = state_dir.join("receipts");
|
||||
let mut by_verdict = serde_json::Map::new();
|
||||
for verdict in ["PASS", "NOTE", "CAUTION", "VETO", "APPEALED"] {
|
||||
by_verdict.insert(verdict.to_string(), Value::from(0));
|
||||
}
|
||||
let mut by_class: BTreeMap<String, i64> = BTreeMap::new();
|
||||
let mut daily: BTreeMap<String, serde_json::Map<String, Value>> = BTreeMap::new();
|
||||
let mut total_runs = 0i64;
|
||||
let mut last_run_at: Option<DateTime<Utc>> = None;
|
||||
let mut truncated = false;
|
||||
|
||||
if let Ok(entries) = bounded_receipt_entries(&receipts_dir, cutoff) {
|
||||
for path in entries {
|
||||
let Ok(raw) = fs::read_to_string(&path) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(receipt) = serde_json::from_str::<Value>(&raw) else {
|
||||
continue;
|
||||
};
|
||||
let Some(created_at) = parse_sanhedrin_timestamp(&receipt["createdAt"]) else {
|
||||
continue;
|
||||
};
|
||||
if created_at < cutoff {
|
||||
continue;
|
||||
}
|
||||
total_runs += 1;
|
||||
if last_run_at.map(|last| created_at > last).unwrap_or(true) {
|
||||
last_run_at = Some(created_at);
|
||||
}
|
||||
let verdict = receipt
|
||||
.get("verdictBar")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or("NOTE")
|
||||
.to_ascii_uppercase();
|
||||
increment_json_counter(&mut by_verdict, &verdict);
|
||||
let day = created_at.date_naive().to_string();
|
||||
let bucket = daily.entry(day).or_insert_with(|| {
|
||||
let mut map = serde_json::Map::new();
|
||||
map.insert("date".to_string(), Value::String(String::new()));
|
||||
map.insert("total".to_string(), Value::from(0));
|
||||
map.insert("pass".to_string(), Value::from(0));
|
||||
map.insert("note".to_string(), Value::from(0));
|
||||
map.insert("caution".to_string(), Value::from(0));
|
||||
map.insert("veto".to_string(), Value::from(0));
|
||||
map.insert("appealed".to_string(), Value::from(0));
|
||||
map.insert("failOpen".to_string(), Value::from(0));
|
||||
map
|
||||
});
|
||||
increment_json_counter(bucket, "total");
|
||||
match verdict.as_str() {
|
||||
"PASS" => increment_json_counter(bucket, "pass"),
|
||||
"NOTE" => increment_json_counter(bucket, "note"),
|
||||
"CAUTION" => increment_json_counter(bucket, "caution"),
|
||||
"VETO" => increment_json_counter(bucket, "veto"),
|
||||
"APPEALED" => increment_json_counter(bucket, "appealed"),
|
||||
_ => increment_json_counter(bucket, "note"),
|
||||
}
|
||||
|
||||
if let Some(claims) = receipt.get("claims").and_then(Value::as_array) {
|
||||
for claim in claims {
|
||||
let class = known_sanhedrin_class(
|
||||
claim
|
||||
.get("class")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or("UNKNOWN"),
|
||||
);
|
||||
*by_class.entry(class).or_insert(0) += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
truncated = total_runs >= 5_000;
|
||||
}
|
||||
|
||||
let appeals = count_jsonl_since(&state_dir.join("appeals.jsonl"), cutoff, false);
|
||||
let fail_open = count_jsonl_since(&state_dir.join("fail-open.jsonl"), cutoff, true);
|
||||
for (date, bucket) in daily.iter_mut() {
|
||||
bucket.insert("date".to_string(), Value::String(date.clone()));
|
||||
}
|
||||
|
||||
Ok(serde_json::json!({
|
||||
"days": days,
|
||||
"stateDir": state_dir,
|
||||
"totalRuns": total_runs,
|
||||
"byVerdict": by_verdict,
|
||||
"byClass": by_class,
|
||||
"appeals": appeals,
|
||||
"failOpen": fail_open,
|
||||
"truncated": truncated,
|
||||
"lastRunAt": last_run_at.map(|dt| dt.to_rfc3339()),
|
||||
"daily": daily.into_values().map(Value::Object).collect::<Vec<_>>(),
|
||||
}))
|
||||
}
|
||||
|
||||
/// Record feedback that a Sanhedrin veto was stale, wrong, or too strict.
|
||||
///
|
||||
/// This intentionally does not promote, demote, suppress, edit, or delete any
|
||||
|
|
@ -468,6 +583,125 @@ fn sanhedrin_state_dir() -> PathBuf {
|
|||
.unwrap_or_else(|| PathBuf::from(".vestige/sanhedrin"))
|
||||
}
|
||||
|
||||
fn sanhedrin_schema_warning(receipt: &Value) -> Option<String> {
|
||||
let schema = receipt.get("schema").and_then(Value::as_str).unwrap_or("");
|
||||
if schema.is_empty() || schema == "vestige.sanhedrin.receipt.v1" {
|
||||
None
|
||||
} else {
|
||||
Some(format!(
|
||||
"Unsupported Sanhedrin receipt schema '{}'; dashboard expects vestige.sanhedrin.receipt.v1",
|
||||
schema
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_sanhedrin_timestamp(value: &Value) -> Option<DateTime<Utc>> {
|
||||
let raw = value.as_str()?;
|
||||
DateTime::parse_from_rfc3339(raw)
|
||||
.map(|dt| dt.with_timezone(&Utc))
|
||||
.ok()
|
||||
}
|
||||
|
||||
fn bounded_receipt_entries(
|
||||
receipts_dir: &FsPath,
|
||||
cutoff: DateTime<Utc>,
|
||||
) -> Result<Vec<PathBuf>, StatusCode> {
|
||||
const MAX_RECEIPTS: usize = 5_000;
|
||||
const MAX_RECEIPT_BYTES: u64 = 256 * 1024;
|
||||
|
||||
let mut entries: Vec<(Option<DateTime<Utc>>, PathBuf)> = Vec::new();
|
||||
let Ok(read_dir) = fs::read_dir(receipts_dir) else {
|
||||
return Ok(Vec::new());
|
||||
};
|
||||
|
||||
for entry in read_dir.flatten() {
|
||||
let path = entry.path();
|
||||
if path.extension().and_then(|ext| ext.to_str()) != Some("json") {
|
||||
continue;
|
||||
}
|
||||
let Ok(metadata) = fs::symlink_metadata(&path) else {
|
||||
continue;
|
||||
};
|
||||
if metadata.file_type().is_symlink()
|
||||
|| !metadata.is_file()
|
||||
|| metadata.len() > MAX_RECEIPT_BYTES
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let modified = metadata.modified().ok().map(DateTime::<Utc>::from);
|
||||
if modified.map(|mtime| mtime < cutoff).unwrap_or(false) {
|
||||
continue;
|
||||
}
|
||||
entries.push((modified, path));
|
||||
}
|
||||
|
||||
entries.sort_by(|(left_time, _), (right_time, _)| right_time.cmp(left_time));
|
||||
Ok(entries
|
||||
.into_iter()
|
||||
.take(MAX_RECEIPTS)
|
||||
.map(|(_, path)| path)
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn increment_json_counter(map: &mut serde_json::Map<String, Value>, key: &str) {
|
||||
let current = map.get(key).and_then(Value::as_i64).unwrap_or(0);
|
||||
map.insert(key.to_string(), Value::from(current + 1));
|
||||
}
|
||||
|
||||
fn count_jsonl_since(path: &FsPath, cutoff: DateTime<Utc>, distinct_run: bool) -> i64 {
|
||||
const MAX_LEDGER_BYTES: u64 = 2 * 1024 * 1024;
|
||||
let Ok(metadata) = fs::symlink_metadata(path) else {
|
||||
return 0;
|
||||
};
|
||||
if metadata.file_type().is_symlink() || !metadata.is_file() || metadata.len() > MAX_LEDGER_BYTES
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
let Ok(file) = fs::File::open(path) else {
|
||||
return 0;
|
||||
};
|
||||
let mut seen_runs = HashSet::new();
|
||||
let mut count = 0i64;
|
||||
for line in BufReader::new(file).lines().map_while(Result::ok) {
|
||||
let Ok(item) = serde_json::from_str::<Value>(&line) else {
|
||||
continue;
|
||||
};
|
||||
let Some(timestamp) = parse_sanhedrin_timestamp(&item["timestamp"]) else {
|
||||
continue;
|
||||
};
|
||||
if timestamp < cutoff {
|
||||
continue;
|
||||
}
|
||||
if distinct_run
|
||||
&& let Some(run_id) = item.get("runId").and_then(Value::as_str)
|
||||
&& !seen_runs.insert(run_id.to_string())
|
||||
{
|
||||
continue;
|
||||
}
|
||||
count += 1;
|
||||
}
|
||||
count
|
||||
}
|
||||
|
||||
fn known_sanhedrin_class(class: &str) -> String {
|
||||
match class {
|
||||
"receipt_lock"
|
||||
| "TECHNICAL"
|
||||
| "BIOGRAPHICAL"
|
||||
| "FINANCIAL"
|
||||
| "ACHIEVEMENT"
|
||||
| "TIMELINE"
|
||||
| "QUANTITATIVE"
|
||||
| "ATTRIBUTION"
|
||||
| "CAUSAL"
|
||||
| "COMPARATIVE"
|
||||
| "EXISTENTIAL"
|
||||
| "VAGUE-QUANTIFIER"
|
||||
| "UNVERIFIED-POSITIVE" => class.to_string(),
|
||||
_ => "OTHER".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn ensure_sanhedrin_dirs(state_dir: &FsPath) -> Result<(), StatusCode> {
|
||||
fs::create_dir_all(state_dir.join("receipts")).map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -176,8 +176,12 @@ fn build_router_inner(state: AppState, port: u16) -> (Router, AppState) {
|
|||
// Wraps crate::tools::cross_reference::execute. Emits
|
||||
// DeepReferenceCompleted so Graph3D can glide, pulse, and arc.
|
||||
.route("/api/deep_reference", post(handlers::deep_reference_query))
|
||||
// Sanhedrin receipts (v2.1.22): latest local hook verdict + appeal training.
|
||||
// Sanhedrin receipts: latest local hook verdict + appeal training.
|
||||
.route("/api/sanhedrin/latest", get(handlers::get_sanhedrin_latest))
|
||||
.route(
|
||||
"/api/sanhedrin/telemetry",
|
||||
get(handlers::get_sanhedrin_telemetry),
|
||||
)
|
||||
.route("/api/sanhedrin/appeal", post(handlers::appeal_sanhedrin))
|
||||
.layer(
|
||||
ServiceBuilder::new()
|
||||
|
|
|
|||
|
|
@ -243,7 +243,22 @@ fn prepare_storage_path(data_dir: Option<PathBuf>) -> io::Result<Option<PathBuf>
|
|||
};
|
||||
|
||||
let data_dir = expand_tilde(data_dir);
|
||||
fs::create_dir_all(&data_dir)?;
|
||||
|
||||
// Check if path exists and is a file (not a directory)
|
||||
if data_dir.exists() && !data_dir.is_dir() {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidInput,
|
||||
format!(
|
||||
"Data directory path exists but is not a directory: {}",
|
||||
data_dir.display()
|
||||
),
|
||||
));
|
||||
}
|
||||
|
||||
// Only create if it doesn't exist (avoids "File exists" error on existing directories)
|
||||
if !data_dir.exists() {
|
||||
fs::create_dir_all(&data_dir)?;
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
{
|
||||
|
|
|
|||
|
|
@ -57,9 +57,15 @@ pub fn schema() -> Value {
|
|||
"description": "Force creation of a new memory even if similar content exists",
|
||||
"default": false
|
||||
},
|
||||
"batchMergePolicy": {
|
||||
"type": "string",
|
||||
"enum": ["force_create", "smart"],
|
||||
"description": "Batch mode only. Defaults to 'force_create' so caller-separated items stay separate. Use 'smart' to allow Prediction Error Gating against existing memories.",
|
||||
"default": "force_create"
|
||||
},
|
||||
"items": {
|
||||
"type": "array",
|
||||
"description": "Batch mode: array of items to save (max 20). Each runs through full cognitive pipeline with Prediction Error Gating. Use at session end or before context compaction.",
|
||||
"description": "Batch mode: array of items to save (max 20). Defaults to force-creating each caller-separated item; set batchMergePolicy='smart' to allow Prediction Error Gating against existing memories. Use at session end or before context compaction.",
|
||||
"maxItems": 20,
|
||||
"items": {
|
||||
"type": "object",
|
||||
|
|
@ -104,6 +110,7 @@ struct SmartIngestArgs {
|
|||
tags: Option<Vec<String>>,
|
||||
source: Option<String>,
|
||||
force_create: Option<bool>,
|
||||
batch_merge_policy: Option<String>,
|
||||
items: Option<Vec<BatchItem>>,
|
||||
}
|
||||
|
||||
|
|
@ -131,8 +138,26 @@ pub async fn execute(
|
|||
|
||||
// Detect mode: batch (items present) vs single (content present)
|
||||
if let Some(items) = args.items {
|
||||
let global_force = args.force_create.unwrap_or(false);
|
||||
return execute_batch(storage, cognitive, items, global_force).await;
|
||||
let batch_merge_policy = args
|
||||
.batch_merge_policy
|
||||
.unwrap_or_else(|| "force_create".to_string());
|
||||
let default_force_create = match batch_merge_policy.as_str() {
|
||||
"force_create" => true,
|
||||
"smart" => false,
|
||||
other => {
|
||||
return Err(format!(
|
||||
"Invalid batchMergePolicy '{}'. Must be 'force_create' or 'smart'.",
|
||||
other
|
||||
));
|
||||
}
|
||||
};
|
||||
let global_force = match args.force_create {
|
||||
Some(true) => true,
|
||||
Some(false) if batch_merge_policy == "smart" => false,
|
||||
Some(false) => default_force_create,
|
||||
None => default_force_create,
|
||||
};
|
||||
return execute_batch(storage, cognitive, items, global_force, &batch_merge_policy).await;
|
||||
}
|
||||
|
||||
// Single mode: content is required
|
||||
|
|
@ -252,6 +277,9 @@ pub async fn execute(
|
|||
"similarity": result.similarity,
|
||||
"predictionError": result.prediction_error,
|
||||
"supersededId": result.superseded_id,
|
||||
"previousContent": result.previous_content,
|
||||
"mergedFrom": result.merged_from,
|
||||
"mergePreview": result.merge_preview,
|
||||
"importanceScore": importance_composite,
|
||||
"reason": result.reason,
|
||||
"explanation": match result.decision.as_str() {
|
||||
|
|
@ -305,6 +333,7 @@ async fn execute_batch(
|
|||
cognitive: &Arc<Mutex<CognitiveEngine>>,
|
||||
items: Vec<BatchItem>,
|
||||
global_force_create: bool,
|
||||
batch_merge_policy: &str,
|
||||
) -> Result<Value, String> {
|
||||
if items.is_empty() {
|
||||
return Err("Items array cannot be empty".to_string());
|
||||
|
|
@ -321,6 +350,7 @@ async fn execute_batch(
|
|||
let updated = 0u32;
|
||||
let mut skipped = 0u32;
|
||||
let mut errors = 0u32;
|
||||
let mut batch_created_node_ids: Vec<String> = Vec::new();
|
||||
|
||||
for (i, item) in items.into_iter().enumerate() {
|
||||
// Skip empty content
|
||||
|
|
@ -400,6 +430,7 @@ async fn execute_batch(
|
|||
let node_type = node.node_type.clone();
|
||||
|
||||
created += 1;
|
||||
batch_created_node_ids.push(node_id.clone());
|
||||
run_post_ingest(
|
||||
cognitive,
|
||||
&node_id,
|
||||
|
|
@ -431,15 +462,18 @@ async fn execute_batch(
|
|||
|
||||
#[cfg(all(feature = "embeddings", feature = "vector-search"))]
|
||||
{
|
||||
match storage.smart_ingest(input) {
|
||||
match storage.smart_ingest_excluding(input, &batch_created_node_ids) {
|
||||
Ok(result) => {
|
||||
let node_id = result.node.id.clone();
|
||||
let node_content = result.node.content.clone();
|
||||
let node_type = result.node.node_type.clone();
|
||||
|
||||
match result.decision.as_str() {
|
||||
"create" | "supersede" | "replace" => created += 1,
|
||||
"update" | "reinforce" | "merge" | "add_context" => updated += 1,
|
||||
"create" | "supersede" | "merge" => {
|
||||
created += 1;
|
||||
batch_created_node_ids.push(node_id.clone());
|
||||
}
|
||||
"update" | "reinforce" | "replace" | "add_context" => updated += 1,
|
||||
_ => created += 1,
|
||||
}
|
||||
|
||||
|
|
@ -458,6 +492,11 @@ async fn execute_batch(
|
|||
"decision": result.decision,
|
||||
"nodeId": node_id,
|
||||
"similarity": result.similarity,
|
||||
"predictionError": result.prediction_error,
|
||||
"supersededId": result.superseded_id,
|
||||
"previousContent": result.previous_content,
|
||||
"mergedFrom": result.merged_from,
|
||||
"mergePreview": result.merge_preview,
|
||||
"importanceScore": importance_composite,
|
||||
"reason": result.reason
|
||||
}));
|
||||
|
|
@ -482,6 +521,7 @@ async fn execute_batch(
|
|||
let node_type = node.node_type.clone();
|
||||
|
||||
created += 1;
|
||||
batch_created_node_ids.push(node_id.clone());
|
||||
run_post_ingest(
|
||||
cognitive,
|
||||
&node_id,
|
||||
|
|
@ -514,6 +554,7 @@ async fn execute_batch(
|
|||
Ok(serde_json::json!({
|
||||
"success": errors == 0,
|
||||
"mode": "batch",
|
||||
"batchMergePolicy": batch_merge_policy,
|
||||
"summary": {
|
||||
"total": results.len(),
|
||||
"created": created,
|
||||
|
|
@ -637,6 +678,7 @@ mod tests {
|
|||
assert_eq!(schema_value["type"], "object");
|
||||
assert!(schema_value["properties"]["content"].is_object());
|
||||
assert!(schema_value["properties"]["forceCreate"].is_object());
|
||||
assert!(schema_value["properties"]["batchMergePolicy"].is_object());
|
||||
assert!(schema_value["properties"]["items"].is_object());
|
||||
// v1.7: no top-level required — content for single mode, items for batch mode
|
||||
assert!(schema_value.get("required").is_none() || schema_value["required"].is_null());
|
||||
|
|
@ -807,9 +849,53 @@ mod tests {
|
|||
assert!(result.is_ok());
|
||||
let value = result.unwrap();
|
||||
assert_eq!(value["mode"], "batch");
|
||||
assert_eq!(value["batchMergePolicy"], "force_create");
|
||||
assert_eq!(value["summary"]["total"], 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_batch_defaults_to_force_create_for_caller_separated_items() {
|
||||
let (storage, _dir) = test_storage().await;
|
||||
let result = execute(
|
||||
&storage,
|
||||
&test_cognitive(),
|
||||
Some(serde_json::json!({
|
||||
"forceCreate": false,
|
||||
"items": [
|
||||
{ "content": "Jira tickets should not auto-assign sprint fields." },
|
||||
{ "content": "Sprint planning summaries should not append Jira status labels." }
|
||||
]
|
||||
})),
|
||||
)
|
||||
.await;
|
||||
|
||||
let value = result.unwrap();
|
||||
assert_eq!(value["batchMergePolicy"], "force_create");
|
||||
assert_eq!(value["summary"]["created"], 2);
|
||||
assert_eq!(value["summary"]["updated"], 0);
|
||||
for item in value["results"].as_array().unwrap() {
|
||||
assert_eq!(item["decision"], "create");
|
||||
assert!(item["reason"].as_str().unwrap().contains("Forced creation"));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_batch_rejects_invalid_merge_policy() {
|
||||
let (storage, _dir) = test_storage().await;
|
||||
let result = execute(
|
||||
&storage,
|
||||
&test_cognitive(),
|
||||
Some(serde_json::json!({
|
||||
"batchMergePolicy": "merge_everything",
|
||||
"items": [{ "content": "Invalid policy should fail." }]
|
||||
})),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(result.is_err());
|
||||
assert!(result.unwrap_err().contains("batchMergePolicy"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_batch_skips_empty_content() {
|
||||
let (storage, _dir) = test_storage().await;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue