mirror of
https://github.com/samvallad33/vestige.git
synced 2026-06-26 21:39:41 +02:00
Release v2.1.23 Receipt Lock hardening
Hardens Sanhedrin Receipt Lock for model-agnostic use, adds fail-open telemetry and receipt docs, fixes smart_ingest batch safety, wires opt-in CUDA Qwen3 device selection, and refreshes dashboard/release assets. Fixes #58 Fixes #60
This commit is contained in:
parent
a8550410b0
commit
5edb163157
161 changed files with 1775 additions and 262 deletions
|
|
@ -14,18 +14,18 @@
|
|||
//! - Synaptic Tagging: Frey & Morris (1997), Redondo & Morris (2011)
|
||||
//! - Hippocampal Indexing: Teyler & Rudy (2007)
|
||||
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use chrono::{Duration, Utc};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use vestige_core::neuroscience::hippocampal_index::{
|
||||
BarcodeGenerator, ContentPointer, ContentType, HippocampalIndex, HippocampalIndexConfig,
|
||||
INDEX_EMBEDDING_DIM, IndexQuery, MemoryBarcode, MemoryIndex,
|
||||
INDEX_EMBEDDING_DIM, IndexQuery, MemoryBarcode,
|
||||
};
|
||||
use vestige_core::neuroscience::spreading_activation::{
|
||||
ActivatedMemory, ActivationConfig, ActivationNetwork, LinkType,
|
||||
ActivationConfig, ActivationNetwork, LinkType,
|
||||
};
|
||||
use vestige_core::neuroscience::synaptic_tagging::{
|
||||
CaptureWindow, DecayFunction, ImportanceEvent, ImportanceEventType, SynapticTaggingConfig,
|
||||
CaptureWindow, ImportanceEvent, ImportanceEventType, SynapticTaggingConfig,
|
||||
SynapticTaggingSystem,
|
||||
};
|
||||
|
||||
|
|
@ -53,6 +53,7 @@ impl Default for SM2State {
|
|||
|
||||
/// SM-2 grade (0-5)
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
#[allow(dead_code)]
|
||||
enum SM2Grade {
|
||||
CompleteBlackout = 0,
|
||||
Incorrect = 1,
|
||||
|
|
@ -142,6 +143,7 @@ impl Default for FSRS6State {
|
|||
|
||||
/// FSRS-6 grade (1-4)
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
#[allow(dead_code)]
|
||||
enum FSRS6Grade {
|
||||
Again = 1,
|
||||
Hard = 2,
|
||||
|
|
@ -273,6 +275,7 @@ fn leitner_review(state: &LeitnerState, correct: bool) -> LeitnerState {
|
|||
// ============================================================================
|
||||
|
||||
/// Fixed interval - always reviews at same interval
|
||||
#[allow(dead_code)]
|
||||
fn fixed_interval_schedule(_correct: bool) -> i32 {
|
||||
7 // Always 7 days
|
||||
}
|
||||
|
|
@ -409,7 +412,7 @@ fn test_fsrs6_vs_sm2_retention_same_reviews() {
|
|||
// FSRS-6: Same number of reviews
|
||||
let mut fsrs_state = FSRS6State::default();
|
||||
let mut total_elapsed = 0.0;
|
||||
for i in 0..TOTAL_REVIEWS {
|
||||
for _ in 0..TOTAL_REVIEWS {
|
||||
let interval = fsrs6_interval(fsrs_state.stability, 0.9, FSRS6_WEIGHTS[20]).max(1);
|
||||
total_elapsed += interval as f64;
|
||||
fsrs_state = fsrs6_review(&fsrs_state, FSRS6Grade::Good, interval as f64);
|
||||
|
|
@ -426,6 +429,11 @@ fn test_fsrs6_vs_sm2_retention_same_reviews() {
|
|||
"FSRS-6 should maintain high retention: {:.2}%",
|
||||
fsrs_retention * 100.0
|
||||
);
|
||||
assert!(sm2_retention > 0.0, "SM-2 retention should be positive");
|
||||
assert!(
|
||||
total_elapsed > 0.0,
|
||||
"FSRS review elapsed time should accumulate"
|
||||
);
|
||||
}
|
||||
|
||||
/// Test that FSRS-6 achieves better retention efficiency over time.
|
||||
|
|
@ -442,7 +450,8 @@ fn test_fsrs6_vs_sm2_reviews_same_retention() {
|
|||
|
||||
// SM-2: Interval growth is linear with EF
|
||||
// After n successful reviews: interval ≈ previous * 2.5
|
||||
let sm2_intervals = vec![1, 6, 15, 38, 95]; // Approximate SM-2 progression
|
||||
let sm2_intervals = [1, 6, 15, 38, 95]; // Approximate SM-2 progression
|
||||
let sm2_final_interval = sm2_intervals[sm2_intervals.len() - 1];
|
||||
|
||||
// FSRS-6: Stability grows based on forgetting curve parameters
|
||||
// This allows for more nuanced interval optimization
|
||||
|
|
@ -467,6 +476,10 @@ fn test_fsrs6_vs_sm2_reviews_same_retention() {
|
|||
"FSRS-6 should produce positive intervals: {}",
|
||||
fsrs_final_interval
|
||||
);
|
||||
assert!(
|
||||
sm2_final_interval > 0,
|
||||
"SM-2 comparison interval should be positive"
|
||||
);
|
||||
|
||||
// Test that stability has grown from initial value
|
||||
assert!(
|
||||
|
|
@ -545,6 +558,12 @@ fn test_fsrs6_vs_fixed_interval() {
|
|||
final_interval,
|
||||
FIXED_INTERVAL
|
||||
);
|
||||
assert!(
|
||||
fsrs_reviews <= fixed_reviews,
|
||||
"FSRS-6 should need no more reviews than fixed interval: {} <= {}",
|
||||
fsrs_reviews,
|
||||
fixed_reviews
|
||||
);
|
||||
}
|
||||
|
||||
/// Test that FSRS-6 beats Leitner box system.
|
||||
|
|
@ -590,6 +609,12 @@ fn test_fsrs6_vs_leitner() {
|
|||
fsrs_final_interval,
|
||||
leitner_max_interval
|
||||
);
|
||||
assert!(
|
||||
fsrs_reviews <= leitner_reviews,
|
||||
"FSRS-6 should need no more reviews than Leitner: {} <= {}",
|
||||
fsrs_reviews,
|
||||
leitner_reviews
|
||||
);
|
||||
}
|
||||
|
||||
/// Test that personalized w20 parameter improves FSRS-6 results.
|
||||
|
|
|
|||
|
|
@ -790,9 +790,9 @@ async fn test_consolidation_connection_strengthening() {
|
|||
let _ = conn_stats.total_memories;
|
||||
}
|
||||
|
||||
// Both cycles should complete successfully - verify duration is tracked
|
||||
// Both cycles should complete successfully and record monotonically.
|
||||
assert!(
|
||||
first_report.duration_ms > 0 || second_report.duration_ms > 0 || true,
|
||||
second_report.completed_at >= first_report.completed_at,
|
||||
"Both consolidation cycles should complete"
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -67,7 +67,7 @@ fn test_adversarial_empty_inputs() {
|
|||
let _ = whitespace_results.len();
|
||||
|
||||
// System should still work with normal nodes
|
||||
let normal_results = network.activate("source", 1.0);
|
||||
let _normal_results = network.activate("source", 1.0);
|
||||
assert!(
|
||||
network.node_count() >= 2,
|
||||
"Network should contain normal nodes"
|
||||
|
|
@ -323,7 +323,7 @@ fn test_adversarial_config_boundaries() {
|
|||
low_decay_net.add_edge("a".to_string(), "b".to_string(), LinkType::Semantic, 0.9);
|
||||
low_decay_net.add_edge("b".to_string(), "c".to_string(), LinkType::Semantic, 0.9);
|
||||
|
||||
let low_results = low_decay_net.activate("a", 1.0);
|
||||
let _low_results = low_decay_net.activate("a", 1.0);
|
||||
// With 0.01 decay, activation drops to 0.9 * 0.01 = 0.009 after one hop
|
||||
// Then 0.009 * 0.9 * 0.01 = 0.000081 after two hops (below most thresholds)
|
||||
|
||||
|
|
@ -411,7 +411,7 @@ fn test_adversarial_cyclic_graphs() {
|
|||
cycle_net.add_edge("c".to_string(), "a".to_string(), LinkType::Semantic, 0.9);
|
||||
|
||||
let start = std::time::Instant::now();
|
||||
let cycle_results = cycle_net.activate("a", 1.0);
|
||||
let _cycle_results = cycle_net.activate("a", 1.0);
|
||||
let duration = start.elapsed();
|
||||
|
||||
// Should still complete quickly
|
||||
|
|
@ -487,7 +487,7 @@ fn test_adversarial_special_numeric_values() {
|
|||
// (The implementation should clamp or validate these)
|
||||
|
||||
// Test with 0.0 activation (should produce no results or minimal)
|
||||
let zero_results = network.activate("normal", 0.0);
|
||||
let _zero_results = network.activate("normal", 0.0);
|
||||
// Might be empty or have very low activation
|
||||
|
||||
// Test with very small activation
|
||||
|
|
|
|||
|
|
@ -144,6 +144,13 @@ fn test_chaos_add_remove_cycles() {
|
|||
|
||||
// Stable structure should be preserved (edges reinforced)
|
||||
let stable_edge_count = network.edge_count();
|
||||
let stable_node_count = network.node_count();
|
||||
assert!(
|
||||
stable_node_count >= initial_node_count,
|
||||
"Stable nodes should be preserved: {} >= {}",
|
||||
stable_node_count,
|
||||
initial_node_count
|
||||
);
|
||||
assert!(
|
||||
stable_edge_count >= initial_edge_count,
|
||||
"Stable edges should be preserved: {} >= {}",
|
||||
|
|
@ -450,8 +457,8 @@ fn test_chaos_ancient_memories() {
|
|||
|
||||
// System should handle this gracefully
|
||||
assert!(
|
||||
result.captured_count() >= 0,
|
||||
"System should handle importance triggering"
|
||||
result.captured_count() <= 3,
|
||||
"Importance triggering should stay bounded by active memories"
|
||||
);
|
||||
|
||||
// All memories should be accessible
|
||||
|
|
|
|||
|
|
@ -373,7 +373,7 @@ fn test_proof_hippocampal_indexing_efficiency() {
|
|||
bf_results.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap());
|
||||
bf_results.truncate(10);
|
||||
|
||||
let bf_duration = bf_start.elapsed();
|
||||
let _bf_duration = bf_start.elapsed();
|
||||
|
||||
// === PROOF OF EFFICIENCY ===
|
||||
|
||||
|
|
@ -566,7 +566,7 @@ fn test_proof_comprehensive_capability_summary() {
|
|||
// === CAPABILITY 4: Asymmetric Temporal Windows ===
|
||||
// Traditional: NO temporal reasoning | Vestige: Biologically-grounded windows
|
||||
|
||||
let window = CaptureWindow::new(9.0, 2.0);
|
||||
let _window = CaptureWindow::new(9.0, 2.0);
|
||||
let asymmetric = 9.0 / 2.0;
|
||||
assert!(
|
||||
asymmetric > 4.0,
|
||||
|
|
|
|||
|
|
@ -303,7 +303,7 @@ fn test_research_frey_morris_synaptic_tagging() {
|
|||
/// theory and episodic memory: updating the index. Hippocampus, 17(12), 1158-1169.
|
||||
#[test]
|
||||
fn test_research_teyler_rudy_hippocampal_indexing() {
|
||||
let config = HippocampalIndexConfig::default();
|
||||
let _config = HippocampalIndexConfig::default();
|
||||
let index = HippocampalIndex::new();
|
||||
let now = Utc::now();
|
||||
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ fn make_dream_memory(id: &str, content: &str, tags: Vec<&str>) -> DreamMemory {
|
|||
}
|
||||
|
||||
/// Create a memory with specific age
|
||||
#[allow(dead_code)]
|
||||
fn make_aged_memory(id: &str, content: &str, tags: Vec<&str>, hours_ago: i64) -> DreamMemory {
|
||||
DreamMemory {
|
||||
id: id.to_string(),
|
||||
|
|
@ -50,6 +51,7 @@ fn make_aged_memory(id: &str, content: &str, tags: Vec<&str>, hours_ago: i64) ->
|
|||
}
|
||||
|
||||
/// Create a memory with access count
|
||||
#[allow(dead_code)]
|
||||
fn make_accessed_memory(
|
||||
id: &str,
|
||||
content: &str,
|
||||
|
|
@ -391,14 +393,14 @@ fn test_connection_graph_decay_and_pruning() {
|
|||
graph.apply_decay(0.5);
|
||||
|
||||
// Prune weak connections
|
||||
let pruned = graph.prune_weak(0.2);
|
||||
let _pruned = graph.prune_weak(0.2);
|
||||
|
||||
// Weak connection (0.3 * 0.5 = 0.15) should be pruned
|
||||
// The pruned count depends on implementation details
|
||||
let stats = graph.get_stats();
|
||||
assert!(
|
||||
stats.total_connections >= 0,
|
||||
"Should have non-negative connections after pruning"
|
||||
stats.total_connections <= 3,
|
||||
"Pruning should not increase connection count"
|
||||
);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -106,13 +106,10 @@ fn test_recall_finds_memories_by_content() {
|
|||
assert_eq!(recall.min_retention, 0.5);
|
||||
|
||||
// Verify search mode
|
||||
match recall.search_mode {
|
||||
SearchMode::Keyword => {
|
||||
// Keyword search uses FTS5
|
||||
assert!(true, "Keyword mode should be supported");
|
||||
}
|
||||
_ => panic!("Expected Keyword search mode"),
|
||||
}
|
||||
assert!(
|
||||
matches!(&recall.search_mode, SearchMode::Keyword),
|
||||
"Expected Keyword search mode"
|
||||
);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
|
|
@ -218,15 +215,10 @@ fn test_memory_lifecycle_follows_expected_pattern() {
|
|||
);
|
||||
|
||||
// Verify state is Review (mature)
|
||||
match state.state {
|
||||
LearningState::Review => {
|
||||
assert!(true, "Mature memory should be in Review state");
|
||||
}
|
||||
_ => {
|
||||
// Also acceptable - depends on FSRS parameters
|
||||
assert!(state.reps >= 10, "Should have processed all reviews");
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
matches!(&state.state, LearningState::Review) || state.reps >= 10,
|
||||
"Mature memory should be in Review or have processed all reviews"
|
||||
);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
|
|
|
|||
|
|
@ -91,7 +91,7 @@ fn test_debugging_intent_detection() {
|
|||
match &result.primary_intent {
|
||||
DetectedIntent::Debugging {
|
||||
suspected_area,
|
||||
symptoms,
|
||||
symptoms: _,
|
||||
} => {
|
||||
assert!(!suspected_area.is_empty(), "Should identify suspected area");
|
||||
// Symptoms may or may not be captured depending on action order
|
||||
|
|
@ -125,7 +125,7 @@ fn test_learning_intent_detection() {
|
|||
|
||||
// Should detect learning with high confidence
|
||||
match &result.primary_intent {
|
||||
DetectedIntent::Learning { topic, level } => {
|
||||
DetectedIntent::Learning { topic, level: _ } => {
|
||||
assert!(!topic.is_empty(), "Should identify learning topic");
|
||||
// Level may vary
|
||||
}
|
||||
|
|
@ -137,7 +137,7 @@ fn test_learning_intent_detection() {
|
|||
}
|
||||
|
||||
// Verify relevant tags
|
||||
let tags = result.primary_intent.relevant_tags();
|
||||
let _tags = result.primary_intent.relevant_tags();
|
||||
// Tags depend on detected intent type
|
||||
}
|
||||
|
||||
|
|
@ -173,9 +173,10 @@ fn test_refactoring_intent_detection() {
|
|||
related_components, ..
|
||||
} => {
|
||||
// Multiple edits could also suggest new feature
|
||||
let _ = related_components;
|
||||
assert!(
|
||||
related_components.len() >= 0,
|
||||
"Should track related components"
|
||||
result.confidence > 0.0,
|
||||
"New feature intent should have positive confidence"
|
||||
);
|
||||
}
|
||||
_ => {
|
||||
|
|
|
|||
|
|
@ -39,7 +39,17 @@ def patched_attr(obj, name, value):
|
|||
|
||||
class SanhedrinClaimModeTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
for key in (
|
||||
"VESTIGE_SANHEDRIN_CLAIM_MODE",
|
||||
"VESTIGE_SANHEDRIN_OUTPUT",
|
||||
"VESTIGE_SANHEDRIN_STAGE_FILE",
|
||||
"VESTIGE_SANHEDRIN_TRANSCRIPT",
|
||||
"VESTIGE_SANHEDRIN_ALLOW_COMMAND_LEDGER",
|
||||
):
|
||||
os.environ.pop(key, None)
|
||||
self.sanhedrin = load_sanhedrin()
|
||||
self.sanhedrin.SANHEDRIN_ENDPOINT = "http://127.0.0.1:8080/v1/chat/completions"
|
||||
self.sanhedrin.MODEL = "test-verifier"
|
||||
|
||||
@contextlib.contextmanager
|
||||
def isolated_receipt_state(self):
|
||||
|
|
@ -64,6 +74,13 @@ class SanhedrinClaimModeTests(unittest.TestCase):
|
|||
self.sanhedrin.main()
|
||||
return stdout.getvalue().strip()
|
||||
|
||||
def test_runtime_has_no_implicit_verifier_model_default(self):
|
||||
with mock.patch.dict(os.environ, {}, clear=True):
|
||||
module = load_sanhedrin()
|
||||
|
||||
self.assertEqual(module.SANHEDRIN_ENDPOINT, "")
|
||||
self.assertEqual(module.MODEL, "")
|
||||
|
||||
def test_receipt_lock_blocks_unbacked_test_claim(self):
|
||||
with self.isolated_receipt_state() as state_dir:
|
||||
out = self.run_main("All tests passed.")
|
||||
|
|
@ -112,6 +129,74 @@ class SanhedrinClaimModeTests(unittest.TestCase):
|
|||
self.assertEqual(receipt["verdictBar"], "APPEALED")
|
||||
self.assertEqual(receipt["claims"][0]["decision"], "appealed")
|
||||
|
||||
def test_receipt_lock_ignores_quotes_fences_and_hedged_verification(self):
|
||||
examples = [
|
||||
'The user said "all tests passed" earlier.',
|
||||
"> all tests passed\nI still need to verify this myself.",
|
||||
"```text\nall tests passed\n```",
|
||||
"I think the tests passed before, but let me verify.",
|
||||
]
|
||||
for example in examples:
|
||||
with self.subTest(example=example), self.isolated_receipt_state() as state_dir:
|
||||
out = self.run_main(example)
|
||||
self.assertEqual(out, "yes")
|
||||
latest = state_dir / "latest.json"
|
||||
if latest.exists():
|
||||
receipt = json.loads(latest.read_text(encoding="utf-8"))
|
||||
self.assertNotEqual(receipt["verdictBar"], "VETO")
|
||||
|
||||
def test_claim_mode_ignores_quoted_and_blockquoted_verification_text(self):
|
||||
examples = [
|
||||
'The user said "all tests passed" earlier.',
|
||||
"> all tests passed\nI still need to verify this myself.",
|
||||
"```text\nall tests passed\n```",
|
||||
]
|
||||
env = {"VESTIGE_SANHEDRIN_CLAIM_MODE": "1", "VESTIGE_SANHEDRIN_OUTPUT": "json"}
|
||||
with mock.patch.dict(os.environ, env, clear=False), patched_attr(
|
||||
self.sanhedrin, "fetch_claim_evidence", lambda _claim: ([], True)
|
||||
):
|
||||
for example in examples:
|
||||
with self.subTest(example=example):
|
||||
out = self.run_main(example)
|
||||
result = json.loads(out)
|
||||
|
||||
self.assertTrue(result["passed"], result)
|
||||
|
||||
def test_receipt_lock_still_blocks_temporal_or_apostrophe_claims(self):
|
||||
examples = [
|
||||
"All tests passed before I pushed the fix.",
|
||||
"All tests passed earlier on the staging branch.",
|
||||
"All tests passed last run.",
|
||||
"Sam's tests passed today.",
|
||||
]
|
||||
for example in examples:
|
||||
with self.subTest(example=example), self.isolated_receipt_state() as state_dir:
|
||||
out = self.run_main(example)
|
||||
receipt = json.loads((state_dir / "latest.json").read_text(encoding="utf-8"))
|
||||
|
||||
self.assertIn("Receipt Lock", out)
|
||||
self.assertEqual(receipt["verdictBar"], "VETO")
|
||||
|
||||
def test_loose_transcript_command_scan_is_disabled_by_default(self):
|
||||
with self.isolated_receipt_state() as state_dir:
|
||||
transcript = state_dir / "transcript.jsonl"
|
||||
transcript.write_text(
|
||||
json.dumps({
|
||||
"role": "assistant",
|
||||
"message": {
|
||||
"content": 'I will not run it, but here is {"command":"cargo test","exit_code":0}.'
|
||||
},
|
||||
}) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
with mock.patch.dict(os.environ, {"VESTIGE_SANHEDRIN_TRANSCRIPT": str(transcript)}, clear=False):
|
||||
out = self.run_main("All tests passed.")
|
||||
receipt = json.loads((state_dir / "latest.json").read_text(encoding="utf-8"))
|
||||
|
||||
self.assertIn("Receipt Lock", out)
|
||||
self.assertEqual(receipt["verdictBar"], "VETO")
|
||||
self.assertEqual(receipt["receipts"], [])
|
||||
|
||||
def test_plain_sam_biographical_achievement_claim_is_check_worthy(self):
|
||||
claims = self.sanhedrin.extract_check_worthy_claims(
|
||||
"Sam graduated from Example University and won the Example AI Challenge."
|
||||
|
|
@ -139,6 +224,23 @@ class SanhedrinClaimModeTests(unittest.TestCase):
|
|||
self.assertTrue(result["legacy_verdict"].startswith("no - "), result)
|
||||
self.assertEqual(result["verdicts"][0]["status"], "REFUTED_BY_ABSENCE")
|
||||
|
||||
def test_missing_model_configuration_fails_open_except_receipt_lock(self):
|
||||
env = {
|
||||
"VESTIGE_SANHEDRIN_CLAIM_MODE": "1",
|
||||
"VESTIGE_SANHEDRIN_OUTPUT": "json",
|
||||
}
|
||||
with mock.patch.dict(os.environ, env, clear=False), patched_attr(
|
||||
self.sanhedrin, "SANHEDRIN_ENDPOINT", ""
|
||||
), patched_attr(self.sanhedrin, "MODEL", ""), patched_attr(
|
||||
self.sanhedrin, "fetch_claim_evidence", lambda _claim: ([], True)
|
||||
):
|
||||
out = self.run_main("Sam attended Example University.")
|
||||
|
||||
result = json.loads(out)
|
||||
self.assertTrue(result["passed"], result)
|
||||
self.assertEqual(result["verdicts"][0]["status"], "NEI")
|
||||
self.assertIn("model not configured", result["verdicts"][0]["reason"])
|
||||
|
||||
def test_vague_user_positive_claim_fails_closed(self):
|
||||
env = {"VESTIGE_SANHEDRIN_CLAIM_MODE": "1", "VESTIGE_SANHEDRIN_OUTPUT": "json"}
|
||||
with mock.patch.dict(os.environ, env, clear=False), patched_attr(
|
||||
|
|
@ -264,6 +366,24 @@ class SanhedrinClaimModeTests(unittest.TestCase):
|
|||
self.assertEqual(result["verdicts"][0]["status"], "NEI")
|
||||
self.assertEqual(result["verdicts"][0]["claim"]["claim_class"], "TECHNICAL")
|
||||
|
||||
def test_claim_sampling_keeps_late_high_severity_claim(self):
|
||||
technical = " ".join(
|
||||
f"The /tmp/example_{i}.py script calls the MCP endpoint successfully."
|
||||
for i in range(12)
|
||||
)
|
||||
claims = self.sanhedrin.extract_check_worthy_claims(
|
||||
f"{technical} Sam won first place at the Example AI Challenge."
|
||||
)
|
||||
|
||||
self.assertLessEqual(len(claims), self.sanhedrin.MAX_CLAIMS)
|
||||
self.assertTrue(
|
||||
any(
|
||||
claim.sam_critical and claim.claim_class == "ACHIEVEMENT"
|
||||
for claim in claims
|
||||
),
|
||||
claims,
|
||||
)
|
||||
|
||||
def test_fetch_evidence_truncates_on_python_character_boundary(self):
|
||||
emoji_out = self.sanhedrin.truncate_chars(("a" * 4) + "🙂" + "tail", 8)
|
||||
combining_out = self.sanhedrin.truncate_chars("Cafe\u0301 tail", 8)
|
||||
|
|
@ -360,7 +480,9 @@ class SanhedrinClaimModeTests(unittest.TestCase):
|
|||
}
|
||||
with mock.patch.dict(os.environ, env, clear=False), patched_attr(
|
||||
self.sanhedrin, "post_json", fake_post_json
|
||||
):
|
||||
), patched_attr(
|
||||
self.sanhedrin, "SANHEDRIN_ENDPOINT", "http://127.0.0.1:8080/v1/chat/completions"
|
||||
), patched_attr(self.sanhedrin, "MODEL", "test-verifier"):
|
||||
out = self.run_main(
|
||||
"Qwen3.6-35B can be served through an OpenAI-compatible chat endpoint."
|
||||
)
|
||||
|
|
@ -372,6 +494,168 @@ class SanhedrinClaimModeTests(unittest.TestCase):
|
|||
self.assertEqual(verdict["durable_evidence_count"], 0)
|
||||
self.assertIn("Durable evidence required", verdict["reason"])
|
||||
|
||||
def test_staged_only_supported_verdict_is_downgraded_without_durable_evidence(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
staged_path = Path(tmp) / "sanhedrin-staged-evidence.json"
|
||||
staged_path.write_text(
|
||||
json.dumps(
|
||||
[
|
||||
{
|
||||
"id": "stage-tech",
|
||||
"trust": 0.95,
|
||||
"preview": "Qwen3.6-35B can be served through a chat endpoint.",
|
||||
}
|
||||
]
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
def fake_post_json(url, _body, _timeout):
|
||||
if url == self.sanhedrin.VESTIGE_ENDPOINT:
|
||||
return {"confidence": 0.0, "evidence": []}
|
||||
if url == self.sanhedrin.SANHEDRIN_ENDPOINT:
|
||||
return {
|
||||
"choices": [
|
||||
{
|
||||
"message": {
|
||||
"content": json.dumps(
|
||||
{
|
||||
"status": "SUPPORTED",
|
||||
"class": "TECHNICAL",
|
||||
"reason": "Staged evidence supports the claim.",
|
||||
"evidence_ids": ["stage-tech"],
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
self.fail(f"unexpected post_json URL: {url}")
|
||||
|
||||
env = {
|
||||
"VESTIGE_SANHEDRIN_CLAIM_MODE": "1",
|
||||
"VESTIGE_SANHEDRIN_OUTPUT": "json",
|
||||
"VESTIGE_SANHEDRIN_STAGE_FILE": str(staged_path),
|
||||
}
|
||||
with mock.patch.dict(os.environ, env, clear=False), patched_attr(
|
||||
self.sanhedrin, "post_json", fake_post_json
|
||||
), patched_attr(
|
||||
self.sanhedrin, "SANHEDRIN_ENDPOINT", "http://127.0.0.1:8080/v1/chat/completions"
|
||||
), patched_attr(self.sanhedrin, "MODEL", "test-verifier"):
|
||||
out = self.run_main(
|
||||
"Qwen3.6-35B can be served through an OpenAI-compatible chat endpoint."
|
||||
)
|
||||
|
||||
result = json.loads(out)
|
||||
verdict = result["verdicts"][0]
|
||||
self.assertTrue(result["passed"], result)
|
||||
self.assertEqual(verdict["status"], "NEI")
|
||||
self.assertEqual(verdict["durable_evidence_count"], 0)
|
||||
self.assertIn("Durable evidence required", verdict["reason"])
|
||||
|
||||
def test_supported_verdict_with_durable_evidence_is_preserved(self):
|
||||
evidence = [
|
||||
self.sanhedrin.EvidenceItem(
|
||||
id="mem-durable",
|
||||
preview="A reliable memory says this backend can use a compatible endpoint.",
|
||||
trust=0.95,
|
||||
durable=True,
|
||||
source="vestige",
|
||||
)
|
||||
]
|
||||
claim = self.sanhedrin.Claim(
|
||||
text="Qwen3.6-35B can be served through an OpenAI-compatible chat endpoint.",
|
||||
claim_class="TECHNICAL",
|
||||
source_index=0,
|
||||
sam_critical=False,
|
||||
)
|
||||
verdict = self.sanhedrin.validate_structured_verdict(
|
||||
claim,
|
||||
{"status": "SUPPORTED", "class": "TECHNICAL", "reason": "Evidence supports it."},
|
||||
evidence,
|
||||
)
|
||||
|
||||
self.assertEqual(verdict.status, "SUPPORTED")
|
||||
|
||||
def test_openai_key_is_not_forwarded_to_arbitrary_or_vestige_endpoints(self):
|
||||
captured_headers = []
|
||||
|
||||
class FakeResponse:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
def read(self):
|
||||
return b"{}"
|
||||
|
||||
def fake_urlopen(req, timeout=None):
|
||||
captured_headers.append(dict(req.header_items()))
|
||||
return FakeResponse()
|
||||
|
||||
env = {"OPENAI_API_KEY": "real-openai-key"}
|
||||
with mock.patch.dict(os.environ, env, clear=False), patched_attr(
|
||||
self.sanhedrin, "SANHEDRIN_ENDPOINT", "http://127.0.0.1:8080/v1/chat/completions"
|
||||
), mock.patch.object(
|
||||
self.sanhedrin.urllib.request, "urlopen", fake_urlopen
|
||||
):
|
||||
self.sanhedrin.post_json(self.sanhedrin.SANHEDRIN_ENDPOINT, {}, 1)
|
||||
self.sanhedrin.post_json(self.sanhedrin.VESTIGE_ENDPOINT, {}, 1)
|
||||
|
||||
self.assertTrue(captured_headers)
|
||||
self.assertTrue(all("Authorization" not in headers for headers in captured_headers))
|
||||
|
||||
def test_sanhedrin_api_key_only_goes_to_configured_sanhedrin_endpoint(self):
|
||||
captured_headers = []
|
||||
|
||||
class FakeResponse:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
def read(self):
|
||||
return b"{}"
|
||||
|
||||
def fake_urlopen(req, timeout=None):
|
||||
captured_headers.append(dict(req.header_items()))
|
||||
return FakeResponse()
|
||||
|
||||
env = {"VESTIGE_SANHEDRIN_API_KEY": "sanhedrin-only-key"}
|
||||
with mock.patch.dict(os.environ, env, clear=False), patched_attr(
|
||||
self.sanhedrin, "SANHEDRIN_ENDPOINT", "http://127.0.0.1:8080/v1/chat/completions"
|
||||
), mock.patch.object(
|
||||
self.sanhedrin.urllib.request, "urlopen", fake_urlopen
|
||||
):
|
||||
self.sanhedrin.post_json(self.sanhedrin.SANHEDRIN_ENDPOINT, {}, 1)
|
||||
self.sanhedrin.post_json(self.sanhedrin.VESTIGE_ENDPOINT, {}, 1)
|
||||
|
||||
self.assertIn("Authorization", captured_headers[0])
|
||||
self.assertNotIn("Authorization", captured_headers[1])
|
||||
|
||||
def test_strict_openai_body_omits_backend_specific_fields(self):
|
||||
with patched_attr(self.sanhedrin, "SANHEDRIN_BACKEND", "openai"):
|
||||
body = self.sanhedrin.sanhedrin_body(
|
||||
[{"role": "user", "content": "judge"}],
|
||||
128,
|
||||
)
|
||||
|
||||
self.assertNotIn("top_k", body)
|
||||
self.assertNotIn("seed", body)
|
||||
self.assertNotIn("chat_template_kwargs", body)
|
||||
|
||||
def test_mlx_body_keeps_backend_specific_fields(self):
|
||||
with patched_attr(self.sanhedrin, "SANHEDRIN_BACKEND", "mlx"):
|
||||
body = self.sanhedrin.sanhedrin_body(
|
||||
[{"role": "user", "content": "judge"}],
|
||||
128,
|
||||
)
|
||||
|
||||
self.assertEqual(body["top_k"], 1)
|
||||
self.assertEqual(body["chat_template_kwargs"], {"enable_thinking": False})
|
||||
|
||||
def test_staged_only_legacy_refuted_line_is_downgraded_without_durable_evidence(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
staged_path = Path(tmp) / "sanhedrin-staged-evidence.json"
|
||||
|
|
@ -413,7 +697,9 @@ class SanhedrinClaimModeTests(unittest.TestCase):
|
|||
}
|
||||
with mock.patch.dict(os.environ, env, clear=False), patched_attr(
|
||||
self.sanhedrin, "post_json", fake_post_json
|
||||
):
|
||||
), patched_attr(
|
||||
self.sanhedrin, "SANHEDRIN_ENDPOINT", "http://127.0.0.1:8080/v1/chat/completions"
|
||||
), patched_attr(self.sanhedrin, "MODEL", "test-verifier"):
|
||||
out = self.run_main(
|
||||
"Qwen3.6-35B can be served through an OpenAI-compatible chat endpoint."
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue