diff --git a/crates/omnigraph-cluster/src/lib.rs b/crates/omnigraph-cluster/src/lib.rs index bb0c66b..da80710 100644 --- a/crates/omnigraph-cluster/src/lib.rs +++ b/crates/omnigraph-cluster/src/lib.rs @@ -4838,3020 +4838,7 @@ fn display_path(path: &Path) -> String { path.display().to_string() } + #[cfg(test)] -mod tests { - use std::fs; - use std::path::Path; - - use omnigraph::db::Omnigraph; - use serde_json::json; - use tempfile::tempdir; - - use super::*; - - const SCHEMA: &str = r#" -node Person { - name: String @key - age: I32? -} -"#; - - const QUERY: &str = r#" -query find_person($name: String) { - match { $p: Person { name: $name } } - return { $p.name, $p.age } -} -"#; - - fn fixture() -> tempfile::TempDir { - let dir = tempdir().unwrap(); - fs::write(dir.path().join("people.pg"), SCHEMA).unwrap(); - fs::write(dir.path().join("people.gq"), QUERY).unwrap(); - fs::write(dir.path().join("base.policy.yaml"), "rules: []\n").unwrap(); - fs::write( - dir.path().join(CLUSTER_CONFIG_FILE), - r#" -version: 1 -metadata: - name: test -state: - backend: cluster - lock: true -graphs: - knowledge: - schema: ./people.pg - queries: - find_person: - file: ./people.gq -policies: - base: - file: ./base.policy.yaml - applies_to: [knowledge] -"#, - ) - .unwrap(); - dir - } - - async fn init_derived_graph(root: &Path) { - let graph_dir = root.join(CLUSTER_GRAPHS_DIR); - fs::create_dir_all(&graph_dir).unwrap(); - let graph = graph_dir.join("knowledge.omni"); - Omnigraph::init(graph.to_string_lossy().as_ref(), SCHEMA) - .await - .unwrap(); - } - - fn write_lock_file(config_dir: &Path, lock_id: &str, operation: &str) { - let state_dir = config_dir.join(CLUSTER_STATE_DIR); - fs::create_dir_all(&state_dir).unwrap(); - fs::write( - state_dir.join("lock.json"), - json!({ - "version": 1, - "lock_id": lock_id, - "operation": operation, - "created_at": "1970-01-01T00:00:00Z", - "pid": 123 - }) - .to_string(), - ) - .unwrap(); - } - - #[test] - fn valid_minimal_config() { - let dir = fixture(); - let out = validate_config_dir(dir.path()); - assert!(out.ok, "{:?}", out.diagnostics); - assert!(out.resource_digests.contains_key("graph.knowledge")); - assert!(out.resource_digests.contains_key("schema.knowledge")); - assert!( - out.dependencies - .iter() - .any(|dep| dep.from == "policy.base" && dep.to == "graph.knowledge") - ); - } - - #[test] - fn unknown_field_rejection() { - let dir = fixture(); - fs::write( - dir.path().join(CLUSTER_CONFIG_FILE), - "version: 1\ngraphs: {}\nwat: true\n", - ) - .unwrap(); - let out = validate_config_dir(dir.path()); - assert!(!out.ok); - assert!(out.diagnostics[0].message.contains("unknown field")); - } - - #[test] - fn future_phase_field_rejection() { - let dir = fixture(); - fs::write( - dir.path().join(CLUSTER_CONFIG_FILE), - "version: 1\ngraphs: {}\npipelines: {}\n", - ) - .unwrap(); - let out = validate_config_dir(dir.path()); - assert!(!out.ok); - assert_eq!(out.diagnostics[0].code, "future_phase_field"); - } - - #[test] - fn duplicate_yaml_key_rejection() { - let dir = fixture(); - fs::write( - dir.path().join(CLUSTER_CONFIG_FILE), - "version: 1\ngraphs: {}\ngraphs: {}\n", - ) - .unwrap(); - let out = validate_config_dir(dir.path()); - assert!(!out.ok); - assert_eq!(out.diagnostics[0].code, "duplicate_yaml_key"); - } - - #[test] - fn duplicate_yaml_key_rejection_keeps_quoted_hashes() { - let diagnostics = - duplicate_key_diagnostics("\"name#display\": one\n\"name#display\": two\n"); - assert_eq!(diagnostics.len(), 1); - assert_eq!(diagnostics[0].code, "duplicate_yaml_key"); - } - - #[test] - fn missing_schema_query_and_policy_files() { - let dir = fixture(); - fs::write( - dir.path().join(CLUSTER_CONFIG_FILE), - r#" -version: 1 -graphs: - knowledge: - schema: ./missing.pg - queries: - find_person: { file: ./missing.gq } -policies: - base: - file: ./missing.policy.yaml - applies_to: [knowledge] -"#, - ) - .unwrap(); - let out = validate_config_dir(dir.path()); - assert!(!out.ok); - let codes: BTreeSet<_> = out.diagnostics.iter().map(|d| d.code.as_str()).collect(); - assert!(codes.contains("schema_file_missing")); - assert!(codes.contains("query_file_missing")); - assert!(codes.contains("policy_file_missing")); - } - - #[test] - fn wrong_kind_and_dangling_refs_fail() { - let dir = fixture(); - fs::write( - dir.path().join(CLUSTER_CONFIG_FILE), - r#" -version: 1 -graphs: - knowledge: - schema: ./people.pg -policies: - base: - file: ./base.policy.yaml - applies_to: [query.knowledge.find_person, missing] -"#, - ) - .unwrap(); - let out = validate_config_dir(dir.path()); - assert!(!out.ok); - let codes: BTreeSet<_> = out.diagnostics.iter().map(|d| d.code.as_str()).collect(); - assert!(codes.contains("wrong_kind_reference")); - assert!(codes.contains("dangling_graph_reference")); - } - - #[test] - fn query_key_mismatch_fails() { - let dir = fixture(); - fs::write( - dir.path().join(CLUSTER_CONFIG_FILE), - r#" -version: 1 -graphs: - knowledge: - schema: ./people.pg - queries: - different: { file: ./people.gq } -"#, - ) - .unwrap(); - let out = validate_config_dir(dir.path()); - assert!(!out.ok); - assert_eq!(out.diagnostics[0].code, "query_key_mismatch"); - } - - #[test] - fn query_typecheck_failure_fails() { - let dir = fixture(); - fs::write( - dir.path().join("people.gq"), - "query find_person() { match { $d: DoesNotExist } return { $d.name } }\n", - ) - .unwrap(); - let out = validate_config_dir(dir.path()); - assert!(!out.ok); - assert!( - out.diagnostics - .iter() - .any(|diagnostic| diagnostic.code == "query_typecheck_error") - ); - } - - #[tokio::test] - async fn missing_state_plans_creates() { - let dir = fixture(); - let out = plan_config_dir(dir.path()).await; - assert!(out.ok, "{:?}", out.diagnostics); - assert!(!out.state_observations.state_found); - assert!(!out.state_observations.locked); - assert!(out.state_observations.lock_acquired); - assert!( - out.changes - .iter() - .all(|c| c.operation == PlanOperation::Create) - ); - assert!(out.changes.iter().any(|c| c.resource == "graph.knowledge")); - assert!(!dir.path().join(CLUSTER_LOCK_FILE).exists()); - } - - #[tokio::test] - async fn config_digest_ignores_yaml_comments_and_formatting() { - let dir = fixture(); - let first = plan_config_dir(dir.path()).await; - assert!(first.ok, "{:?}", first.diagnostics); - - fs::write( - dir.path().join(CLUSTER_CONFIG_FILE), - r#" -# Same semantic config as the fixture, intentionally rendered differently. -version: 1 -metadata: { name: test } -state: { backend: cluster, lock: true } -graphs: - knowledge: - schema: ./people.pg - queries: { find_person: { file: ./people.gq } } -policies: - base: - file: ./base.policy.yaml - applies_to: - - knowledge -"#, - ) - .unwrap(); - - let second = plan_config_dir(dir.path()).await; - assert!(second.ok, "{:?}", second.diagnostics); - assert_eq!( - first.desired_revision.config_digest, - second.desired_revision.config_digest - ); - } - - #[tokio::test] - async fn existing_state_plans_update_and_delete_deterministically() { - let dir = fixture(); - let first = plan_config_dir(dir.path()).await; - let state_dir = dir.path().join("__cluster"); - fs::create_dir_all(&state_dir).unwrap(); - fs::write( - state_dir.join("state.json"), - serde_json::to_string_pretty(&json!({ - "version": 1, - "applied_revision": { - "config_digest": "old", - "resources": { - "graph.knowledge": { "digest": first.resource_digests["graph.knowledge"] }, - "policy.old": { "digest": "abc" }, - "schema.knowledge": { "digest": "old-schema" } - } - } - })) - .unwrap(), - ) - .unwrap(); - - let out = plan_config_dir(dir.path()).await; - assert!(out.ok, "{:?}", out.diagnostics); - let rendered: Vec<_> = out - .changes - .iter() - .map(|change| (change.resource.as_str(), &change.operation)) - .collect(); - assert_eq!( - rendered, - vec![ - ("policy.base", &PlanOperation::Create), - ("policy.old", &PlanOperation::Delete), - ("query.knowledge.find_person", &PlanOperation::Create), - ("schema.knowledge", &PlanOperation::Update), - ] - ); - } - - #[tokio::test] - async fn old_minimal_state_json_still_plans_with_default_revision() { - let dir = fixture(); - let state_dir = dir.path().join(CLUSTER_STATE_DIR); - fs::create_dir_all(&state_dir).unwrap(); - fs::write( - state_dir.join("state.json"), - r#"{ - "version": 1, - "applied_revision": { - "config_digest": "old", - "resources": { - "graph.knowledge": { "digest": "old-graph" } - } - } -}"#, - ) - .unwrap(); - - let out = plan_config_dir(dir.path()).await; - assert!(out.ok, "{:?}", out.diagnostics); - assert_eq!(out.state_observations.state_revision, 0); - assert!(out.state_observations.state_cas.is_some()); - assert!(out.changes.iter().any(|change| { - change.resource == "graph.knowledge" && change.operation == PlanOperation::Update - })); - } - - #[test] - fn extended_state_json_status_surfaces_statuses() { - let dir = fixture(); - let state_dir = dir.path().join(CLUSTER_STATE_DIR); - fs::create_dir_all(&state_dir).unwrap(); - let state = r#"{ - "version": 1, - "state_revision": 42, - "applied_revision": { - "config_digest": "applied-config", - "resources": { - "graph.knowledge": { "digest": "graph-digest" } - } - }, - "resource_statuses": { - "graph.knowledge": { - "status": "applied", - "conditions": ["healthy"], - "message": "ready" - } - }, - "approval_records": {}, - "recovery_records": {}, - "observations": { - "graph.knowledge": { "manifest_version": 12 } - } -}"#; - fs::write(state_dir.join("state.json"), state).unwrap(); - - let out = status_config_dir(dir.path()); - assert!(out.ok, "{:?}", out.diagnostics); - assert!(out.state_observations.state_found); - assert_eq!(out.state_observations.state_revision, 42); - assert_eq!( - out.state_observations.state_cas.as_deref(), - Some(format!("sha256:{}", sha256_hex(state.as_bytes())).as_str()) - ); - assert_eq!( - out.resource_digests - .get("graph.knowledge") - .map(String::as_str), - Some("graph-digest") - ); - assert_eq!( - out.resource_statuses["graph.knowledge"].status, - ResourceLifecycleStatus::Applied - ); - } - - #[test] - fn missing_state_status_succeeds_with_warning() { - let dir = fixture(); - let out = status_config_dir(dir.path()); - assert!(out.ok, "{:?}", out.diagnostics); - assert!(!out.state_observations.state_found); - assert_eq!(out.state_observations.state_revision, 0); - assert!( - out.diagnostics - .iter() - .any(|diagnostic| diagnostic.code == "state_missing") - ); - } - - #[test] - fn invalid_state_status_fails() { - let dir = fixture(); - let state_dir = dir.path().join(CLUSTER_STATE_DIR); - fs::create_dir_all(&state_dir).unwrap(); - fs::write(state_dir.join("state.json"), "{").unwrap(); - - let out = status_config_dir(dir.path()); - assert!(!out.ok); - assert!(out.state_observations.state_found); - assert!( - out.diagnostics - .iter() - .any(|diagnostic| diagnostic.code == "invalid_state_json") - ); - } - - #[test] - fn status_surfaces_full_lock_metadata() { - let dir = fixture(); - write_lock_file(dir.path(), "held-lock", "refresh"); - - let out = status_config_dir(dir.path()); - assert!(out.ok, "{:?}", out.diagnostics); - assert!(out.state_observations.locked); - assert_eq!(out.state_observations.lock_id.as_deref(), Some("held-lock")); - assert_eq!( - out.state_observations.lock_operation.as_deref(), - Some("refresh") - ); - assert_eq!( - out.state_observations.lock_created_at.as_deref(), - Some("1970-01-01T00:00:00Z") - ); - assert_eq!(out.state_observations.lock_pid, Some(123)); - assert!(out.state_observations.lock_age_seconds.is_some()); - } - - #[test] - fn force_unlock_matching_id_removes_lock() { - let dir = fixture(); - write_lock_file(dir.path(), "held-lock", "plan"); - - let out = force_unlock_config_dir(dir.path(), "held-lock"); - assert!(out.ok, "{:?}", out.diagnostics); - assert!(out.lock_removed); - assert_eq!(out.state_observations.lock_id.as_deref(), Some("held-lock")); - assert_eq!( - out.state_observations.lock_operation.as_deref(), - Some("plan") - ); - assert!(!dir.path().join(CLUSTER_LOCK_FILE).exists()); - } - - #[test] - fn force_unlock_wrong_id_fails_and_preserves_lock() { - let dir = fixture(); - write_lock_file(dir.path(), "held-lock", "plan"); - - let out = force_unlock_config_dir(dir.path(), "other-lock"); - assert!(!out.ok); - assert!(!out.lock_removed); - assert_eq!(out.state_observations.lock_id.as_deref(), Some("held-lock")); - assert!( - out.diagnostics - .iter() - .any(|diagnostic| diagnostic.code == "state_lock_id_mismatch") - ); - assert!(dir.path().join(CLUSTER_LOCK_FILE).exists()); - } - - #[test] - fn force_unlock_missing_lock_fails() { - let dir = fixture(); - - let out = force_unlock_config_dir(dir.path(), "held-lock"); - assert!(!out.ok); - assert!(!out.lock_removed); - assert!(!out.state_observations.locked); - assert!( - out.diagnostics - .iter() - .any(|diagnostic| diagnostic.code == "state_lock_missing") - ); - } - - #[test] - fn force_unlock_invalid_lock_json_fails_and_preserves_lock() { - let dir = fixture(); - let state_dir = dir.path().join(CLUSTER_STATE_DIR); - fs::create_dir_all(&state_dir).unwrap(); - fs::write(state_dir.join("lock.json"), "{").unwrap(); - - let out = force_unlock_config_dir(dir.path(), "held-lock"); - assert!(!out.ok); - assert!(!out.lock_removed); - assert!( - out.diagnostics - .iter() - .any(|diagnostic| diagnostic.code == "invalid_state_lock") - ); - assert!(dir.path().join(CLUSTER_LOCK_FILE).exists()); - } - - #[test] - fn force_unlock_unsupported_lock_version_fails_and_preserves_lock() { - let dir = fixture(); - let state_dir = dir.path().join(CLUSTER_STATE_DIR); - fs::create_dir_all(&state_dir).unwrap(); - fs::write( - state_dir.join("lock.json"), - r#"{"version":2,"lock_id":"held-lock","operation":"plan","created_at":"1970-01-01T00:00:00Z","pid":123}"#, - ) - .unwrap(); - - let out = force_unlock_config_dir(dir.path(), "held-lock"); - assert!(!out.ok); - assert!(!out.lock_removed); - assert!( - out.diagnostics - .iter() - .any(|diagnostic| diagnostic.code == "unsupported_state_lock_version") - ); - assert!(dir.path().join(CLUSTER_LOCK_FILE).exists()); - } - - #[test] - fn force_unlock_external_state_backend_rejected() { - let dir = fixture(); - write_lock_file(dir.path(), "held-lock", "plan"); - fs::write( - dir.path().join(CLUSTER_CONFIG_FILE), - r#" -version: 1 -state: - backend: s3://state-bucket/cluster -graphs: - knowledge: - schema: ./people.pg -"#, - ) - .unwrap(); - - let out = force_unlock_config_dir(dir.path(), "held-lock"); - assert!(!out.ok); - assert!(!out.lock_removed); - assert!( - out.diagnostics - .iter() - .any(|diagnostic| diagnostic.code == "unsupported_state_backend") - ); - assert!(dir.path().join(CLUSTER_LOCK_FILE).exists()); - } - - #[tokio::test] - async fn plan_succeeds_after_force_unlock() { - let dir = fixture(); - write_lock_file(dir.path(), "held-lock", "plan"); - - let locked = plan_config_dir(dir.path()).await; - assert!(!locked.ok); - assert!( - locked - .diagnostics - .iter() - .any(|diagnostic| diagnostic.code == "state_lock_held") - ); - - let unlocked = force_unlock_config_dir(dir.path(), "held-lock"); - assert!(unlocked.ok, "{:?}", unlocked.diagnostics); - - let out = plan_config_dir(dir.path()).await; - assert!(out.ok, "{:?}", out.diagnostics); - } - - #[tokio::test] - async fn plan_reports_state_cas_revision_and_removes_lock() { - let dir = fixture(); - let state_dir = dir.path().join(CLUSTER_STATE_DIR); - fs::create_dir_all(&state_dir).unwrap(); - let state = r#"{ - "version": 1, - "state_revision": 7, - "applied_revision": { - "config_digest": "old", - "resources": { - "graph.knowledge": { "digest": "old-graph" } - } - } -}"#; - fs::write(state_dir.join("state.json"), state).unwrap(); - - let out = plan_config_dir(dir.path()).await; - assert!(out.ok, "{:?}", out.diagnostics); - assert_eq!(out.state_observations.state_revision, 7); - assert_eq!( - out.state_observations.state_cas.as_deref(), - Some(format!("sha256:{}", sha256_hex(state.as_bytes())).as_str()) - ); - assert!(!out.state_observations.locked); - assert!(out.state_observations.lock_id.is_none()); - assert!(out.state_observations.lock_acquired); - assert!(out.state_observations.acquired_lock_id.is_some()); - assert!( - !dir.path().join(CLUSTER_LOCK_FILE).exists(), - "plan must release lock before returning" - ); - } - - #[tokio::test] - async fn existing_lock_makes_plan_fail() { - let dir = fixture(); - let state_dir = dir.path().join(CLUSTER_STATE_DIR); - fs::create_dir_all(&state_dir).unwrap(); - fs::write( - state_dir.join("lock.json"), - r#"{ - "version": 1, - "lock_id": "held-lock", - "operation": "plan", - "created_at": "2026-06-08T00:00:00Z", - "pid": 123 -}"#, - ) - .unwrap(); - - let out = plan_config_dir(dir.path()).await; - assert!(!out.ok); - assert!(out.state_observations.locked); - assert_eq!(out.state_observations.lock_id.as_deref(), Some("held-lock")); - assert!(!out.state_observations.lock_acquired); - assert!(out.state_observations.acquired_lock_id.is_none()); - assert_eq!( - out.state_observations.lock_operation.as_deref(), - Some("plan") - ); - assert!( - out.diagnostics - .iter() - .any(|diagnostic| diagnostic.code == "state_lock_held") - ); - assert!(out.diagnostics.iter().any(|diagnostic| { - diagnostic.code == "state_lock_held" - && diagnostic.message.contains("force-unlock held-lock") - })); - } - - #[tokio::test] - async fn state_lock_false_bypasses_lock_with_warning() { - let dir = fixture(); - fs::write( - dir.path().join(CLUSTER_CONFIG_FILE), - r#" -version: 1 -state: - backend: cluster - lock: false -graphs: - knowledge: - schema: ./people.pg -"#, - ) - .unwrap(); - - let out = plan_config_dir(dir.path()).await; - assert!(out.ok, "{:?}", out.diagnostics); - assert!(!out.state_observations.locked); - assert!(!out.state_observations.lock_acquired); - assert!( - out.diagnostics - .iter() - .any(|diagnostic| diagnostic.code == "state_lock_disabled") - ); - assert!(!dir.path().join(CLUSTER_LOCK_FILE).exists()); - } - - #[test] - fn external_state_backend_rejected() { - let dir = fixture(); - fs::write( - dir.path().join(CLUSTER_CONFIG_FILE), - "version: 1\nstate:\n backend: s3://bucket/state\ngraphs: {}\n", - ) - .unwrap(); - let out = validate_config_dir(dir.path()); - assert!(!out.ok); - assert_eq!(out.diagnostics[0].code, "unsupported_state_backend"); - } - - #[tokio::test] - async fn external_state_backend_plan_rejected() { - let dir = fixture(); - fs::write( - dir.path().join(CLUSTER_CONFIG_FILE), - "version: 1\nstate:\n backend: s3://bucket/state\ngraphs: {}\n", - ) - .unwrap(); - let out = plan_config_dir(dir.path()).await; - assert!(!out.ok); - assert!( - out.diagnostics - .iter() - .any(|diagnostic| diagnostic.code == "unsupported_state_backend") - ); - } - - #[tokio::test] - async fn import_missing_state_creates_state_with_graph_observation() { - let dir = fixture(); - init_derived_graph(dir.path()).await; - - let out = import_config_dir(dir.path()).await; - assert!(out.ok, "{:?}", out.diagnostics); - assert_eq!(out.state_observations.state_revision, 1); - assert!(out.state_observations.state_cas.is_some()); - assert!(!out.state_observations.locked); - assert!(out.state_observations.lock_acquired); - assert!(out.state_observations.acquired_lock_id.is_some()); - assert!(!dir.path().join(CLUSTER_LOCK_FILE).exists()); - assert_eq!( - out.resource_digests - .get("schema.knowledge") - .map(String::as_str), - Some(sha256_hex(SCHEMA.as_bytes()).as_str()) - ); - assert!(out.observations["graph.knowledge"]["manifest_version"].is_number()); - assert_eq!( - out.observations["graph.knowledge"]["schema_matches_desired"], - true - ); - - let state: serde_json::Value = - serde_json::from_str(&fs::read_to_string(dir.path().join(CLUSTER_STATE_FILE)).unwrap()) - .unwrap(); - assert_eq!(state["state_revision"], 1); - assert_eq!( - state["resource_statuses"]["graph.knowledge"]["status"], - "applied" - ); - } - - #[tokio::test] - async fn import_existing_state_fails() { - let dir = fixture(); - let state_dir = dir.path().join(CLUSTER_STATE_DIR); - fs::create_dir_all(&state_dir).unwrap(); - fs::write( - state_dir.join("state.json"), - r#"{"version":1,"applied_revision":{"resources":{}}}"#, - ) - .unwrap(); - - let out = import_config_dir(dir.path()).await; - assert!(!out.ok); - assert!( - out.diagnostics - .iter() - .any(|diagnostic| diagnostic.code == "state_already_exists") - ); - } - - #[tokio::test] - async fn refresh_missing_state_fails() { - let dir = fixture(); - let out = refresh_config_dir(dir.path()).await; - assert!(!out.ok); - assert!( - out.diagnostics - .iter() - .any(|diagnostic| diagnostic.code == "state_missing") - ); - } - - #[tokio::test] - async fn refresh_existing_minimal_state_increments_revision_and_updates_cas() { - let dir = fixture(); - init_derived_graph(dir.path()).await; - let state_dir = dir.path().join(CLUSTER_STATE_DIR); - fs::create_dir_all(&state_dir).unwrap(); - fs::write( - state_dir.join("state.json"), - r#"{"version":1,"applied_revision":{"config_digest":"old","resources":{"graph.knowledge":{"digest":"old"}}}}"#, - ) - .unwrap(); - - let out = refresh_config_dir(dir.path()).await; - assert!(out.ok, "{:?}", out.diagnostics); - assert_eq!(out.state_observations.state_revision, 1); - assert!(out.state_observations.state_cas.is_some()); - assert!(!out.state_observations.locked); - assert!(out.state_observations.lock_acquired); - assert_eq!( - out.resource_statuses["graph.knowledge"].status, - ResourceLifecycleStatus::Applied - ); - assert!(!dir.path().join(CLUSTER_LOCK_FILE).exists()); - } - - #[tokio::test] - async fn refresh_records_live_schema_digest_and_manifest_version() { - let dir = fixture(); - init_derived_graph(dir.path()).await; - let state_dir = dir.path().join(CLUSTER_STATE_DIR); - fs::create_dir_all(&state_dir).unwrap(); - fs::write( - state_dir.join("state.json"), - r#"{"version":1,"state_revision":4,"applied_revision":{"resources":{}}}"#, - ) - .unwrap(); - - let out = refresh_config_dir(dir.path()).await; - assert!(out.ok, "{:?}", out.diagnostics); - assert_eq!(out.state_observations.state_revision, 5); - assert_eq!( - out.observations["graph.knowledge"]["schema_digest"], - sha256_hex(SCHEMA.as_bytes()) - ); - assert!(out.observations["graph.knowledge"]["manifest_version"].is_u64()); - } - - #[tokio::test] - async fn missing_derived_graph_root_marks_drifted_and_plans_creates() { - let dir = fixture(); - let state_dir = dir.path().join(CLUSTER_STATE_DIR); - fs::create_dir_all(&state_dir).unwrap(); - fs::write( - state_dir.join("state.json"), - r#"{"version":1,"applied_revision":{"resources":{"graph.knowledge":{"digest":"old-graph"},"schema.knowledge":{"digest":"old-schema"}}}}"#, - ) - .unwrap(); - - let out = refresh_config_dir(dir.path()).await; - assert!(out.ok, "{:?}", out.diagnostics); - assert_eq!( - out.resource_statuses["graph.knowledge"].status, - ResourceLifecycleStatus::Drifted - ); - assert!(!out.resource_digests.contains_key("graph.knowledge")); - assert_eq!(out.observations["graph.knowledge"]["exists"], false); - - let plan = plan_config_dir(dir.path()).await; - assert!(plan.ok, "{:?}", plan.diagnostics); - assert!(plan.changes.iter().any(|change| { - change.resource == "graph.knowledge" && change.operation == PlanOperation::Create - })); - assert!(plan.changes.iter().any(|change| { - change.resource == "schema.knowledge" && change.operation == PlanOperation::Create - })); - } - - #[tokio::test] - async fn live_schema_mismatch_marks_drifted_and_causes_plan_update() { - let dir = fixture(); - init_derived_graph(dir.path()).await; - fs::write( - dir.path().join("people.pg"), - SCHEMA.replace("age: I32?", "age: I32?\n nickname: String?"), - ) - .unwrap(); - let state_dir = dir.path().join(CLUSTER_STATE_DIR); - fs::create_dir_all(&state_dir).unwrap(); - fs::write( - state_dir.join("state.json"), - r#"{"version":1,"applied_revision":{"resources":{"graph.knowledge":{"digest":"old-graph"},"schema.knowledge":{"digest":"old-schema"}}}}"#, - ) - .unwrap(); - - let out = refresh_config_dir(dir.path()).await; - assert!(out.ok, "{:?}", out.diagnostics); - assert_eq!( - out.resource_statuses["schema.knowledge"].status, - ResourceLifecycleStatus::Drifted - ); - assert_eq!( - out.observations["graph.knowledge"]["schema_matches_desired"], - false - ); - - let plan = plan_config_dir(dir.path()).await; - assert!(plan.ok, "{:?}", plan.diagnostics); - assert!(plan.changes.iter().any(|change| { - change.resource == "schema.knowledge" && change.operation == PlanOperation::Update - })); - } - - #[tokio::test] - async fn existing_lock_makes_refresh_fail() { - let dir = fixture(); - let state_dir = dir.path().join(CLUSTER_STATE_DIR); - fs::create_dir_all(&state_dir).unwrap(); - fs::write( - state_dir.join("state.json"), - r#"{"version":1,"applied_revision":{"resources":{}}}"#, - ) - .unwrap(); - fs::write( - state_dir.join("lock.json"), - r#"{"version":1,"lock_id":"held-lock","operation":"refresh","created_at":"2026-06-08T00:00:00Z","pid":123}"#, - ) - .unwrap(); - - let out = refresh_config_dir(dir.path()).await; - assert!(!out.ok); - assert!(out.state_observations.locked); - assert_eq!(out.state_observations.lock_id.as_deref(), Some("held-lock")); - assert!(!out.state_observations.lock_acquired); - assert_eq!( - out.state_observations.lock_operation.as_deref(), - Some("refresh") - ); - assert!( - out.diagnostics - .iter() - .any(|diagnostic| diagnostic.code == "state_lock_held") - ); - assert!(out.diagnostics.iter().any(|diagnostic| { - diagnostic.code == "state_lock_held" - && diagnostic.message.contains("force-unlock held-lock") - })); - } - - #[tokio::test] - async fn state_lock_false_bypasses_refresh_lock_with_warning() { - let dir = fixture(); - init_derived_graph(dir.path()).await; - fs::write( - dir.path().join(CLUSTER_CONFIG_FILE), - r#" -version: 1 -state: - backend: cluster - lock: false -graphs: - knowledge: - schema: ./people.pg -"#, - ) - .unwrap(); - let state_dir = dir.path().join(CLUSTER_STATE_DIR); - fs::create_dir_all(&state_dir).unwrap(); - fs::write( - state_dir.join("state.json"), - r#"{"version":1,"applied_revision":{"resources":{}}}"#, - ) - .unwrap(); - - let out = refresh_config_dir(dir.path()).await; - assert!(out.ok, "{:?}", out.diagnostics); - assert!(!out.state_observations.locked); - assert!(!out.state_observations.lock_acquired); - assert!( - out.diagnostics - .iter() - .any(|diagnostic| diagnostic.code == "state_lock_disabled") - ); - } - - #[tokio::test] - async fn external_state_backend_refresh_rejected() { - let dir = fixture(); - fs::write( - dir.path().join(CLUSTER_CONFIG_FILE), - "version: 1\nstate:\n backend: s3://bucket/state\ngraphs: {}\n", - ) - .unwrap(); - - let out = refresh_config_dir(dir.path()).await; - assert!(!out.ok); - assert!( - out.diagnostics - .iter() - .any(|diagnostic| diagnostic.code == "unsupported_state_backend") - ); - } - - #[tokio::test] - async fn import_graph_open_error_does_not_create_state() { - let dir = fixture(); - fs::create_dir_all(dir.path().join(CLUSTER_GRAPHS_DIR).join("knowledge.omni")).unwrap(); - - let out = import_config_dir(dir.path()).await; - assert!(!out.ok); - assert!( - out.diagnostics - .iter() - .any(|diagnostic| diagnostic.code == "graph_observation_error") - ); - assert!(!dir.path().join(CLUSTER_STATE_FILE).exists()); - } - - // ---- config-only apply (Stage 3A) ---- - - /// Seed a state.json that simulates "graph exists with the desired schema, - /// queries/policies not yet applied" by borrowing the desired digests. - fn write_applyable_state(config_dir: &Path) { - let out = validate_config_dir(config_dir); - assert!(out.ok, "{:?}", out.diagnostics); - let schema_digest = out.resource_digests.get("schema.knowledge").unwrap().clone(); - let graph_composite = - graph_digest("knowledge", Some(&schema_digest), Some(&BTreeMap::new())); - write_state_resources( - config_dir, - &[ - ("graph.knowledge", graph_composite.as_str()), - ("schema.knowledge", schema_digest.as_str()), - ], - ); - } - - fn write_state_resources(config_dir: &Path, resources: &[(&str, &str)]) { - let resource_map: serde_json::Map = resources - .iter() - .map(|(address, digest)| ((*address).to_string(), json!({ "digest": digest }))) - .collect(); - let state_dir = config_dir.join(CLUSTER_STATE_DIR); - fs::create_dir_all(&state_dir).unwrap(); - fs::write( - state_dir.join("state.json"), - serde_json::to_string_pretty(&json!({ - "version": 1, - "state_revision": 1, - "applied_revision": { "resources": resource_map } - })) - .unwrap(), - ) - .unwrap(); - } - - fn read_state_json(config_dir: &Path) -> serde_json::Value { - serde_json::from_str(&fs::read_to_string(config_dir.join(CLUSTER_STATE_FILE)).unwrap()) - .unwrap() - } - - fn query_payload_path(config_dir: &Path, digest: &str) -> std::path::PathBuf { - config_dir - .join(CLUSTER_RESOURCES_DIR) - .join("query/knowledge/find_person") - .join(format!("{digest}.gq")) - } - - fn policy_payload_path(config_dir: &Path, digest: &str) -> std::path::PathBuf { - config_dir - .join(CLUSTER_RESOURCES_DIR) - .join("policy/base") - .join(format!("{digest}.yaml")) - } - - #[tokio::test] - async fn apply_without_state_fails_with_state_missing() { - let dir = fixture(); - let out = apply_config_dir(dir.path()).await; - assert!(!out.ok); - assert!( - out.diagnostics - .iter() - .any(|diagnostic| diagnostic.code == "state_missing" - && diagnostic.message.contains("cluster import")) - ); - assert!(!dir.path().join(CLUSTER_STATE_FILE).exists()); - assert!(!dir.path().join(CLUSTER_RESOURCES_DIR).exists()); - assert!(!dir.path().join(CLUSTER_LOCK_FILE).exists()); - } - - #[tokio::test] - async fn apply_writes_payloads_state_and_statuses() { - let dir = fixture(); - write_applyable_state(dir.path()); - let desired = validate_config_dir(dir.path()); - let query_digest = desired - .resource_digests - .get("query.knowledge.find_person") - .unwrap() - .clone(); - let policy_digest = desired.resource_digests.get("policy.base").unwrap().clone(); - let schema_digest = desired - .resource_digests - .get("schema.knowledge") - .unwrap() - .clone(); - - let out = apply_config_dir(dir.path()).await; - assert!(out.ok, "{:?}", out.diagnostics); - assert_eq!(out.applied_count, 2); - assert_eq!(out.deferred_count, 0); - assert!(out.converged); - assert!(out.state_written); - - let query_blob = query_payload_path(dir.path(), &query_digest); - assert_eq!(fs::read_to_string(&query_blob).unwrap(), QUERY); - let policy_blob = policy_payload_path(dir.path(), &policy_digest); - assert_eq!(fs::read_to_string(&policy_blob).unwrap(), "rules: []\n"); - - let state = read_state_json(dir.path()); - assert_eq!(state["state_revision"], 2); - let resources = &state["applied_revision"]["resources"]; - assert_eq!( - resources["query.knowledge.find_person"]["digest"], - query_digest - ); - assert_eq!(resources["policy.base"]["digest"], policy_digest); - let expected_composite = graph_digest( - "knowledge", - Some(&schema_digest), - Some( - &[("find_person".to_string(), query_digest.clone())] - .into_iter() - .collect(), - ), - ); - assert_eq!(resources["graph.knowledge"]["digest"], expected_composite); - assert_eq!( - state["applied_revision"]["config_digest"], - desired_revision_digest(&out) - ); - assert_eq!( - state["resource_statuses"]["query.knowledge.find_person"]["status"], - "applied" - ); - assert_eq!(state["resource_statuses"]["policy.base"]["status"], "applied"); - assert!(!dir.path().join(CLUSTER_LOCK_FILE).exists()); - } - - fn desired_revision_digest(out: &ApplyOutput) -> String { - out.desired_revision.config_digest.clone().unwrap() - } - - #[tokio::test] - async fn apply_update_changes_query_digest_and_keeps_old_blob() { - let dir = fixture(); - let desired = validate_config_dir(dir.path()); - let schema_digest = desired - .resource_digests - .get("schema.knowledge") - .unwrap() - .clone(); - let old_digest = "0".repeat(64); - let graph_composite = - graph_digest("knowledge", Some(&schema_digest), Some(&BTreeMap::new())); - write_state_resources( - dir.path(), - &[ - ("graph.knowledge", graph_composite.as_str()), - ("schema.knowledge", schema_digest.as_str()), - ("query.knowledge.find_person", old_digest.as_str()), - ], - ); - let old_blob = query_payload_path(dir.path(), &old_digest); - fs::create_dir_all(old_blob.parent().unwrap()).unwrap(); - fs::write(&old_blob, "old query source").unwrap(); - - let out = apply_config_dir(dir.path()).await; - assert!(out.ok, "{:?}", out.diagnostics); - let new_digest = desired - .resource_digests - .get("query.knowledge.find_person") - .unwrap(); - let state = read_state_json(dir.path()); - assert_eq!( - state["applied_revision"]["resources"]["query.knowledge.find_person"]["digest"], - *new_digest - ); - assert_eq!(fs::read_to_string(&old_blob).unwrap(), "old query source"); - assert!(query_payload_path(dir.path(), new_digest).exists()); - } - - #[tokio::test] - async fn apply_deletes_removed_resources_but_keeps_blobs() { - let dir = fixture(); - let desired = validate_config_dir(dir.path()); - let schema_digest = desired - .resource_digests - .get("schema.knowledge") - .unwrap() - .clone(); - let stale_query_digest = "1".repeat(64); - let stale_policy_digest = "2".repeat(64); - let graph_composite = - graph_digest("knowledge", Some(&schema_digest), Some(&BTreeMap::new())); - write_state_resources( - dir.path(), - &[ - ("graph.knowledge", graph_composite.as_str()), - ("schema.knowledge", schema_digest.as_str()), - ("query.knowledge.orphan", stale_query_digest.as_str()), - ("policy.old", stale_policy_digest.as_str()), - ], - ); - let stale_blob = dir - .path() - .join(CLUSTER_RESOURCES_DIR) - .join("policy/old") - .join(format!("{stale_policy_digest}.yaml")); - fs::create_dir_all(stale_blob.parent().unwrap()).unwrap(); - fs::write(&stale_blob, "old policy").unwrap(); - - let out = apply_config_dir(dir.path()).await; - assert!(out.ok, "{:?}", out.diagnostics); - assert!(out.converged); - let state = read_state_json(dir.path()); - let resources = &state["applied_revision"]["resources"]; - assert!(resources.get("query.knowledge.orphan").is_none()); - assert!(resources.get("policy.old").is_none()); - assert!( - state["resource_statuses"] - .get("query.knowledge.orphan") - .is_none() - ); - // Deleted resources leave their content-addressed blobs in place; GC is - // a later stage. - assert_eq!(fs::read_to_string(&stale_blob).unwrap(), "old policy"); - // The composite no longer includes the orphan query. - let query_digest = desired - .resource_digests - .get("query.knowledge.find_person") - .unwrap() - .clone(); - let expected_composite = graph_digest( - "knowledge", - Some(&schema_digest), - Some(&[("find_person".to_string(), query_digest)].into_iter().collect()), - ); - assert_eq!(resources["graph.knowledge"]["digest"], expected_composite); - } - - #[tokio::test] - async fn apply_schema_update_and_dependent_query_in_one_run() { - let dir = fixture(); - init_derived_graph(dir.path()).await; - write_applyable_state(dir.path()); - // Schema update + a query update that depends on the new field: one - // apply executes the schema migration first, then the catalog write. - fs::write(dir.path().join("people.pg"), SCHEMA_V2).unwrap(); - fs::write( - dir.path().join("people.gq"), - "\nquery find_person($name: String) {\n match { $p: Person { name: $name } }\n return { $p.name, $p.bio }\n}\n", - ) - .unwrap(); - - let out = apply_config_dir(dir.path()).await; - assert!(out.ok, "{:?}", out.diagnostics); - assert!(out.converged, "{out:?}"); - let by_resource: BTreeMap<&str, &PlanChange> = out - .changes - .iter() - .map(|change| (change.resource.as_str(), change)) - .collect(); - assert_eq!( - by_resource["schema.knowledge"].disposition, - Some(ApplyDisposition::Applied) - ); - assert_eq!( - by_resource["query.knowledge.find_person"].disposition, - Some(ApplyDisposition::Applied) - ); - assert_eq!( - by_resource["graph.knowledge"].disposition, - Some(ApplyDisposition::Derived) - ); - // The live graph carries the new schema. - let db = Omnigraph::open_read_only(&derived_graph_uri(dir.path(), "knowledge")) - .await - .unwrap(); - let desired = validate_config_dir(dir.path()); - assert_eq!( - sha256_hex(db.schema_source().as_bytes()), - desired.resource_digests["schema.knowledge"] - ); - let state = read_state_json(dir.path()); - assert_eq!( - state["applied_revision"]["resources"]["schema.knowledge"]["digest"], - desired.resource_digests["schema.knowledge"] - ); - // Sidecar retired after the CAS landed. - assert!( - !dir.path().join(CLUSTER_RECOVERIES_DIR).exists() - || fs::read_dir(dir.path().join(CLUSTER_RECOVERIES_DIR)) - .unwrap() - .next() - .is_none() - ); - } - - #[tokio::test] - async fn apply_unsupported_schema_change_fails_loudly() { - let dir = fixture(); - init_derived_graph(dir.path()).await; - write_applyable_state(dir.path()); - // Property type changes are unsupported by the engine planner. - fs::write( - dir.path().join("people.pg"), - "\nnode Person {\n name: String @key\n age: I64?\n}\n", - ) - .unwrap(); - - let out = apply_config_dir(dir.path()).await; - assert!(!out.ok); - assert!(out.diagnostics.iter().any(|diagnostic| { - diagnostic.code == "schema_apply_failed" - && diagnostic.message.contains("changing property type") - })); - let by_resource: BTreeMap<&str, &PlanChange> = out - .changes - .iter() - .map(|change| (change.resource.as_str(), change)) - .collect(); - assert_eq!( - by_resource["schema.knowledge"].disposition, - Some(ApplyDisposition::Blocked) - ); - assert_eq!( - by_resource["schema.knowledge"].reason.as_deref(), - Some("schema_apply_failed") - ); - // The live schema and the ledger are unchanged. - let state = read_state_json(dir.path()); - let desired = validate_config_dir(dir.path()); - assert_ne!( - state["applied_revision"]["resources"]["schema.knowledge"]["digest"], - desired.resource_digests["schema.knowledge"] - ); - // Second run: the sweep retires the stale sidecar (ledger consistent) - // and the run fails just as loudly — idempotent loudness. - let second = apply_config_dir(dir.path()).await; - assert!(!second.ok); - assert!( - second - .diagnostics - .iter() - .any(|diagnostic| diagnostic.code == "schema_apply_failed") - ); - } - - #[tokio::test] - async fn apply_blocks_schema_update_while_recovery_pending() { - let dir = fixture(); - init_derived_graph(dir.path()).await; - write_state_resources(dir.path(), &[("schema.knowledge", "stale-digest")]); - fs::write(dir.path().join("people.pg"), SCHEMA_V2).unwrap(); - // A pending sidecar whose intent matches neither live nor recorded. - write_schema_apply_sidecar(dir.path(), "knowledge", "intended-digest", "01PENDS"); - - let out = apply_config_dir(dir.path()).await; - let by_resource: BTreeMap<&str, &PlanChange> = out - .changes - .iter() - .map(|change| (change.resource.as_str(), change)) - .collect(); - assert_eq!( - by_resource["schema.knowledge"].disposition, - Some(ApplyDisposition::Blocked) - ); - assert_eq!( - by_resource["schema.knowledge"].reason.as_deref(), - Some("cluster_recovery_pending") - ); - } - - #[tokio::test] - async fn apply_creates_graph_and_unblocks_dependents() { - let dir = fixture(); - write_state_resources(dir.path(), &[]); - - let out = apply_config_dir(dir.path()).await; - assert!(out.ok, "{:?}", out.diagnostics); - assert!(out.converged, "{out:?}"); - let by_resource: BTreeMap<&str, &PlanChange> = out - .changes - .iter() - .map(|change| (change.resource.as_str(), change)) - .collect(); - // Stage 4A: the create executes, and its dependents apply in-run. - assert_eq!( - by_resource["graph.knowledge"].disposition, - Some(ApplyDisposition::Applied) - ); - assert_eq!( - by_resource["schema.knowledge"].disposition, - Some(ApplyDisposition::Applied) - ); - assert_eq!( - by_resource["query.knowledge.find_person"].disposition, - Some(ApplyDisposition::Applied) - ); - assert_eq!( - by_resource["policy.base"].disposition, - Some(ApplyDisposition::Applied) - ); - // The graph exists on disk and opens; state records everything. - let graph_uri = derived_graph_uri(dir.path(), "knowledge"); - let db = Omnigraph::open_read_only(&graph_uri).await.unwrap(); - let desired = validate_config_dir(dir.path()); - assert_eq!( - sha256_hex(db.schema_source().as_bytes()), - desired.resource_digests["schema.knowledge"] - ); - let state = read_state_json(dir.path()); - assert_eq!( - state["applied_revision"]["resources"]["schema.knowledge"]["digest"], - desired.resource_digests["schema.knowledge"] - ); - assert_eq!( - state["resource_statuses"]["graph.knowledge"]["status"], - "applied" - ); - // The create's sidecar was retired after the state CAS landed. - assert!( - !dir.path().join(CLUSTER_RECOVERIES_DIR).exists() - || fs::read_dir(dir.path().join(CLUSTER_RECOVERIES_DIR)) - .unwrap() - .next() - .is_none() - ); - } - - #[tokio::test] - async fn apply_create_failure_blocks_dependents_and_keeps_sidecar() { - let dir = fixture(); - write_state_resources(dir.path(), &[]); - // Make the init fail its strict preflight: a junk _schema.pg already - // sits at the derived root (the engine refuses to overwrite it). - let root = dir.path().join(CLUSTER_GRAPHS_DIR).join("knowledge.omni"); - fs::create_dir_all(&root).unwrap(); - fs::write(root.join("_schema.pg"), "junk").unwrap(); - - let out = apply_config_dir(dir.path()).await; - assert!(!out.ok); - assert!( - out.diagnostics - .iter() - .any(|diagnostic| diagnostic.code == "graph_create_failed") - ); - let by_resource: BTreeMap<&str, &PlanChange> = out - .changes - .iter() - .map(|change| (change.resource.as_str(), change)) - .collect(); - // Dependents are demoted: the run tells the truth about what executed. - assert_eq!( - by_resource["graph.knowledge"].disposition, - Some(ApplyDisposition::Blocked) - ); - assert_eq!( - by_resource["query.knowledge.find_person"].disposition, - Some(ApplyDisposition::Blocked) - ); - assert_eq!( - by_resource["query.knowledge.find_person"].reason.as_deref(), - Some("dependency_not_applied") - ); - assert_eq!( - by_resource["policy.base"].disposition, - Some(ApplyDisposition::Blocked) - ); - assert!(!out.converged); - // The sidecar stays for the sweep to classify next run. - assert!( - fs::read_dir(dir.path().join(CLUSTER_RECOVERIES_DIR)) - .unwrap() - .next() - .is_some() - ); - // No graph digests moved. - let state = read_state_json(dir.path()); - assert!( - state["applied_revision"]["resources"] - .as_object() - .unwrap() - .is_empty() - ); - } - - #[tokio::test] - async fn apply_blocks_graph_delete_without_approval() { - let dir = fixture(); - let desired = validate_config_dir(dir.path()); - let schema_digest = desired - .resource_digests - .get("schema.knowledge") - .unwrap() - .clone(); - let graph_composite = - graph_digest("knowledge", Some(&schema_digest), Some(&BTreeMap::new())); - write_state_resources( - dir.path(), - &[ - ("graph.knowledge", graph_composite.as_str()), - ("schema.knowledge", schema_digest.as_str()), - ("graph.old", "3333"), - ("schema.old", "4444"), - ("query.old.q", "5555"), - ], - ); - - let out = apply_config_dir(dir.path()).await; - assert!(out.ok, "{:?}", out.diagnostics); - assert!(!out.converged); - let by_resource: BTreeMap<&str, &PlanChange> = out - .changes - .iter() - .map(|change| (change.resource.as_str(), change)) - .collect(); - // Stage 4C: deletes are gated, not deferred — every subtree change - // blocks on the single graph-level approval. - assert_eq!( - by_resource["graph.old"].disposition, - Some(ApplyDisposition::Blocked) - ); - assert_eq!( - by_resource["graph.old"].reason.as_deref(), - Some("approval_required") - ); - assert_eq!( - by_resource["schema.old"].reason.as_deref(), - Some("approval_required") - ); - assert_eq!( - by_resource["query.old.q"].reason.as_deref(), - Some("approval_required") - ); - // State intact; nothing destroyed without the artifact. - let state = read_state_json(dir.path()); - let resources = &state["applied_revision"]["resources"]; - assert_eq!(resources["graph.old"]["digest"], "3333"); - assert_eq!(resources["schema.old"]["digest"], "4444"); - assert_eq!(resources["query.old.q"]["digest"], "5555"); - } - - #[tokio::test] - async fn approve_writes_digest_bound_artifact() { - let dir = fixture(); - write_applyable_state(dir.path()); - // Seed a deletable subtree. - let state = read_state_json(dir.path()); - let graph_digest_str = state["applied_revision"]["resources"]["graph.knowledge"]["digest"] - .as_str() - .unwrap() - .to_string(); - let schema_digest_str = state["applied_revision"]["resources"]["schema.knowledge"] - ["digest"] - .as_str() - .unwrap() - .to_string(); - write_state_resources( - dir.path(), - &[ - ("graph.knowledge", graph_digest_str.as_str()), - ("schema.knowledge", schema_digest_str.as_str()), - ("graph.old", "3333"), - ("schema.old", "4444"), - ], - ); - - let out = approve_config_dir(dir.path(), "graph.old", "andrew").await; - assert!(out.ok, "{:?}", out.diagnostics); - let approval_id = out.approval_id.clone().unwrap(); - let artifact: serde_json::Value = serde_json::from_str( - &fs::read_to_string( - dir.path() - .join(CLUSTER_APPROVALS_DIR) - .join(format!("{approval_id}.json")), - ) - .unwrap(), - ) - .unwrap(); - assert_eq!(artifact["resource"], "graph.old"); - assert_eq!(artifact["operation"], "delete"); - assert_eq!(artifact["approved_by"], "andrew"); - assert_eq!(artifact["bound_before_digest"], "3333"); - assert!(artifact["bound_after_digest"].is_null()); - assert!(artifact["bound_config_digest"].is_string()); - assert!(artifact["consumed_at"].is_null()); - - // A non-gated address is refused. - let not_gated = approve_config_dir(dir.path(), "query.knowledge.find_person", "andrew").await; - assert!(!not_gated.ok); - assert!( - not_gated - .diagnostics - .iter() - .any(|diagnostic| diagnostic.code == "approval_not_required") - ); - } - - #[tokio::test] - async fn stale_approval_is_ignored() { - let dir = fixture(); - write_applyable_state(dir.path()); - let state = read_state_json(dir.path()); - let graph_digest_str = state["applied_revision"]["resources"]["graph.knowledge"]["digest"] - .as_str() - .unwrap() - .to_string(); - let schema_digest_str = state["applied_revision"]["resources"]["schema.knowledge"] - ["digest"] - .as_str() - .unwrap() - .to_string(); - write_state_resources( - dir.path(), - &[ - ("graph.knowledge", graph_digest_str.as_str()), - ("schema.knowledge", schema_digest_str.as_str()), - ("graph.old", "3333"), - ], - ); - let approved = approve_config_dir(dir.path(), "graph.old", "andrew").await; - assert!(approved.ok, "{:?}", approved.diagnostics); - // The config moves after approval: the bound config digest no longer - // matches and the artifact authorizes nothing. - fs::write(dir.path().join("base.policy.yaml"), "rules: [] # moved\n").unwrap(); - - let out = apply_config_dir(dir.path()).await; - assert!( - out.diagnostics - .iter() - .any(|diagnostic| diagnostic.code == "approval_stale"), - "{:?}", - out.diagnostics - ); - let by_resource: BTreeMap<&str, &PlanChange> = out - .changes - .iter() - .map(|change| (change.resource.as_str(), change)) - .collect(); - assert_eq!( - by_resource["graph.old"].reason.as_deref(), - Some("approval_required") - ); - let state = read_state_json(dir.path()); - assert_eq!( - state["applied_revision"]["resources"]["graph.old"]["digest"], - "3333" - ); - } - - #[tokio::test] - async fn compute_approvals_one_gate_per_subtree() { - let dir = fixture(); - write_applyable_state(dir.path()); - let state = read_state_json(dir.path()); - let g = state["applied_revision"]["resources"]["graph.knowledge"]["digest"] - .as_str() - .unwrap() - .to_string(); - let sc = state["applied_revision"]["resources"]["schema.knowledge"]["digest"] - .as_str() - .unwrap() - .to_string(); - write_state_resources( - dir.path(), - &[ - ("graph.knowledge", g.as_str()), - ("schema.knowledge", sc.as_str()), - ("graph.old", "3333"), - ("schema.old", "4444"), - ("query.old.q", "5555"), - ], - ); - let plan = plan_config_dir(dir.path()).await; - let gated: Vec<&str> = plan - .approvals_required - .iter() - .map(|gate| gate.resource.as_str()) - .collect(); - assert_eq!(gated, vec!["graph.old"], "{plan:?}"); - assert!(!plan.approvals_required[0].satisfied); - } - - #[tokio::test] - async fn apply_is_idempotent() { - let dir = fixture(); - write_applyable_state(dir.path()); - - let first = apply_config_dir(dir.path()).await; - assert!(first.ok, "{:?}", first.diagnostics); - assert!(first.state_written); - let state_after_first = fs::read_to_string(dir.path().join(CLUSTER_STATE_FILE)).unwrap(); - - let second = apply_config_dir(dir.path()).await; - assert!(second.ok, "{:?}", second.diagnostics); - assert!(second.changes.is_empty()); - assert_eq!(second.applied_count, 0); - assert!(second.converged); - assert!(!second.state_written); - let state_after_second = fs::read_to_string(dir.path().join(CLUSTER_STATE_FILE)).unwrap(); - assert_eq!(state_after_first, state_after_second); - assert_eq!(second.state_observations.state_revision, 2); - } - - #[tokio::test] - async fn apply_respects_held_lock() { - let dir = fixture(); - write_applyable_state(dir.path()); - write_lock_file(dir.path(), "held-lock", "plan"); - - let out = apply_config_dir(dir.path()).await; - assert!(!out.ok); - assert!( - out.diagnostics - .iter() - .any(|diagnostic| diagnostic.code == "state_lock_held") - ); - // The held lock survives a refused apply, and nothing was written. - assert!(dir.path().join(CLUSTER_LOCK_FILE).exists()); - assert!(!dir.path().join(CLUSTER_RESOURCES_DIR).exists()); - let state = read_state_json(dir.path()); - assert_eq!(state["state_revision"], 1); - } - - #[tokio::test] - async fn apply_state_lock_false_bypasses_with_warning() { - let dir = fixture(); - fs::write( - dir.path().join(CLUSTER_CONFIG_FILE), - r#" -version: 1 -state: - backend: cluster - lock: false -graphs: - knowledge: - schema: ./people.pg - queries: - find_person: - file: ./people.gq -"#, - ) - .unwrap(); - write_applyable_state(dir.path()); - - let out = apply_config_dir(dir.path()).await; - assert!(out.ok, "{:?}", out.diagnostics); - assert!(out.state_written); - assert!(!out.state_observations.lock_acquired); - assert!( - out.diagnostics - .iter() - .any(|diagnostic| diagnostic.code == "state_lock_disabled") - ); - assert!(!dir.path().join(CLUSTER_LOCK_FILE).exists()); - } - - #[tokio::test] - async fn apply_skips_existing_payload_blob() { - let dir = fixture(); - write_applyable_state(dir.path()); - let desired = validate_config_dir(dir.path()); - let query_digest = desired - .resource_digests - .get("query.knowledge.find_person") - .unwrap() - .clone(); - // Content-addressed blobs are trusted by name: an existing file is - // never rewritten. - let blob = query_payload_path(dir.path(), &query_digest); - fs::create_dir_all(blob.parent().unwrap()).unwrap(); - fs::write(&blob, "pre-existing").unwrap(); - - let out = apply_config_dir(dir.path()).await; - assert!(out.ok, "{:?}", out.diagnostics); - assert_eq!(fs::read_to_string(&blob).unwrap(), "pre-existing"); - } - - #[tokio::test] - async fn apply_invalid_config_fails_before_lock() { - let dir = fixture(); - fs::write( - dir.path().join(CLUSTER_CONFIG_FILE), - "version: 1\nnot_a_field: true\n", - ) - .unwrap(); - - let out = apply_config_dir(dir.path()).await; - assert!(!out.ok); - // Config errors bail before the lock or any state directory exists. - assert!(!dir.path().join(CLUSTER_STATE_DIR).exists()); - } - - /// When the state write fails after payloads landed, the output must - /// report the statuses actually on disk — not the unpersisted in-memory - /// mutations (phantom `applied` entries would mislead automation that - /// reads `resource_statuses` independently of `ok`). - #[cfg(unix)] - #[tokio::test] - async fn apply_state_write_failure_reports_persisted_statuses() { - use std::os::unix::fs::PermissionsExt; - - let dir = fixture(); - // lock: false so the only write into __cluster/ is state.json itself. - fs::write( - dir.path().join(CLUSTER_CONFIG_FILE), - r#" -version: 1 -state: - backend: cluster - lock: false -graphs: - knowledge: - schema: ./people.pg - queries: - find_person: - file: ./people.gq -"#, - ) - .unwrap(); - write_applyable_state(dir.path()); - // Pre-create the payload blob so the payload phase is a no-op and the - // failure lands exactly at the state write. - let desired = validate_config_dir(dir.path()); - let query_digest = desired - .resource_digests - .get("query.knowledge.find_person") - .unwrap(); - let blob = query_payload_path(dir.path(), query_digest); - fs::create_dir_all(blob.parent().unwrap()).unwrap(); - fs::write(&blob, QUERY).unwrap(); - - let state_dir = dir.path().join(CLUSTER_STATE_DIR); - fs::set_permissions(&state_dir, fs::Permissions::from_mode(0o555)).unwrap(); - // Running as root ignores permission bits; skip rather than flake. - if fs::write(state_dir.join("probe"), b"x").is_ok() { - let _ = fs::remove_file(state_dir.join("probe")); - fs::set_permissions(&state_dir, fs::Permissions::from_mode(0o755)).unwrap(); - eprintln!("skipping: permissions are not enforced (running as root)"); - return; - } - - let out = apply_config_dir(dir.path()).await; - fs::set_permissions(&state_dir, fs::Permissions::from_mode(0o755)).unwrap(); - - assert!(!out.ok); - assert!(!out.state_written); - assert!( - out.diagnostics - .iter() - .any(|diagnostic| diagnostic.code == "state_write_error"), - "{:?}", - out.diagnostics - ); - // The seeded state has no statuses; the failed apply must not invent - // the in-memory `applied` ones it failed to persist. - assert!( - out.resource_statuses.is_empty(), - "unpersisted statuses leaked into output: {:?}", - out.resource_statuses - ); - } - - // ---- catalog payload verification (Stage 3B) ---- - - /// Converge a fixture dir and return the query blob path. - async fn converge_fixture(config_dir: &Path) -> std::path::PathBuf { - write_applyable_state(config_dir); - let out = apply_config_dir(config_dir).await; - assert!(out.ok && out.converged, "{:?}", out.diagnostics); - let desired = validate_config_dir(config_dir); - query_payload_path( - config_dir, - desired - .resource_digests - .get("query.knowledge.find_person") - .unwrap(), - ) - } - - #[tokio::test] - async fn status_reports_missing_payload_read_only() { - let dir = fixture(); - let blob = converge_fixture(dir.path()).await; - let state_before = fs::read_to_string(dir.path().join(CLUSTER_STATE_FILE)).unwrap(); - fs::remove_file(&blob).unwrap(); - - let out = status_config_dir(dir.path()); - assert!(out.ok, "{:?}", out.diagnostics); - assert!(out.diagnostics.iter().any(|diagnostic| { - diagnostic.code == "catalog_payload_missing" - && diagnostic.path == "query.knowledge.find_person" - })); - // Read-only: persisted statuses and state bytes untouched. - assert_eq!( - out.resource_statuses["query.knowledge.find_person"].status, - ResourceLifecycleStatus::Applied - ); - assert_eq!( - fs::read_to_string(dir.path().join(CLUSTER_STATE_FILE)).unwrap(), - state_before - ); - } - - #[tokio::test] - async fn refresh_removes_digest_and_drifts_on_missing_payload() { - let dir = fixture(); - init_derived_graph(dir.path()).await; - let blob = converge_fixture(dir.path()).await; - fs::remove_file(&blob).unwrap(); - - let out = refresh_config_dir(dir.path()).await; - assert!(out.ok, "{:?}", out.diagnostics); - assert!( - out.diagnostics - .iter() - .any(|diagnostic| diagnostic.code == "catalog_payload_missing") - ); - let status = &out.resource_statuses["query.knowledge.find_person"]; - assert_eq!(status.status, ResourceLifecycleStatus::Drifted); - assert!(status.conditions.contains(&"payload_missing".to_string())); - let state = read_state_json(dir.path()); - assert!( - state["applied_revision"]["resources"] - .get("query.knowledge.find_person") - .is_none(), - "{state}" - ); - } - - #[tokio::test] - async fn refresh_drifts_on_corrupted_payload() { - let dir = fixture(); - init_derived_graph(dir.path()).await; - let blob = converge_fixture(dir.path()).await; - fs::write(&blob, "corrupted content").unwrap(); - - let out = refresh_config_dir(dir.path()).await; - assert!(out.ok, "{:?}", out.diagnostics); - let status = &out.resource_statuses["query.knowledge.find_person"]; - assert_eq!(status.status, ResourceLifecycleStatus::Drifted); - assert!(status.conditions.contains(&"payload_mismatch".to_string())); - let state = read_state_json(dir.path()); - assert!( - state["applied_revision"]["resources"] - .get("query.knowledge.find_person") - .is_none() - ); - } - - #[tokio::test] - async fn refresh_flags_unreadable_payload_as_error() { - let dir = fixture(); - init_derived_graph(dir.path()).await; - let blob = converge_fixture(dir.path()).await; - // A same-named directory yields a non-NotFound IO error portably. - fs::remove_file(&blob).unwrap(); - fs::create_dir(&blob).unwrap(); - - let out = refresh_config_dir(dir.path()).await; - assert!(!out.ok); - assert!( - out.diagnostics - .iter() - .any(|diagnostic| diagnostic.code == "catalog_payload_read_error") - ); - let status = &out.resource_statuses["query.knowledge.find_person"]; - assert_eq!(status.status, ResourceLifecycleStatus::Error); - assert!(status.conditions.contains(&"payload_read_error".to_string())); - // Transient IO keeps the digest: no spurious republish. - let state = read_state_json(dir.path()); - assert!( - state["applied_revision"]["resources"] - .get("query.knowledge.find_person") - .is_some() - ); - } - - #[tokio::test] - async fn payload_drift_self_heals_through_refresh_plan_apply() { - let dir = fixture(); - init_derived_graph(dir.path()).await; - let blob = converge_fixture(dir.path()).await; - let original = fs::read_to_string(&blob).unwrap(); - fs::remove_file(&blob).unwrap(); - - let refresh = refresh_config_dir(dir.path()).await; - assert!(refresh.ok, "{:?}", refresh.diagnostics); - - let plan = plan_config_dir(dir.path()).await; - let query_change = plan - .changes - .iter() - .find(|change| change.resource == "query.knowledge.find_person") - .expect("plan must propose recreating the query"); - assert_eq!(query_change.operation, PlanOperation::Create); - assert_eq!(query_change.disposition, Some(ApplyDisposition::Applied)); - - let apply = apply_config_dir(dir.path()).await; - assert!(apply.ok && apply.converged, "{:?}", apply.diagnostics); - assert_eq!(fs::read_to_string(&blob).unwrap(), original); - - let status = status_config_dir(dir.path()); - assert!( - !status - .diagnostics - .iter() - .any(|diagnostic| diagnostic.code.starts_with("catalog_payload")), - "{:?}", - status.diagnostics - ); - } - - #[test] - fn verification_skips_graph_and_schema_resources() { - let dir = fixture(); - write_applyable_state(dir.path()); // graph + schema digests only, no blobs - - let out = status_config_dir(dir.path()); - assert!( - !out.diagnostics - .iter() - .any(|diagnostic| diagnostic.code.starts_with("catalog_payload")), - "{:?}", - out.diagnostics - ); - } - - // ---- recovery sidecars + sweep (Stage 4A) ---- - - fn derived_graph_uri(config_dir: &Path, graph_id: &str) -> String { - display_path( - &config_dir - .join(CLUSTER_GRAPHS_DIR) - .join(format!("{graph_id}.omni")), - ) - } - - fn write_create_sidecar( - config_dir: &Path, - graph_id: &str, - desired_schema_digest: &str, - operation_id: &str, - ) -> PathBuf { - let dir = config_dir.join(CLUSTER_RECOVERIES_DIR); - fs::create_dir_all(&dir).unwrap(); - let path = dir.join(format!("{operation_id}.json")); - fs::write( - &path, - serde_json::to_string_pretty(&json!({ - "schema_version": 1, - "operation_id": operation_id, - "started_at": "1970-01-01T00:00:00Z", - "kind": "graph_create", - "graph_id": graph_id, - "graph_uri": derived_graph_uri(config_dir, graph_id), - "desired_schema_digest": desired_schema_digest, - })) - .unwrap(), - ) - .unwrap(); - path - } - - #[tokio::test] - async fn sweep_removes_sidecar_when_root_absent() { - let dir = fixture(); - write_applyable_state(dir.path()); - let sidecar = write_create_sidecar(dir.path(), "knowledge", "irrelevant", "01ROW1"); - - let out = apply_config_dir(dir.path()).await; - assert!(out.ok, "{:?}", out.diagnostics); - // Row 1: nothing moved; intent removed, run proceeds normally. - assert!(!sidecar.exists()); - assert!(out.converged); - } - - #[tokio::test] - async fn sweep_rolls_forward_completed_create() { - let dir = fixture(); - init_derived_graph(dir.path()).await; - write_state_resources(dir.path(), &[]); // state predates the create - let desired = validate_config_dir(dir.path()); - let schema_digest = desired.resource_digests["schema.knowledge"].clone(); - let sidecar = write_create_sidecar(dir.path(), "knowledge", &schema_digest, "01ROW4"); - - let out = apply_config_dir(dir.path()).await; - assert!(out.ok, "{:?}", out.diagnostics); - assert!( - out.diagnostics - .iter() - .any(|diagnostic| diagnostic.code == "cluster_recovery_rolled_forward") - ); - // Row 4: ledger converged to observable reality, audit recorded, - // sidecar retired after the CAS landed. - let state = read_state_json(dir.path()); - assert_eq!( - state["applied_revision"]["resources"]["schema.knowledge"]["digest"], - schema_digest - ); - assert!( - state["recovery_records"] - .as_object() - .unwrap() - .values() - .any(|record| record["outcome"] == "rolled_forward" - && record["graph_id"] == "knowledge") - ); - assert!(!sidecar.exists()); - // With the graph rolled forward, the same run converges the catalog. - assert!(out.converged, "{out:?}"); - } - - #[tokio::test] - async fn sweep_completes_already_recorded_create() { - let dir = fixture(); - init_derived_graph(dir.path()).await; - write_applyable_state(dir.path()); // state already records graph+schema - let desired = validate_config_dir(dir.path()); - let sidecar = write_create_sidecar( - dir.path(), - "knowledge", - &desired.resource_digests["schema.knowledge"], - "01ROW2", - ); - - let out = apply_config_dir(dir.path()).await; - assert!(out.ok, "{:?}", out.diagnostics); - // Row 2: outcome was already durable; no audit entry, sidecar retired. - assert!(!sidecar.exists()); - let state = read_state_json(dir.path()); - assert!( - state["recovery_records"] - .as_object() - .is_none_or(|records| records.is_empty()), - "{state}" - ); - } - - #[tokio::test] - async fn sweep_keeps_sidecar_for_incomplete_root() { - let dir = fixture(); - write_applyable_state(dir.path()); - // A root that exists but cannot be opened: the engine's partial-init gap. - let root = dir.path().join(CLUSTER_GRAPHS_DIR).join("knowledge.omni"); - fs::create_dir_all(&root).unwrap(); - fs::write(root.join("_schema.pg"), "junk").unwrap(); - let sidecar = write_create_sidecar(dir.path(), "knowledge", "whatever", "01ROW5"); - - let out = apply_config_dir(dir.path()).await; - assert!(!out.ok); - assert!( - out.diagnostics - .iter() - .any(|diagnostic| diagnostic.code == "graph_create_incomplete") - ); - // Row 5: never auto-delete; sidecar and root stay for the operator, - // and the Error status is persisted by the run's state write. - assert!(sidecar.exists()); - assert!(root.exists()); - let state = read_state_json(dir.path()); - assert_eq!(state["resource_statuses"]["graph.knowledge"]["status"], "error"); - assert!( - state["resource_statuses"]["graph.knowledge"]["conditions"] - .as_array() - .unwrap() - .iter() - .any(|condition| condition == "graph_create_incomplete") - ); - } - - #[tokio::test] - async fn sweep_flags_unexpected_schema_as_pending() { - let dir = fixture(); - write_state_resources(dir.path(), &[]); - // Live graph exists with a schema the sidecar never intended. - let graph_dir = dir.path().join(CLUSTER_GRAPHS_DIR); - fs::create_dir_all(&graph_dir).unwrap(); - Omnigraph::init( - &derived_graph_uri(dir.path(), "knowledge"), - "\nnode Other {\n name: String @key\n}\n", - ) - .await - .unwrap(); - let desired = validate_config_dir(dir.path()); - let sidecar = write_create_sidecar( - dir.path(), - "knowledge", - &desired.resource_digests["schema.knowledge"], - "01ROW6", - ); - - let out = apply_config_dir(dir.path()).await; - assert!(out.ok, "{:?}", out.diagnostics); // warning, not error - assert!( - out.diagnostics - .iter() - .any(|diagnostic| diagnostic.code == "cluster_recovery_pending") - ); - // Row 6: refuse to guess; sidecar kept, Drifted persisted. - assert!(sidecar.exists()); - let state = read_state_json(dir.path()); - assert_eq!( - state["resource_statuses"]["graph.knowledge"]["status"], - "drifted" - ); - assert!( - state["resource_statuses"]["graph.knowledge"]["conditions"] - .as_array() - .unwrap() - .iter() - .any(|condition| condition == "actual_applied_state_pending") - ); - } - - #[tokio::test] - async fn apply_blocks_create_while_recovery_pending() { - let dir = fixture(); - write_state_resources(dir.path(), &[]); - // A kept (row 5) sidecar: partial root that cannot be opened. - let root = dir.path().join(CLUSTER_GRAPHS_DIR).join("knowledge.omni"); - fs::create_dir_all(&root).unwrap(); - fs::write(root.join("_schema.pg"), "junk").unwrap(); - let sidecar = write_create_sidecar(dir.path(), "knowledge", "whatever", "01PEND"); - - let out = apply_config_dir(dir.path()).await; - assert!(!out.ok); // row 5 is an error condition - let by_resource: BTreeMap<&str, &PlanChange> = out - .changes - .iter() - .map(|change| (change.resource.as_str(), change)) - .collect(); - // The pending recovery blocks the create and its dependents; the - // executor never attempts the init. - assert_eq!( - by_resource["graph.knowledge"].disposition, - Some(ApplyDisposition::Blocked) - ); - assert_eq!( - by_resource["graph.knowledge"].reason.as_deref(), - Some("cluster_recovery_pending") - ); - assert_eq!( - by_resource["query.knowledge.find_person"].reason.as_deref(), - Some("cluster_recovery_pending") - ); - assert_eq!( - by_resource["policy.base"].reason.as_deref(), - Some("cluster_recovery_pending") - ); - assert!(sidecar.exists()); - // The sweep's Error status is what persists — not a generic Blocked. - let state = read_state_json(dir.path()); - assert_eq!(state["resource_statuses"]["graph.knowledge"]["status"], "error"); - } - - #[tokio::test] - async fn plan_embeds_migration_preview_for_schema_update() { - let dir = fixture(); - init_derived_graph(dir.path()).await; - write_applyable_state(dir.path()); - fs::write( - dir.path().join("people.pg"), - "\nnode Person {\n name: String @key\n age: I32?\n bio: String?\n}\n", - ) - .unwrap(); - - let out = plan_config_dir(dir.path()).await; - assert!(out.ok, "{:?}", out.diagnostics); - let schema_change = out - .changes - .iter() - .find(|change| change.resource == "schema.knowledge") - .unwrap(); - let migration = schema_change.migration.as_ref().expect("preview embedded"); - assert!(migration.supported); - assert!( - serde_json::to_string(&migration.steps) - .unwrap() - .contains("add_property"), - "{migration:?}" - ); - } - - #[tokio::test] - async fn plan_warns_when_preview_unavailable() { - let dir = fixture(); - write_applyable_state(dir.path()); // digests recorded, but no live root - fs::write( - dir.path().join("people.pg"), - "\nnode Person {\n name: String @key\n age: I32?\n bio: String?\n}\n", - ) - .unwrap(); - - let out = plan_config_dir(dir.path()).await; - assert!(out.ok, "{:?}", out.diagnostics); - let schema_change = out - .changes - .iter() - .find(|change| change.resource == "schema.knowledge") - .unwrap(); - assert!(schema_change.migration.is_none()); - assert!( - out.diagnostics - .iter() - .any(|diagnostic| diagnostic.code == "schema_preview_unavailable") - ); - } - - fn write_schema_apply_sidecar( - config_dir: &Path, - graph_id: &str, - desired_schema_digest: &str, - operation_id: &str, - ) -> PathBuf { - let dir = config_dir.join(CLUSTER_RECOVERIES_DIR); - fs::create_dir_all(&dir).unwrap(); - let path = dir.join(format!("{operation_id}.json")); - fs::write( - &path, - serde_json::to_string_pretty(&json!({ - "schema_version": 1, - "operation_id": operation_id, - "started_at": "1970-01-01T00:00:00Z", - "kind": "schema_apply", - "graph_id": graph_id, - "graph_uri": derived_graph_uri(config_dir, graph_id), - "desired_schema_digest": desired_schema_digest, - })) - .unwrap(), - ) - .unwrap(); - path - } - - const SCHEMA_V2: &str = "\nnode Person {\n name: String @key\n age: I32?\n bio: String?\n}\n"; - - #[tokio::test] - async fn sweep_retires_schema_sidecar_when_ledger_consistent() { - let dir = fixture(); - init_derived_graph(dir.path()).await; - write_applyable_state(dir.path()); // state digest == live digest - let sidecar = - write_schema_apply_sidecar(dir.path(), "knowledge", "never-applied", "01SROW1"); - - let out = apply_config_dir(dir.path()).await; - assert!(out.ok, "{:?}", out.diagnostics); - assert!(!sidecar.exists()); - let state = read_state_json(dir.path()); - assert!( - state["recovery_records"] - .as_object() - .is_none_or(|records| records.is_empty()) - ); - } - - #[tokio::test] - async fn sweep_rolls_forward_completed_schema_apply() { - let dir = fixture(); - init_derived_graph(dir.path()).await; - write_applyable_state(dir.path()); - // The schema apply completed on the graph out-of-process... - let graph_uri = derived_graph_uri(dir.path(), "knowledge"); - let db = Omnigraph::open(&graph_uri).await.unwrap(); - db.apply_schema(SCHEMA_V2).await.unwrap(); - // ...the desired config matches it, and the sidecar records the intent. - fs::write(dir.path().join("people.pg"), SCHEMA_V2).unwrap(); - let desired = validate_config_dir(dir.path()); - let v2_digest = desired.resource_digests["schema.knowledge"].clone(); - let sidecar = write_schema_apply_sidecar(dir.path(), "knowledge", &v2_digest, "01SROW3"); - - let out = apply_config_dir(dir.path()).await; - assert!(out.ok, "{:?}", out.diagnostics); - assert!( - out.diagnostics - .iter() - .any(|diagnostic| diagnostic.code == "cluster_recovery_rolled_forward") - ); - assert!(!sidecar.exists()); - let state = read_state_json(dir.path()); - assert_eq!( - state["applied_revision"]["resources"]["schema.knowledge"]["digest"], - v2_digest - ); - assert!( - state["recovery_records"] - .as_object() - .unwrap() - .values() - .any(|record| record["kind"] == "schema_apply" - && record["outcome"] == "rolled_forward") - ); - assert!(out.converged, "{out:?}"); - } - - #[tokio::test] - async fn sweep_flags_unexpected_schema_apply_state_as_pending() { - let dir = fixture(); - init_derived_graph(dir.path()).await; // live = v1 - write_state_resources(dir.path(), &[("schema.knowledge", "stale-digest")]); - // Sidecar intended a digest that is neither live nor recorded. - let sidecar = - write_schema_apply_sidecar(dir.path(), "knowledge", "intended-digest", "01SROW6"); - - let out = apply_config_dir(dir.path()).await; - assert!(out.ok, "{:?}", out.diagnostics); // warnings only - assert!( - out.diagnostics - .iter() - .any(|diagnostic| diagnostic.code == "cluster_recovery_pending") - ); - assert!(sidecar.exists()); - let state = read_state_json(dir.path()); - assert_eq!( - state["resource_statuses"]["schema.knowledge"]["status"], - "drifted" - ); - } - - #[tokio::test] - async fn sweep_keeps_schema_sidecar_for_unopenable_root() { - let dir = fixture(); - write_applyable_state(dir.path()); - let root = dir.path().join(CLUSTER_GRAPHS_DIR).join("knowledge.omni"); - fs::create_dir_all(&root).unwrap(); // exists, won't open - let sidecar = - write_schema_apply_sidecar(dir.path(), "knowledge", "whatever", "01SROWX"); - - let out = apply_config_dir(dir.path()).await; - assert!(out.ok, "{:?}", out.diagnostics); // warning: cannot verify - assert!( - out.diagnostics - .iter() - .any(|diagnostic| diagnostic.code == "cluster_recovery_pending") - ); - assert!(sidecar.exists()); - } - - /// Seed: converged knowledge subtree + a stale `old` graph subtree with a - /// real directory on disk. - fn seed_deletable_state(config_dir: &Path) { - write_applyable_state(config_dir); - let state = read_state_json(config_dir); - let g = state["applied_revision"]["resources"]["graph.knowledge"]["digest"] - .as_str() - .unwrap() - .to_string(); - let sc = state["applied_revision"]["resources"]["schema.knowledge"]["digest"] - .as_str() - .unwrap() - .to_string(); - write_state_resources( - config_dir, - &[ - ("graph.knowledge", g.as_str()), - ("schema.knowledge", sc.as_str()), - ("graph.old", "3333"), - ("schema.old", "4444"), - ("query.old.q", "5555"), - ], - ); - let root = config_dir.join(CLUSTER_GRAPHS_DIR).join("old.omni"); - fs::create_dir_all(&root).unwrap(); - fs::write(root.join("_schema.pg"), "stale").unwrap(); - } - - #[tokio::test] - async fn apply_executes_approved_graph_delete() { - let dir = fixture(); - seed_deletable_state(dir.path()); - let approved = approve_config_dir(dir.path(), "graph.old", "andrew").await; - assert!(approved.ok, "{:?}", approved.diagnostics); - let approval_id = approved.approval_id.clone().unwrap(); - - let out = apply_config_dir(dir.path()).await; - assert!(out.ok, "{:?}", out.diagnostics); - assert!(out.converged, "{out:?}"); - let by_resource: BTreeMap<&str, &PlanChange> = out - .changes - .iter() - .map(|change| (change.resource.as_str(), change)) - .collect(); - assert_eq!(by_resource["graph.old"].disposition, Some(ApplyDisposition::Applied)); - assert_eq!(by_resource["schema.old"].disposition, Some(ApplyDisposition::Applied)); - assert_eq!(by_resource["query.old.q"].disposition, Some(ApplyDisposition::Applied)); - // The root is gone; the subtree is tombstoned out of the ledger. - assert!(!dir.path().join(CLUSTER_GRAPHS_DIR).join("old.omni").exists()); - let state = read_state_json(dir.path()); - let resources = state["applied_revision"]["resources"].as_object().unwrap(); - assert!(!resources.contains_key("graph.old")); - assert!(!resources.contains_key("schema.old")); - assert!(!resources.contains_key("query.old.q")); - assert_eq!(state["observations"]["graph.old"]["kind"], "tombstone"); - assert_eq!(state["observations"]["graph.old"]["approval_id"], approval_id); - // Approval consumed in BOTH stores: ledger summary + artifact file. - assert!(state["approval_records"][&approval_id]["consumed_at"].is_string()); - let artifact: serde_json::Value = serde_json::from_str( - &fs::read_to_string( - dir.path() - .join(CLUSTER_APPROVALS_DIR) - .join(format!("{approval_id}.json")), - ) - .unwrap(), - ) - .unwrap(); - assert!(artifact["consumed_at"].is_string(), "{artifact}"); - // Sidecar retired. - assert!( - fs::read_dir(dir.path().join(CLUSTER_RECOVERIES_DIR)) - .map(|mut entries| entries.next().is_none()) - .unwrap_or(true) - ); - // A consumed approval authorizes nothing further (idempotent re-apply). - let again = apply_config_dir(dir.path()).await; - assert!(again.ok && again.converged && !again.state_written, "{again:?}"); - } - - fn write_delete_sidecar( - config_dir: &Path, - graph_id: &str, - approval_id: Option<&str>, - operation_id: &str, - ) -> PathBuf { - let dir = config_dir.join(CLUSTER_RECOVERIES_DIR); - fs::create_dir_all(&dir).unwrap(); - let path = dir.join(format!("{operation_id}.json")); - fs::write( - &path, - serde_json::to_string_pretty(&json!({ - "schema_version": 1, - "operation_id": operation_id, - "started_at": "1970-01-01T00:00:00Z", - "kind": "graph_delete", - "graph_id": graph_id, - "graph_uri": derived_graph_uri(config_dir, graph_id), - "desired_schema_digest": "", - "approval_id": approval_id, - })) - .unwrap(), - ) - .unwrap(); - path - } - - #[tokio::test] - async fn sweep_retires_delete_sidecar_when_tombstoned() { - let dir = fixture(); - write_applyable_state(dir.path()); // no graph.old in state, no root - let sidecar = write_delete_sidecar(dir.path(), "old", None, "01DROW7"); - - let out = apply_config_dir(dir.path()).await; - assert!(out.ok, "{:?}", out.diagnostics); - assert!(!sidecar.exists()); - let state = read_state_json(dir.path()); - assert!( - state["recovery_records"] - .as_object() - .is_none_or(|records| records.is_empty()) - ); - } - - #[tokio::test] - async fn sweep_rolls_forward_completed_delete() { - let dir = fixture(); - seed_deletable_state(dir.path()); - // Approve, then simulate: root removed, state stale, sidecar present. - let approved = approve_config_dir(dir.path(), "graph.old", "andrew").await; - let approval_id = approved.approval_id.unwrap(); - fs::remove_dir_all(dir.path().join(CLUSTER_GRAPHS_DIR).join("old.omni")).unwrap(); - let sidecar = write_delete_sidecar(dir.path(), "old", Some(&approval_id), "01DROW7B"); - - let out = apply_config_dir(dir.path()).await; - assert!(out.ok, "{:?}", out.diagnostics); - assert!( - out.diagnostics - .iter() - .any(|diagnostic| diagnostic.code == "cluster_recovery_rolled_forward") - ); - assert!(!sidecar.exists()); - let state = read_state_json(dir.path()); - assert!( - !state["applied_revision"]["resources"] - .as_object() - .unwrap() - .contains_key("graph.old") - ); - assert_eq!(state["observations"]["graph.old"]["kind"], "tombstone"); - assert!(state["approval_records"][&approval_id]["consumed_at"].is_string()); - assert!( - state["recovery_records"] - .as_object() - .unwrap() - .values() - .any(|record| record["kind"] == "graph_delete" - && record["outcome"] == "rolled_forward") - ); - // The artifact file is marked consumed post-CAS. - let artifact: serde_json::Value = serde_json::from_str( - &fs::read_to_string( - dir.path() - .join(CLUSTER_APPROVALS_DIR) - .join(format!("{approval_id}.json")), - ) - .unwrap(), - ) - .unwrap(); - assert!(artifact["consumed_at"].is_string()); - assert!(out.converged, "{out:?}"); - } - - #[tokio::test] - async fn sweep_reproposes_incomplete_delete() { - let dir = fixture(); - seed_deletable_state(dir.path()); // root present - let approved = approve_config_dir(dir.path(), "graph.old", "andrew").await; - assert!(approved.ok); - let sidecar = write_delete_sidecar(dir.path(), "old", approved.approval_id.as_deref(), "01DROW8"); - - // Row 8: the stale intent is retired with a warning, and the same run - // re-executes the still-approved delete to completion. - let out = apply_config_dir(dir.path()).await; - assert!(out.ok, "{:?}", out.diagnostics); - assert!( - out.diagnostics - .iter() - .any(|diagnostic| diagnostic.code == "graph_delete_incomplete") - ); - assert!(!sidecar.exists()); - assert!(!dir.path().join(CLUSTER_GRAPHS_DIR).join("old.omni").exists()); - assert!(out.converged, "{out:?}"); - } - - // ---- policy bindings in the applied revision (5A) ---- - - #[tokio::test] - async fn apply_records_policy_bindings() { - let dir = fixture(); - write_applyable_state(dir.path()); - - let out = apply_config_dir(dir.path()).await; - assert!(out.ok && out.converged, "{:?}", out.diagnostics); - let state = read_state_json(dir.path()); - assert_eq!( - state["applied_revision"]["resources"]["policy.base"]["applies_to"], - serde_json::json!(["graph.knowledge"]), - "{state}" - ); - // Non-policy entries carry no bindings field at all. - assert!( - state["applied_revision"]["resources"]["query.knowledge.find_person"] - .get("applies_to") - .is_none() - ); - } - - #[tokio::test] - async fn binding_change_is_a_visible_plan_change() { - let dir = fixture(); - write_applyable_state(dir.path()); - let converge = apply_config_dir(dir.path()).await; - assert!(converge.converged, "{converge:?}"); - // Edit ONLY applies_to: the policy file digest is unchanged. - fs::write( - dir.path().join(CLUSTER_CONFIG_FILE), - r#" -version: 1 -metadata: - name: test -state: - backend: cluster - lock: true -graphs: - knowledge: - schema: ./people.pg - queries: - find_person: - file: ./people.gq -policies: - base: - file: ./base.policy.yaml - applies_to: [cluster, knowledge] -"#, - ) - .unwrap(); - - let plan = plan_config_dir(dir.path()).await; - let change = plan - .changes - .iter() - .find(|change| change.resource == "policy.base") - .expect("binding change must be visible in plan"); - assert!(change.binding_change); - assert_eq!(change.operation, PlanOperation::Update); - assert_eq!(change.before_digest, change.after_digest); - - let out = apply_config_dir(dir.path()).await; - assert!(out.ok && out.converged, "{out:?}"); - let state = read_state_json(dir.path()); - assert_eq!( - state["applied_revision"]["resources"]["policy.base"]["applies_to"], - serde_json::json!(["cluster", "graph.knowledge"]) - ); - // Idempotent: a second run sees no changes. - let again = apply_config_dir(dir.path()).await; - assert!(again.changes.is_empty() && !again.state_written, "{again:?}"); - } - - #[tokio::test] - async fn pre_5a_state_backfills_bindings() { - let dir = fixture(); - write_applyable_state(dir.path()); - let converge = apply_config_dir(dir.path()).await; - assert!(converge.converged, "{converge:?}"); - // Strip the bindings from the state entry (a pre-5A ledger). - let mut state: serde_json::Value = serde_json::from_str( - &fs::read_to_string(dir.path().join(CLUSTER_STATE_FILE)).unwrap(), - ) - .unwrap(); - state["applied_revision"]["resources"]["policy.base"] - .as_object_mut() - .unwrap() - .remove("applies_to"); - fs::write( - dir.path().join(CLUSTER_STATE_FILE), - serde_json::to_string_pretty(&state).unwrap(), - ) - .unwrap(); - - let plan = plan_config_dir(dir.path()).await; - assert!( - plan.changes - .iter() - .any(|change| change.resource == "policy.base" && change.binding_change), - "{plan:?}" - ); - let out = apply_config_dir(dir.path()).await; - assert!(out.ok && out.converged, "{out:?}"); - let healed = read_state_json(dir.path()); - assert_eq!( - healed["applied_revision"]["resources"]["policy.base"]["applies_to"], - serde_json::json!(["graph.knowledge"]) - ); - } - - #[tokio::test] - async fn bindings_survive_refresh() { - let dir = fixture(); - init_derived_graph(dir.path()).await; - write_applyable_state(dir.path()); - let converge = apply_config_dir(dir.path()).await; - assert!(converge.converged, "{converge:?}"); - - let refresh = refresh_config_dir(dir.path()).await; - assert!(refresh.ok, "{:?}", refresh.diagnostics); - let state = read_state_json(dir.path()); - assert_eq!( - state["applied_revision"]["resources"]["policy.base"]["applies_to"], - serde_json::json!(["graph.knowledge"]) - ); - } - - // ---- serving snapshot (5B read-only loader) ---- - - #[tokio::test] - async fn serving_snapshot_reads_converged_cluster() { - let dir = fixture(); - init_derived_graph(dir.path()).await; - write_applyable_state(dir.path()); - let converge = apply_config_dir(dir.path()).await; - assert!(converge.converged, "{converge:?}"); - - let snapshot = read_serving_snapshot(dir.path()).expect("converged cluster must serve"); - assert_eq!(snapshot.graphs.len(), 1); - assert_eq!(snapshot.graphs[0].graph_id, "knowledge"); - assert!(snapshot.graphs[0].root.ends_with("graphs/knowledge.omni")); - assert_eq!(snapshot.queries.len(), 1); - assert_eq!(snapshot.queries[0].name, "find_person"); - assert!(snapshot.queries[0].source.contains("query find_person")); - assert_eq!(snapshot.policies.len(), 1); - assert_eq!(snapshot.policies[0].applies_to, vec!["graph.knowledge"]); - assert!(snapshot.policies[0].blob_path.exists()); - } - - #[test] - fn serving_snapshot_refuses_missing_state() { - let dir = fixture(); - let err = read_serving_snapshot(dir.path()).unwrap_err(); - assert!( - err.iter().any(|diagnostic| diagnostic.code == "cluster_state_missing"), - "{err:?}" - ); - } - - #[tokio::test] - async fn serving_snapshot_refuses_pending_recovery() { - let dir = fixture(); - init_derived_graph(dir.path()).await; - write_applyable_state(dir.path()); - apply_config_dir(dir.path()).await; - write_schema_apply_sidecar(dir.path(), "knowledge", "whatever", "01SERVE"); - - let err = read_serving_snapshot(dir.path()).unwrap_err(); - assert!( - err.iter().any(|diagnostic| diagnostic.code == "cluster_recovery_pending"), - "{err:?}" - ); - } - - #[tokio::test] - async fn serving_snapshot_refuses_tampered_blob_and_stripped_bindings() { - let dir = fixture(); - init_derived_graph(dir.path()).await; - write_applyable_state(dir.path()); - apply_config_dir(dir.path()).await; - // Tamper with the query blob... - let snapshot = read_serving_snapshot(dir.path()).unwrap(); - let desired = validate_config_dir(dir.path()); - let query_digest = &desired.resource_digests["query.knowledge.find_person"]; - let blob = dir - .path() - .join(CLUSTER_RESOURCES_DIR) - .join("query/knowledge/find_person") - .join(format!("{query_digest}.gq")); - fs::write(&blob, "tampered").unwrap(); - // ...and strip the policy bindings (pre-5A ledger). - let mut state: serde_json::Value = serde_json::from_str( - &fs::read_to_string(dir.path().join(CLUSTER_STATE_FILE)).unwrap(), - ) - .unwrap(); - state["applied_revision"]["resources"]["policy.base"] - .as_object_mut() - .unwrap() - .remove("applies_to"); - fs::write( - dir.path().join(CLUSTER_STATE_FILE), - serde_json::to_string_pretty(&state).unwrap(), - ) - .unwrap(); - - let err = read_serving_snapshot(dir.path()).unwrap_err(); - assert!( - err.iter() - .any(|diagnostic| diagnostic.code == "catalog_payload_digest_mismatch"), - "{err:?}" - ); - assert!( - err.iter().any(|diagnostic| diagnostic.code == "policy_bindings_missing"), - "{err:?}" - ); - let _ = snapshot; // the pre-tamper read succeeded - } - - #[test] - fn serving_snapshot_refuses_empty_cluster() { - let dir = fixture(); - write_state_resources(dir.path(), &[]); // state exists, no graphs - - let err = read_serving_snapshot(dir.path()).unwrap_err(); - assert!( - err.iter().any(|diagnostic| diagnostic.code == "cluster_empty"), - "{err:?}" - ); - } - - // ---- query discovery (Terraform-style declaration) ---- - - #[test] - fn queries_directory_discovers_every_declaration() { - let dir = tempfile::tempdir().unwrap(); - fs::write(dir.path().join("people.pg"), "\nnode Person {\n name: String @key\n}\n").unwrap(); - fs::create_dir(dir.path().join("queries")).unwrap(); - fs::write( - dir.path().join("queries/people.gq"), - "\nquery find_person($name: String) {\n match { $p: Person { name: $name } }\n return { $p.name }\n}\n\nquery all_people() {\n match { $p: Person }\n return { $p.name }\n}\n", - ) - .unwrap(); - fs::write( - dir.path().join("queries/extra.gq"), - "\nquery count_people() {\n match { $p: Person }\n return { count($p) }\n}\n", - ) - .unwrap(); - fs::write(dir.path().join("queries/notes.txt"), "ignored").unwrap(); - fs::write( - dir.path().join("cluster.yaml"), - "version: 1\ngraphs:\n knowledge:\n schema: ./people.pg\n queries: ./queries/\n", - ) - .unwrap(); - - let out = validate_config_dir(dir.path()); - assert!(out.ok, "{:?}", out.diagnostics); - let names: Vec<&str> = out - .resource_digests - .keys() - .filter_map(|address| address.strip_prefix("query.knowledge.")) - .collect(); - assert_eq!(names, vec!["all_people", "count_people", "find_person"]); - } - - #[test] - fn queries_list_and_single_file_forms_discover() { - let dir = tempfile::tempdir().unwrap(); - fs::write(dir.path().join("people.pg"), "\nnode Person {\n name: String @key\n}\n").unwrap(); - fs::write( - dir.path().join("a.gq"), - "\nquery find_person($name: String) {\n match { $p: Person { name: $name } }\n return { $p.name }\n}\n", - ) - .unwrap(); - fs::write( - dir.path().join("b.gq"), - "\nquery all_people() {\n match { $p: Person }\n return { $p.name }\n}\n", - ) - .unwrap(); - fs::write( - dir.path().join("cluster.yaml"), - "version: 1\ngraphs:\n knowledge:\n schema: ./people.pg\n queries: [./a.gq, ./b.gq]\n", - ) - .unwrap(); - let out = validate_config_dir(dir.path()); - assert!(out.ok, "{:?}", out.diagnostics); - assert!(out.resource_digests.contains_key("query.knowledge.find_person")); - assert!(out.resource_digests.contains_key("query.knowledge.all_people")); - - // Single-file string form - fs::write( - dir.path().join("cluster.yaml"), - "version: 1\ngraphs:\n knowledge:\n schema: ./people.pg\n queries: ./a.gq\n", - ) - .unwrap(); - let out = validate_config_dir(dir.path()); - assert!(out.ok, "{:?}", out.diagnostics); - assert!(out.resource_digests.contains_key("query.knowledge.find_person")); - assert!(!out.resource_digests.contains_key("query.knowledge.all_people")); - } - - #[test] - fn query_discovery_rejects_duplicates_and_parse_errors() { - let dir = tempfile::tempdir().unwrap(); - fs::write(dir.path().join("people.pg"), "\nnode Person {\n name: String @key\n}\n").unwrap(); - let decl = "\nquery find_person($name: String) {\n match { $p: Person { name: $name } }\n return { $p.name }\n}\n"; - fs::write(dir.path().join("a.gq"), decl).unwrap(); - fs::write(dir.path().join("b.gq"), decl).unwrap(); - fs::write( - dir.path().join("cluster.yaml"), - "version: 1\ngraphs:\n knowledge:\n schema: ./people.pg\n queries: [./a.gq, ./b.gq]\n", - ) - .unwrap(); - let out = validate_config_dir(dir.path()); - assert!(!out.ok); - assert!( - out.diagnostics - .iter() - .any(|diagnostic| diagnostic.code == "duplicate_query_name"), - "{:?}", - out.diagnostics - ); - - fs::write(dir.path().join("broken.gq"), "query {{{ nope").unwrap(); - fs::write( - dir.path().join("cluster.yaml"), - "version: 1\ngraphs:\n knowledge:\n schema: ./people.pg\n queries: ./broken.gq\n", - ) - .unwrap(); - let out = validate_config_dir(dir.path()); - assert!(!out.ok); - assert!( - out.diagnostics - .iter() - .any(|diagnostic| diagnostic.code == "query_parse_error"), - "{:?}", - out.diagnostics - ); - } - - #[test] - fn status_warns_on_pending_recovery_sidecar() { - let dir = fixture(); - write_applyable_state(dir.path()); - write_create_sidecar(dir.path(), "knowledge", "irrelevant", "01STATUS"); - - let out = status_config_dir(dir.path()); - assert!(out.ok, "{:?}", out.diagnostics); - assert!( - out.diagnostics - .iter() - .any(|diagnostic| diagnostic.code == "cluster_recovery_pending" - && diagnostic.severity == DiagnosticSeverity::Warning) - ); - } - - #[tokio::test] - async fn plan_annotates_apply_dispositions() { - let dir = fixture(); - let out = plan_config_dir(dir.path()).await; - assert!(out.ok, "{:?}", out.diagnostics); - let by_resource: BTreeMap<&str, &PlanChange> = out - .changes - .iter() - .map(|change| (change.resource.as_str(), change)) - .collect(); - // Stage 4A: graph/schema creates are executable, and dependents ride - // the same run — plan previews exactly that. - assert_eq!( - by_resource["graph.knowledge"].disposition, - Some(ApplyDisposition::Applied) - ); - assert_eq!( - by_resource["schema.knowledge"].disposition, - Some(ApplyDisposition::Applied) - ); - assert_eq!( - by_resource["query.knowledge.find_person"].disposition, - Some(ApplyDisposition::Applied) - ); - assert_eq!( - by_resource["policy.base"].disposition, - Some(ApplyDisposition::Applied) - ); - } -} +#[path = "tests.rs"] +mod tests; diff --git a/crates/omnigraph-cluster/src/tests.rs b/crates/omnigraph-cluster/src/tests.rs new file mode 100644 index 0000000..a03c522 --- /dev/null +++ b/crates/omnigraph-cluster/src/tests.rs @@ -0,0 +1,3019 @@ +//! In-source test suite, moved verbatim from lib.rs (modularization). +//! Indentation is preserved exactly — embedded raw-string fixtures +//! (cluster.yaml/JSON bodies) are content, not formatting. +#![allow(clippy::all)] + + use std::fs; + use std::path::Path; + + use omnigraph::db::Omnigraph; + use serde_json::json; + use tempfile::tempdir; + + use super::*; + + const SCHEMA: &str = r#" +node Person { + name: String @key + age: I32? +} +"#; + + const QUERY: &str = r#" +query find_person($name: String) { + match { $p: Person { name: $name } } + return { $p.name, $p.age } +} +"#; + + fn fixture() -> tempfile::TempDir { + let dir = tempdir().unwrap(); + fs::write(dir.path().join("people.pg"), SCHEMA).unwrap(); + fs::write(dir.path().join("people.gq"), QUERY).unwrap(); + fs::write(dir.path().join("base.policy.yaml"), "rules: []\n").unwrap(); + fs::write( + dir.path().join(CLUSTER_CONFIG_FILE), + r#" +version: 1 +metadata: + name: test +state: + backend: cluster + lock: true +graphs: + knowledge: + schema: ./people.pg + queries: + find_person: + file: ./people.gq +policies: + base: + file: ./base.policy.yaml + applies_to: [knowledge] +"#, + ) + .unwrap(); + dir + } + + async fn init_derived_graph(root: &Path) { + let graph_dir = root.join(CLUSTER_GRAPHS_DIR); + fs::create_dir_all(&graph_dir).unwrap(); + let graph = graph_dir.join("knowledge.omni"); + Omnigraph::init(graph.to_string_lossy().as_ref(), SCHEMA) + .await + .unwrap(); + } + + fn write_lock_file(config_dir: &Path, lock_id: &str, operation: &str) { + let state_dir = config_dir.join(CLUSTER_STATE_DIR); + fs::create_dir_all(&state_dir).unwrap(); + fs::write( + state_dir.join("lock.json"), + json!({ + "version": 1, + "lock_id": lock_id, + "operation": operation, + "created_at": "1970-01-01T00:00:00Z", + "pid": 123 + }) + .to_string(), + ) + .unwrap(); + } + + #[test] + fn valid_minimal_config() { + let dir = fixture(); + let out = validate_config_dir(dir.path()); + assert!(out.ok, "{:?}", out.diagnostics); + assert!(out.resource_digests.contains_key("graph.knowledge")); + assert!(out.resource_digests.contains_key("schema.knowledge")); + assert!( + out.dependencies + .iter() + .any(|dep| dep.from == "policy.base" && dep.to == "graph.knowledge") + ); + } + + #[test] + fn unknown_field_rejection() { + let dir = fixture(); + fs::write( + dir.path().join(CLUSTER_CONFIG_FILE), + "version: 1\ngraphs: {}\nwat: true\n", + ) + .unwrap(); + let out = validate_config_dir(dir.path()); + assert!(!out.ok); + assert!(out.diagnostics[0].message.contains("unknown field")); + } + + #[test] + fn future_phase_field_rejection() { + let dir = fixture(); + fs::write( + dir.path().join(CLUSTER_CONFIG_FILE), + "version: 1\ngraphs: {}\npipelines: {}\n", + ) + .unwrap(); + let out = validate_config_dir(dir.path()); + assert!(!out.ok); + assert_eq!(out.diagnostics[0].code, "future_phase_field"); + } + + #[test] + fn duplicate_yaml_key_rejection() { + let dir = fixture(); + fs::write( + dir.path().join(CLUSTER_CONFIG_FILE), + "version: 1\ngraphs: {}\ngraphs: {}\n", + ) + .unwrap(); + let out = validate_config_dir(dir.path()); + assert!(!out.ok); + assert_eq!(out.diagnostics[0].code, "duplicate_yaml_key"); + } + + #[test] + fn duplicate_yaml_key_rejection_keeps_quoted_hashes() { + let diagnostics = + duplicate_key_diagnostics("\"name#display\": one\n\"name#display\": two\n"); + assert_eq!(diagnostics.len(), 1); + assert_eq!(diagnostics[0].code, "duplicate_yaml_key"); + } + + #[test] + fn missing_schema_query_and_policy_files() { + let dir = fixture(); + fs::write( + dir.path().join(CLUSTER_CONFIG_FILE), + r#" +version: 1 +graphs: + knowledge: + schema: ./missing.pg + queries: + find_person: { file: ./missing.gq } +policies: + base: + file: ./missing.policy.yaml + applies_to: [knowledge] +"#, + ) + .unwrap(); + let out = validate_config_dir(dir.path()); + assert!(!out.ok); + let codes: BTreeSet<_> = out.diagnostics.iter().map(|d| d.code.as_str()).collect(); + assert!(codes.contains("schema_file_missing")); + assert!(codes.contains("query_file_missing")); + assert!(codes.contains("policy_file_missing")); + } + + #[test] + fn wrong_kind_and_dangling_refs_fail() { + let dir = fixture(); + fs::write( + dir.path().join(CLUSTER_CONFIG_FILE), + r#" +version: 1 +graphs: + knowledge: + schema: ./people.pg +policies: + base: + file: ./base.policy.yaml + applies_to: [query.knowledge.find_person, missing] +"#, + ) + .unwrap(); + let out = validate_config_dir(dir.path()); + assert!(!out.ok); + let codes: BTreeSet<_> = out.diagnostics.iter().map(|d| d.code.as_str()).collect(); + assert!(codes.contains("wrong_kind_reference")); + assert!(codes.contains("dangling_graph_reference")); + } + + #[test] + fn query_key_mismatch_fails() { + let dir = fixture(); + fs::write( + dir.path().join(CLUSTER_CONFIG_FILE), + r#" +version: 1 +graphs: + knowledge: + schema: ./people.pg + queries: + different: { file: ./people.gq } +"#, + ) + .unwrap(); + let out = validate_config_dir(dir.path()); + assert!(!out.ok); + assert_eq!(out.diagnostics[0].code, "query_key_mismatch"); + } + + #[test] + fn query_typecheck_failure_fails() { + let dir = fixture(); + fs::write( + dir.path().join("people.gq"), + "query find_person() { match { $d: DoesNotExist } return { $d.name } }\n", + ) + .unwrap(); + let out = validate_config_dir(dir.path()); + assert!(!out.ok); + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "query_typecheck_error") + ); + } + + #[tokio::test] + async fn missing_state_plans_creates() { + let dir = fixture(); + let out = plan_config_dir(dir.path()).await; + assert!(out.ok, "{:?}", out.diagnostics); + assert!(!out.state_observations.state_found); + assert!(!out.state_observations.locked); + assert!(out.state_observations.lock_acquired); + assert!( + out.changes + .iter() + .all(|c| c.operation == PlanOperation::Create) + ); + assert!(out.changes.iter().any(|c| c.resource == "graph.knowledge")); + assert!(!dir.path().join(CLUSTER_LOCK_FILE).exists()); + } + + #[tokio::test] + async fn config_digest_ignores_yaml_comments_and_formatting() { + let dir = fixture(); + let first = plan_config_dir(dir.path()).await; + assert!(first.ok, "{:?}", first.diagnostics); + + fs::write( + dir.path().join(CLUSTER_CONFIG_FILE), + r#" +# Same semantic config as the fixture, intentionally rendered differently. +version: 1 +metadata: { name: test } +state: { backend: cluster, lock: true } +graphs: + knowledge: + schema: ./people.pg + queries: { find_person: { file: ./people.gq } } +policies: + base: + file: ./base.policy.yaml + applies_to: + - knowledge +"#, + ) + .unwrap(); + + let second = plan_config_dir(dir.path()).await; + assert!(second.ok, "{:?}", second.diagnostics); + assert_eq!( + first.desired_revision.config_digest, + second.desired_revision.config_digest + ); + } + + #[tokio::test] + async fn existing_state_plans_update_and_delete_deterministically() { + let dir = fixture(); + let first = plan_config_dir(dir.path()).await; + let state_dir = dir.path().join("__cluster"); + fs::create_dir_all(&state_dir).unwrap(); + fs::write( + state_dir.join("state.json"), + serde_json::to_string_pretty(&json!({ + "version": 1, + "applied_revision": { + "config_digest": "old", + "resources": { + "graph.knowledge": { "digest": first.resource_digests["graph.knowledge"] }, + "policy.old": { "digest": "abc" }, + "schema.knowledge": { "digest": "old-schema" } + } + } + })) + .unwrap(), + ) + .unwrap(); + + let out = plan_config_dir(dir.path()).await; + assert!(out.ok, "{:?}", out.diagnostics); + let rendered: Vec<_> = out + .changes + .iter() + .map(|change| (change.resource.as_str(), &change.operation)) + .collect(); + assert_eq!( + rendered, + vec![ + ("policy.base", &PlanOperation::Create), + ("policy.old", &PlanOperation::Delete), + ("query.knowledge.find_person", &PlanOperation::Create), + ("schema.knowledge", &PlanOperation::Update), + ] + ); + } + + #[tokio::test] + async fn old_minimal_state_json_still_plans_with_default_revision() { + let dir = fixture(); + let state_dir = dir.path().join(CLUSTER_STATE_DIR); + fs::create_dir_all(&state_dir).unwrap(); + fs::write( + state_dir.join("state.json"), + r#"{ + "version": 1, + "applied_revision": { + "config_digest": "old", + "resources": { + "graph.knowledge": { "digest": "old-graph" } + } + } +}"#, + ) + .unwrap(); + + let out = plan_config_dir(dir.path()).await; + assert!(out.ok, "{:?}", out.diagnostics); + assert_eq!(out.state_observations.state_revision, 0); + assert!(out.state_observations.state_cas.is_some()); + assert!(out.changes.iter().any(|change| { + change.resource == "graph.knowledge" && change.operation == PlanOperation::Update + })); + } + + #[test] + fn extended_state_json_status_surfaces_statuses() { + let dir = fixture(); + let state_dir = dir.path().join(CLUSTER_STATE_DIR); + fs::create_dir_all(&state_dir).unwrap(); + let state = r#"{ + "version": 1, + "state_revision": 42, + "applied_revision": { + "config_digest": "applied-config", + "resources": { + "graph.knowledge": { "digest": "graph-digest" } + } + }, + "resource_statuses": { + "graph.knowledge": { + "status": "applied", + "conditions": ["healthy"], + "message": "ready" + } + }, + "approval_records": {}, + "recovery_records": {}, + "observations": { + "graph.knowledge": { "manifest_version": 12 } + } +}"#; + fs::write(state_dir.join("state.json"), state).unwrap(); + + let out = status_config_dir(dir.path()); + assert!(out.ok, "{:?}", out.diagnostics); + assert!(out.state_observations.state_found); + assert_eq!(out.state_observations.state_revision, 42); + assert_eq!( + out.state_observations.state_cas.as_deref(), + Some(format!("sha256:{}", sha256_hex(state.as_bytes())).as_str()) + ); + assert_eq!( + out.resource_digests + .get("graph.knowledge") + .map(String::as_str), + Some("graph-digest") + ); + assert_eq!( + out.resource_statuses["graph.knowledge"].status, + ResourceLifecycleStatus::Applied + ); + } + + #[test] + fn missing_state_status_succeeds_with_warning() { + let dir = fixture(); + let out = status_config_dir(dir.path()); + assert!(out.ok, "{:?}", out.diagnostics); + assert!(!out.state_observations.state_found); + assert_eq!(out.state_observations.state_revision, 0); + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "state_missing") + ); + } + + #[test] + fn invalid_state_status_fails() { + let dir = fixture(); + let state_dir = dir.path().join(CLUSTER_STATE_DIR); + fs::create_dir_all(&state_dir).unwrap(); + fs::write(state_dir.join("state.json"), "{").unwrap(); + + let out = status_config_dir(dir.path()); + assert!(!out.ok); + assert!(out.state_observations.state_found); + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "invalid_state_json") + ); + } + + #[test] + fn status_surfaces_full_lock_metadata() { + let dir = fixture(); + write_lock_file(dir.path(), "held-lock", "refresh"); + + let out = status_config_dir(dir.path()); + assert!(out.ok, "{:?}", out.diagnostics); + assert!(out.state_observations.locked); + assert_eq!(out.state_observations.lock_id.as_deref(), Some("held-lock")); + assert_eq!( + out.state_observations.lock_operation.as_deref(), + Some("refresh") + ); + assert_eq!( + out.state_observations.lock_created_at.as_deref(), + Some("1970-01-01T00:00:00Z") + ); + assert_eq!(out.state_observations.lock_pid, Some(123)); + assert!(out.state_observations.lock_age_seconds.is_some()); + } + + #[test] + fn force_unlock_matching_id_removes_lock() { + let dir = fixture(); + write_lock_file(dir.path(), "held-lock", "plan"); + + let out = force_unlock_config_dir(dir.path(), "held-lock"); + assert!(out.ok, "{:?}", out.diagnostics); + assert!(out.lock_removed); + assert_eq!(out.state_observations.lock_id.as_deref(), Some("held-lock")); + assert_eq!( + out.state_observations.lock_operation.as_deref(), + Some("plan") + ); + assert!(!dir.path().join(CLUSTER_LOCK_FILE).exists()); + } + + #[test] + fn force_unlock_wrong_id_fails_and_preserves_lock() { + let dir = fixture(); + write_lock_file(dir.path(), "held-lock", "plan"); + + let out = force_unlock_config_dir(dir.path(), "other-lock"); + assert!(!out.ok); + assert!(!out.lock_removed); + assert_eq!(out.state_observations.lock_id.as_deref(), Some("held-lock")); + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "state_lock_id_mismatch") + ); + assert!(dir.path().join(CLUSTER_LOCK_FILE).exists()); + } + + #[test] + fn force_unlock_missing_lock_fails() { + let dir = fixture(); + + let out = force_unlock_config_dir(dir.path(), "held-lock"); + assert!(!out.ok); + assert!(!out.lock_removed); + assert!(!out.state_observations.locked); + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "state_lock_missing") + ); + } + + #[test] + fn force_unlock_invalid_lock_json_fails_and_preserves_lock() { + let dir = fixture(); + let state_dir = dir.path().join(CLUSTER_STATE_DIR); + fs::create_dir_all(&state_dir).unwrap(); + fs::write(state_dir.join("lock.json"), "{").unwrap(); + + let out = force_unlock_config_dir(dir.path(), "held-lock"); + assert!(!out.ok); + assert!(!out.lock_removed); + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "invalid_state_lock") + ); + assert!(dir.path().join(CLUSTER_LOCK_FILE).exists()); + } + + #[test] + fn force_unlock_unsupported_lock_version_fails_and_preserves_lock() { + let dir = fixture(); + let state_dir = dir.path().join(CLUSTER_STATE_DIR); + fs::create_dir_all(&state_dir).unwrap(); + fs::write( + state_dir.join("lock.json"), + r#"{"version":2,"lock_id":"held-lock","operation":"plan","created_at":"1970-01-01T00:00:00Z","pid":123}"#, + ) + .unwrap(); + + let out = force_unlock_config_dir(dir.path(), "held-lock"); + assert!(!out.ok); + assert!(!out.lock_removed); + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "unsupported_state_lock_version") + ); + assert!(dir.path().join(CLUSTER_LOCK_FILE).exists()); + } + + #[test] + fn force_unlock_external_state_backend_rejected() { + let dir = fixture(); + write_lock_file(dir.path(), "held-lock", "plan"); + fs::write( + dir.path().join(CLUSTER_CONFIG_FILE), + r#" +version: 1 +state: + backend: s3://state-bucket/cluster +graphs: + knowledge: + schema: ./people.pg +"#, + ) + .unwrap(); + + let out = force_unlock_config_dir(dir.path(), "held-lock"); + assert!(!out.ok); + assert!(!out.lock_removed); + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "unsupported_state_backend") + ); + assert!(dir.path().join(CLUSTER_LOCK_FILE).exists()); + } + + #[tokio::test] + async fn plan_succeeds_after_force_unlock() { + let dir = fixture(); + write_lock_file(dir.path(), "held-lock", "plan"); + + let locked = plan_config_dir(dir.path()).await; + assert!(!locked.ok); + assert!( + locked + .diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "state_lock_held") + ); + + let unlocked = force_unlock_config_dir(dir.path(), "held-lock"); + assert!(unlocked.ok, "{:?}", unlocked.diagnostics); + + let out = plan_config_dir(dir.path()).await; + assert!(out.ok, "{:?}", out.diagnostics); + } + + #[tokio::test] + async fn plan_reports_state_cas_revision_and_removes_lock() { + let dir = fixture(); + let state_dir = dir.path().join(CLUSTER_STATE_DIR); + fs::create_dir_all(&state_dir).unwrap(); + let state = r#"{ + "version": 1, + "state_revision": 7, + "applied_revision": { + "config_digest": "old", + "resources": { + "graph.knowledge": { "digest": "old-graph" } + } + } +}"#; + fs::write(state_dir.join("state.json"), state).unwrap(); + + let out = plan_config_dir(dir.path()).await; + assert!(out.ok, "{:?}", out.diagnostics); + assert_eq!(out.state_observations.state_revision, 7); + assert_eq!( + out.state_observations.state_cas.as_deref(), + Some(format!("sha256:{}", sha256_hex(state.as_bytes())).as_str()) + ); + assert!(!out.state_observations.locked); + assert!(out.state_observations.lock_id.is_none()); + assert!(out.state_observations.lock_acquired); + assert!(out.state_observations.acquired_lock_id.is_some()); + assert!( + !dir.path().join(CLUSTER_LOCK_FILE).exists(), + "plan must release lock before returning" + ); + } + + #[tokio::test] + async fn existing_lock_makes_plan_fail() { + let dir = fixture(); + let state_dir = dir.path().join(CLUSTER_STATE_DIR); + fs::create_dir_all(&state_dir).unwrap(); + fs::write( + state_dir.join("lock.json"), + r#"{ + "version": 1, + "lock_id": "held-lock", + "operation": "plan", + "created_at": "2026-06-08T00:00:00Z", + "pid": 123 +}"#, + ) + .unwrap(); + + let out = plan_config_dir(dir.path()).await; + assert!(!out.ok); + assert!(out.state_observations.locked); + assert_eq!(out.state_observations.lock_id.as_deref(), Some("held-lock")); + assert!(!out.state_observations.lock_acquired); + assert!(out.state_observations.acquired_lock_id.is_none()); + assert_eq!( + out.state_observations.lock_operation.as_deref(), + Some("plan") + ); + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "state_lock_held") + ); + assert!(out.diagnostics.iter().any(|diagnostic| { + diagnostic.code == "state_lock_held" + && diagnostic.message.contains("force-unlock held-lock") + })); + } + + #[tokio::test] + async fn state_lock_false_bypasses_lock_with_warning() { + let dir = fixture(); + fs::write( + dir.path().join(CLUSTER_CONFIG_FILE), + r#" +version: 1 +state: + backend: cluster + lock: false +graphs: + knowledge: + schema: ./people.pg +"#, + ) + .unwrap(); + + let out = plan_config_dir(dir.path()).await; + assert!(out.ok, "{:?}", out.diagnostics); + assert!(!out.state_observations.locked); + assert!(!out.state_observations.lock_acquired); + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "state_lock_disabled") + ); + assert!(!dir.path().join(CLUSTER_LOCK_FILE).exists()); + } + + #[test] + fn external_state_backend_rejected() { + let dir = fixture(); + fs::write( + dir.path().join(CLUSTER_CONFIG_FILE), + "version: 1\nstate:\n backend: s3://bucket/state\ngraphs: {}\n", + ) + .unwrap(); + let out = validate_config_dir(dir.path()); + assert!(!out.ok); + assert_eq!(out.diagnostics[0].code, "unsupported_state_backend"); + } + + #[tokio::test] + async fn external_state_backend_plan_rejected() { + let dir = fixture(); + fs::write( + dir.path().join(CLUSTER_CONFIG_FILE), + "version: 1\nstate:\n backend: s3://bucket/state\ngraphs: {}\n", + ) + .unwrap(); + let out = plan_config_dir(dir.path()).await; + assert!(!out.ok); + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "unsupported_state_backend") + ); + } + + #[tokio::test] + async fn import_missing_state_creates_state_with_graph_observation() { + let dir = fixture(); + init_derived_graph(dir.path()).await; + + let out = import_config_dir(dir.path()).await; + assert!(out.ok, "{:?}", out.diagnostics); + assert_eq!(out.state_observations.state_revision, 1); + assert!(out.state_observations.state_cas.is_some()); + assert!(!out.state_observations.locked); + assert!(out.state_observations.lock_acquired); + assert!(out.state_observations.acquired_lock_id.is_some()); + assert!(!dir.path().join(CLUSTER_LOCK_FILE).exists()); + assert_eq!( + out.resource_digests + .get("schema.knowledge") + .map(String::as_str), + Some(sha256_hex(SCHEMA.as_bytes()).as_str()) + ); + assert!(out.observations["graph.knowledge"]["manifest_version"].is_number()); + assert_eq!( + out.observations["graph.knowledge"]["schema_matches_desired"], + true + ); + + let state: serde_json::Value = + serde_json::from_str(&fs::read_to_string(dir.path().join(CLUSTER_STATE_FILE)).unwrap()) + .unwrap(); + assert_eq!(state["state_revision"], 1); + assert_eq!( + state["resource_statuses"]["graph.knowledge"]["status"], + "applied" + ); + } + + #[tokio::test] + async fn import_existing_state_fails() { + let dir = fixture(); + let state_dir = dir.path().join(CLUSTER_STATE_DIR); + fs::create_dir_all(&state_dir).unwrap(); + fs::write( + state_dir.join("state.json"), + r#"{"version":1,"applied_revision":{"resources":{}}}"#, + ) + .unwrap(); + + let out = import_config_dir(dir.path()).await; + assert!(!out.ok); + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "state_already_exists") + ); + } + + #[tokio::test] + async fn refresh_missing_state_fails() { + let dir = fixture(); + let out = refresh_config_dir(dir.path()).await; + assert!(!out.ok); + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "state_missing") + ); + } + + #[tokio::test] + async fn refresh_existing_minimal_state_increments_revision_and_updates_cas() { + let dir = fixture(); + init_derived_graph(dir.path()).await; + let state_dir = dir.path().join(CLUSTER_STATE_DIR); + fs::create_dir_all(&state_dir).unwrap(); + fs::write( + state_dir.join("state.json"), + r#"{"version":1,"applied_revision":{"config_digest":"old","resources":{"graph.knowledge":{"digest":"old"}}}}"#, + ) + .unwrap(); + + let out = refresh_config_dir(dir.path()).await; + assert!(out.ok, "{:?}", out.diagnostics); + assert_eq!(out.state_observations.state_revision, 1); + assert!(out.state_observations.state_cas.is_some()); + assert!(!out.state_observations.locked); + assert!(out.state_observations.lock_acquired); + assert_eq!( + out.resource_statuses["graph.knowledge"].status, + ResourceLifecycleStatus::Applied + ); + assert!(!dir.path().join(CLUSTER_LOCK_FILE).exists()); + } + + #[tokio::test] + async fn refresh_records_live_schema_digest_and_manifest_version() { + let dir = fixture(); + init_derived_graph(dir.path()).await; + let state_dir = dir.path().join(CLUSTER_STATE_DIR); + fs::create_dir_all(&state_dir).unwrap(); + fs::write( + state_dir.join("state.json"), + r#"{"version":1,"state_revision":4,"applied_revision":{"resources":{}}}"#, + ) + .unwrap(); + + let out = refresh_config_dir(dir.path()).await; + assert!(out.ok, "{:?}", out.diagnostics); + assert_eq!(out.state_observations.state_revision, 5); + assert_eq!( + out.observations["graph.knowledge"]["schema_digest"], + sha256_hex(SCHEMA.as_bytes()) + ); + assert!(out.observations["graph.knowledge"]["manifest_version"].is_u64()); + } + + #[tokio::test] + async fn missing_derived_graph_root_marks_drifted_and_plans_creates() { + let dir = fixture(); + let state_dir = dir.path().join(CLUSTER_STATE_DIR); + fs::create_dir_all(&state_dir).unwrap(); + fs::write( + state_dir.join("state.json"), + r#"{"version":1,"applied_revision":{"resources":{"graph.knowledge":{"digest":"old-graph"},"schema.knowledge":{"digest":"old-schema"}}}}"#, + ) + .unwrap(); + + let out = refresh_config_dir(dir.path()).await; + assert!(out.ok, "{:?}", out.diagnostics); + assert_eq!( + out.resource_statuses["graph.knowledge"].status, + ResourceLifecycleStatus::Drifted + ); + assert!(!out.resource_digests.contains_key("graph.knowledge")); + assert_eq!(out.observations["graph.knowledge"]["exists"], false); + + let plan = plan_config_dir(dir.path()).await; + assert!(plan.ok, "{:?}", plan.diagnostics); + assert!(plan.changes.iter().any(|change| { + change.resource == "graph.knowledge" && change.operation == PlanOperation::Create + })); + assert!(plan.changes.iter().any(|change| { + change.resource == "schema.knowledge" && change.operation == PlanOperation::Create + })); + } + + #[tokio::test] + async fn live_schema_mismatch_marks_drifted_and_causes_plan_update() { + let dir = fixture(); + init_derived_graph(dir.path()).await; + fs::write( + dir.path().join("people.pg"), + SCHEMA.replace("age: I32?", "age: I32?\n nickname: String?"), + ) + .unwrap(); + let state_dir = dir.path().join(CLUSTER_STATE_DIR); + fs::create_dir_all(&state_dir).unwrap(); + fs::write( + state_dir.join("state.json"), + r#"{"version":1,"applied_revision":{"resources":{"graph.knowledge":{"digest":"old-graph"},"schema.knowledge":{"digest":"old-schema"}}}}"#, + ) + .unwrap(); + + let out = refresh_config_dir(dir.path()).await; + assert!(out.ok, "{:?}", out.diagnostics); + assert_eq!( + out.resource_statuses["schema.knowledge"].status, + ResourceLifecycleStatus::Drifted + ); + assert_eq!( + out.observations["graph.knowledge"]["schema_matches_desired"], + false + ); + + let plan = plan_config_dir(dir.path()).await; + assert!(plan.ok, "{:?}", plan.diagnostics); + assert!(plan.changes.iter().any(|change| { + change.resource == "schema.knowledge" && change.operation == PlanOperation::Update + })); + } + + #[tokio::test] + async fn existing_lock_makes_refresh_fail() { + let dir = fixture(); + let state_dir = dir.path().join(CLUSTER_STATE_DIR); + fs::create_dir_all(&state_dir).unwrap(); + fs::write( + state_dir.join("state.json"), + r#"{"version":1,"applied_revision":{"resources":{}}}"#, + ) + .unwrap(); + fs::write( + state_dir.join("lock.json"), + r#"{"version":1,"lock_id":"held-lock","operation":"refresh","created_at":"2026-06-08T00:00:00Z","pid":123}"#, + ) + .unwrap(); + + let out = refresh_config_dir(dir.path()).await; + assert!(!out.ok); + assert!(out.state_observations.locked); + assert_eq!(out.state_observations.lock_id.as_deref(), Some("held-lock")); + assert!(!out.state_observations.lock_acquired); + assert_eq!( + out.state_observations.lock_operation.as_deref(), + Some("refresh") + ); + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "state_lock_held") + ); + assert!(out.diagnostics.iter().any(|diagnostic| { + diagnostic.code == "state_lock_held" + && diagnostic.message.contains("force-unlock held-lock") + })); + } + + #[tokio::test] + async fn state_lock_false_bypasses_refresh_lock_with_warning() { + let dir = fixture(); + init_derived_graph(dir.path()).await; + fs::write( + dir.path().join(CLUSTER_CONFIG_FILE), + r#" +version: 1 +state: + backend: cluster + lock: false +graphs: + knowledge: + schema: ./people.pg +"#, + ) + .unwrap(); + let state_dir = dir.path().join(CLUSTER_STATE_DIR); + fs::create_dir_all(&state_dir).unwrap(); + fs::write( + state_dir.join("state.json"), + r#"{"version":1,"applied_revision":{"resources":{}}}"#, + ) + .unwrap(); + + let out = refresh_config_dir(dir.path()).await; + assert!(out.ok, "{:?}", out.diagnostics); + assert!(!out.state_observations.locked); + assert!(!out.state_observations.lock_acquired); + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "state_lock_disabled") + ); + } + + #[tokio::test] + async fn external_state_backend_refresh_rejected() { + let dir = fixture(); + fs::write( + dir.path().join(CLUSTER_CONFIG_FILE), + "version: 1\nstate:\n backend: s3://bucket/state\ngraphs: {}\n", + ) + .unwrap(); + + let out = refresh_config_dir(dir.path()).await; + assert!(!out.ok); + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "unsupported_state_backend") + ); + } + + #[tokio::test] + async fn import_graph_open_error_does_not_create_state() { + let dir = fixture(); + fs::create_dir_all(dir.path().join(CLUSTER_GRAPHS_DIR).join("knowledge.omni")).unwrap(); + + let out = import_config_dir(dir.path()).await; + assert!(!out.ok); + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "graph_observation_error") + ); + assert!(!dir.path().join(CLUSTER_STATE_FILE).exists()); + } + + // ---- config-only apply (Stage 3A) ---- + + /// Seed a state.json that simulates "graph exists with the desired schema, + /// queries/policies not yet applied" by borrowing the desired digests. + fn write_applyable_state(config_dir: &Path) { + let out = validate_config_dir(config_dir); + assert!(out.ok, "{:?}", out.diagnostics); + let schema_digest = out.resource_digests.get("schema.knowledge").unwrap().clone(); + let graph_composite = + graph_digest("knowledge", Some(&schema_digest), Some(&BTreeMap::new())); + write_state_resources( + config_dir, + &[ + ("graph.knowledge", graph_composite.as_str()), + ("schema.knowledge", schema_digest.as_str()), + ], + ); + } + + fn write_state_resources(config_dir: &Path, resources: &[(&str, &str)]) { + let resource_map: serde_json::Map = resources + .iter() + .map(|(address, digest)| ((*address).to_string(), json!({ "digest": digest }))) + .collect(); + let state_dir = config_dir.join(CLUSTER_STATE_DIR); + fs::create_dir_all(&state_dir).unwrap(); + fs::write( + state_dir.join("state.json"), + serde_json::to_string_pretty(&json!({ + "version": 1, + "state_revision": 1, + "applied_revision": { "resources": resource_map } + })) + .unwrap(), + ) + .unwrap(); + } + + fn read_state_json(config_dir: &Path) -> serde_json::Value { + serde_json::from_str(&fs::read_to_string(config_dir.join(CLUSTER_STATE_FILE)).unwrap()) + .unwrap() + } + + fn query_payload_path(config_dir: &Path, digest: &str) -> std::path::PathBuf { + config_dir + .join(CLUSTER_RESOURCES_DIR) + .join("query/knowledge/find_person") + .join(format!("{digest}.gq")) + } + + fn policy_payload_path(config_dir: &Path, digest: &str) -> std::path::PathBuf { + config_dir + .join(CLUSTER_RESOURCES_DIR) + .join("policy/base") + .join(format!("{digest}.yaml")) + } + + #[tokio::test] + async fn apply_without_state_fails_with_state_missing() { + let dir = fixture(); + let out = apply_config_dir(dir.path()).await; + assert!(!out.ok); + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "state_missing" + && diagnostic.message.contains("cluster import")) + ); + assert!(!dir.path().join(CLUSTER_STATE_FILE).exists()); + assert!(!dir.path().join(CLUSTER_RESOURCES_DIR).exists()); + assert!(!dir.path().join(CLUSTER_LOCK_FILE).exists()); + } + + #[tokio::test] + async fn apply_writes_payloads_state_and_statuses() { + let dir = fixture(); + write_applyable_state(dir.path()); + let desired = validate_config_dir(dir.path()); + let query_digest = desired + .resource_digests + .get("query.knowledge.find_person") + .unwrap() + .clone(); + let policy_digest = desired.resource_digests.get("policy.base").unwrap().clone(); + let schema_digest = desired + .resource_digests + .get("schema.knowledge") + .unwrap() + .clone(); + + let out = apply_config_dir(dir.path()).await; + assert!(out.ok, "{:?}", out.diagnostics); + assert_eq!(out.applied_count, 2); + assert_eq!(out.deferred_count, 0); + assert!(out.converged); + assert!(out.state_written); + + let query_blob = query_payload_path(dir.path(), &query_digest); + assert_eq!(fs::read_to_string(&query_blob).unwrap(), QUERY); + let policy_blob = policy_payload_path(dir.path(), &policy_digest); + assert_eq!(fs::read_to_string(&policy_blob).unwrap(), "rules: []\n"); + + let state = read_state_json(dir.path()); + assert_eq!(state["state_revision"], 2); + let resources = &state["applied_revision"]["resources"]; + assert_eq!( + resources["query.knowledge.find_person"]["digest"], + query_digest + ); + assert_eq!(resources["policy.base"]["digest"], policy_digest); + let expected_composite = graph_digest( + "knowledge", + Some(&schema_digest), + Some( + &[("find_person".to_string(), query_digest.clone())] + .into_iter() + .collect(), + ), + ); + assert_eq!(resources["graph.knowledge"]["digest"], expected_composite); + assert_eq!( + state["applied_revision"]["config_digest"], + desired_revision_digest(&out) + ); + assert_eq!( + state["resource_statuses"]["query.knowledge.find_person"]["status"], + "applied" + ); + assert_eq!(state["resource_statuses"]["policy.base"]["status"], "applied"); + assert!(!dir.path().join(CLUSTER_LOCK_FILE).exists()); + } + + fn desired_revision_digest(out: &ApplyOutput) -> String { + out.desired_revision.config_digest.clone().unwrap() + } + + #[tokio::test] + async fn apply_update_changes_query_digest_and_keeps_old_blob() { + let dir = fixture(); + let desired = validate_config_dir(dir.path()); + let schema_digest = desired + .resource_digests + .get("schema.knowledge") + .unwrap() + .clone(); + let old_digest = "0".repeat(64); + let graph_composite = + graph_digest("knowledge", Some(&schema_digest), Some(&BTreeMap::new())); + write_state_resources( + dir.path(), + &[ + ("graph.knowledge", graph_composite.as_str()), + ("schema.knowledge", schema_digest.as_str()), + ("query.knowledge.find_person", old_digest.as_str()), + ], + ); + let old_blob = query_payload_path(dir.path(), &old_digest); + fs::create_dir_all(old_blob.parent().unwrap()).unwrap(); + fs::write(&old_blob, "old query source").unwrap(); + + let out = apply_config_dir(dir.path()).await; + assert!(out.ok, "{:?}", out.diagnostics); + let new_digest = desired + .resource_digests + .get("query.knowledge.find_person") + .unwrap(); + let state = read_state_json(dir.path()); + assert_eq!( + state["applied_revision"]["resources"]["query.knowledge.find_person"]["digest"], + *new_digest + ); + assert_eq!(fs::read_to_string(&old_blob).unwrap(), "old query source"); + assert!(query_payload_path(dir.path(), new_digest).exists()); + } + + #[tokio::test] + async fn apply_deletes_removed_resources_but_keeps_blobs() { + let dir = fixture(); + let desired = validate_config_dir(dir.path()); + let schema_digest = desired + .resource_digests + .get("schema.knowledge") + .unwrap() + .clone(); + let stale_query_digest = "1".repeat(64); + let stale_policy_digest = "2".repeat(64); + let graph_composite = + graph_digest("knowledge", Some(&schema_digest), Some(&BTreeMap::new())); + write_state_resources( + dir.path(), + &[ + ("graph.knowledge", graph_composite.as_str()), + ("schema.knowledge", schema_digest.as_str()), + ("query.knowledge.orphan", stale_query_digest.as_str()), + ("policy.old", stale_policy_digest.as_str()), + ], + ); + let stale_blob = dir + .path() + .join(CLUSTER_RESOURCES_DIR) + .join("policy/old") + .join(format!("{stale_policy_digest}.yaml")); + fs::create_dir_all(stale_blob.parent().unwrap()).unwrap(); + fs::write(&stale_blob, "old policy").unwrap(); + + let out = apply_config_dir(dir.path()).await; + assert!(out.ok, "{:?}", out.diagnostics); + assert!(out.converged); + let state = read_state_json(dir.path()); + let resources = &state["applied_revision"]["resources"]; + assert!(resources.get("query.knowledge.orphan").is_none()); + assert!(resources.get("policy.old").is_none()); + assert!( + state["resource_statuses"] + .get("query.knowledge.orphan") + .is_none() + ); + // Deleted resources leave their content-addressed blobs in place; GC is + // a later stage. + assert_eq!(fs::read_to_string(&stale_blob).unwrap(), "old policy"); + // The composite no longer includes the orphan query. + let query_digest = desired + .resource_digests + .get("query.knowledge.find_person") + .unwrap() + .clone(); + let expected_composite = graph_digest( + "knowledge", + Some(&schema_digest), + Some(&[("find_person".to_string(), query_digest)].into_iter().collect()), + ); + assert_eq!(resources["graph.knowledge"]["digest"], expected_composite); + } + + #[tokio::test] + async fn apply_schema_update_and_dependent_query_in_one_run() { + let dir = fixture(); + init_derived_graph(dir.path()).await; + write_applyable_state(dir.path()); + // Schema update + a query update that depends on the new field: one + // apply executes the schema migration first, then the catalog write. + fs::write(dir.path().join("people.pg"), SCHEMA_V2).unwrap(); + fs::write( + dir.path().join("people.gq"), + "\nquery find_person($name: String) {\n match { $p: Person { name: $name } }\n return { $p.name, $p.bio }\n}\n", + ) + .unwrap(); + + let out = apply_config_dir(dir.path()).await; + assert!(out.ok, "{:?}", out.diagnostics); + assert!(out.converged, "{out:?}"); + let by_resource: BTreeMap<&str, &PlanChange> = out + .changes + .iter() + .map(|change| (change.resource.as_str(), change)) + .collect(); + assert_eq!( + by_resource["schema.knowledge"].disposition, + Some(ApplyDisposition::Applied) + ); + assert_eq!( + by_resource["query.knowledge.find_person"].disposition, + Some(ApplyDisposition::Applied) + ); + assert_eq!( + by_resource["graph.knowledge"].disposition, + Some(ApplyDisposition::Derived) + ); + // The live graph carries the new schema. + let db = Omnigraph::open_read_only(&derived_graph_uri(dir.path(), "knowledge")) + .await + .unwrap(); + let desired = validate_config_dir(dir.path()); + assert_eq!( + sha256_hex(db.schema_source().as_bytes()), + desired.resource_digests["schema.knowledge"] + ); + let state = read_state_json(dir.path()); + assert_eq!( + state["applied_revision"]["resources"]["schema.knowledge"]["digest"], + desired.resource_digests["schema.knowledge"] + ); + // Sidecar retired after the CAS landed. + assert!( + !dir.path().join(CLUSTER_RECOVERIES_DIR).exists() + || fs::read_dir(dir.path().join(CLUSTER_RECOVERIES_DIR)) + .unwrap() + .next() + .is_none() + ); + } + + #[tokio::test] + async fn apply_unsupported_schema_change_fails_loudly() { + let dir = fixture(); + init_derived_graph(dir.path()).await; + write_applyable_state(dir.path()); + // Property type changes are unsupported by the engine planner. + fs::write( + dir.path().join("people.pg"), + "\nnode Person {\n name: String @key\n age: I64?\n}\n", + ) + .unwrap(); + + let out = apply_config_dir(dir.path()).await; + assert!(!out.ok); + assert!(out.diagnostics.iter().any(|diagnostic| { + diagnostic.code == "schema_apply_failed" + && diagnostic.message.contains("changing property type") + })); + let by_resource: BTreeMap<&str, &PlanChange> = out + .changes + .iter() + .map(|change| (change.resource.as_str(), change)) + .collect(); + assert_eq!( + by_resource["schema.knowledge"].disposition, + Some(ApplyDisposition::Blocked) + ); + assert_eq!( + by_resource["schema.knowledge"].reason.as_deref(), + Some("schema_apply_failed") + ); + // The live schema and the ledger are unchanged. + let state = read_state_json(dir.path()); + let desired = validate_config_dir(dir.path()); + assert_ne!( + state["applied_revision"]["resources"]["schema.knowledge"]["digest"], + desired.resource_digests["schema.knowledge"] + ); + // Second run: the sweep retires the stale sidecar (ledger consistent) + // and the run fails just as loudly — idempotent loudness. + let second = apply_config_dir(dir.path()).await; + assert!(!second.ok); + assert!( + second + .diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "schema_apply_failed") + ); + } + + #[tokio::test] + async fn apply_blocks_schema_update_while_recovery_pending() { + let dir = fixture(); + init_derived_graph(dir.path()).await; + write_state_resources(dir.path(), &[("schema.knowledge", "stale-digest")]); + fs::write(dir.path().join("people.pg"), SCHEMA_V2).unwrap(); + // A pending sidecar whose intent matches neither live nor recorded. + write_schema_apply_sidecar(dir.path(), "knowledge", "intended-digest", "01PENDS"); + + let out = apply_config_dir(dir.path()).await; + let by_resource: BTreeMap<&str, &PlanChange> = out + .changes + .iter() + .map(|change| (change.resource.as_str(), change)) + .collect(); + assert_eq!( + by_resource["schema.knowledge"].disposition, + Some(ApplyDisposition::Blocked) + ); + assert_eq!( + by_resource["schema.knowledge"].reason.as_deref(), + Some("cluster_recovery_pending") + ); + } + + #[tokio::test] + async fn apply_creates_graph_and_unblocks_dependents() { + let dir = fixture(); + write_state_resources(dir.path(), &[]); + + let out = apply_config_dir(dir.path()).await; + assert!(out.ok, "{:?}", out.diagnostics); + assert!(out.converged, "{out:?}"); + let by_resource: BTreeMap<&str, &PlanChange> = out + .changes + .iter() + .map(|change| (change.resource.as_str(), change)) + .collect(); + // Stage 4A: the create executes, and its dependents apply in-run. + assert_eq!( + by_resource["graph.knowledge"].disposition, + Some(ApplyDisposition::Applied) + ); + assert_eq!( + by_resource["schema.knowledge"].disposition, + Some(ApplyDisposition::Applied) + ); + assert_eq!( + by_resource["query.knowledge.find_person"].disposition, + Some(ApplyDisposition::Applied) + ); + assert_eq!( + by_resource["policy.base"].disposition, + Some(ApplyDisposition::Applied) + ); + // The graph exists on disk and opens; state records everything. + let graph_uri = derived_graph_uri(dir.path(), "knowledge"); + let db = Omnigraph::open_read_only(&graph_uri).await.unwrap(); + let desired = validate_config_dir(dir.path()); + assert_eq!( + sha256_hex(db.schema_source().as_bytes()), + desired.resource_digests["schema.knowledge"] + ); + let state = read_state_json(dir.path()); + assert_eq!( + state["applied_revision"]["resources"]["schema.knowledge"]["digest"], + desired.resource_digests["schema.knowledge"] + ); + assert_eq!( + state["resource_statuses"]["graph.knowledge"]["status"], + "applied" + ); + // The create's sidecar was retired after the state CAS landed. + assert!( + !dir.path().join(CLUSTER_RECOVERIES_DIR).exists() + || fs::read_dir(dir.path().join(CLUSTER_RECOVERIES_DIR)) + .unwrap() + .next() + .is_none() + ); + } + + #[tokio::test] + async fn apply_create_failure_blocks_dependents_and_keeps_sidecar() { + let dir = fixture(); + write_state_resources(dir.path(), &[]); + // Make the init fail its strict preflight: a junk _schema.pg already + // sits at the derived root (the engine refuses to overwrite it). + let root = dir.path().join(CLUSTER_GRAPHS_DIR).join("knowledge.omni"); + fs::create_dir_all(&root).unwrap(); + fs::write(root.join("_schema.pg"), "junk").unwrap(); + + let out = apply_config_dir(dir.path()).await; + assert!(!out.ok); + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "graph_create_failed") + ); + let by_resource: BTreeMap<&str, &PlanChange> = out + .changes + .iter() + .map(|change| (change.resource.as_str(), change)) + .collect(); + // Dependents are demoted: the run tells the truth about what executed. + assert_eq!( + by_resource["graph.knowledge"].disposition, + Some(ApplyDisposition::Blocked) + ); + assert_eq!( + by_resource["query.knowledge.find_person"].disposition, + Some(ApplyDisposition::Blocked) + ); + assert_eq!( + by_resource["query.knowledge.find_person"].reason.as_deref(), + Some("dependency_not_applied") + ); + assert_eq!( + by_resource["policy.base"].disposition, + Some(ApplyDisposition::Blocked) + ); + assert!(!out.converged); + // The sidecar stays for the sweep to classify next run. + assert!( + fs::read_dir(dir.path().join(CLUSTER_RECOVERIES_DIR)) + .unwrap() + .next() + .is_some() + ); + // No graph digests moved. + let state = read_state_json(dir.path()); + assert!( + state["applied_revision"]["resources"] + .as_object() + .unwrap() + .is_empty() + ); + } + + #[tokio::test] + async fn apply_blocks_graph_delete_without_approval() { + let dir = fixture(); + let desired = validate_config_dir(dir.path()); + let schema_digest = desired + .resource_digests + .get("schema.knowledge") + .unwrap() + .clone(); + let graph_composite = + graph_digest("knowledge", Some(&schema_digest), Some(&BTreeMap::new())); + write_state_resources( + dir.path(), + &[ + ("graph.knowledge", graph_composite.as_str()), + ("schema.knowledge", schema_digest.as_str()), + ("graph.old", "3333"), + ("schema.old", "4444"), + ("query.old.q", "5555"), + ], + ); + + let out = apply_config_dir(dir.path()).await; + assert!(out.ok, "{:?}", out.diagnostics); + assert!(!out.converged); + let by_resource: BTreeMap<&str, &PlanChange> = out + .changes + .iter() + .map(|change| (change.resource.as_str(), change)) + .collect(); + // Stage 4C: deletes are gated, not deferred — every subtree change + // blocks on the single graph-level approval. + assert_eq!( + by_resource["graph.old"].disposition, + Some(ApplyDisposition::Blocked) + ); + assert_eq!( + by_resource["graph.old"].reason.as_deref(), + Some("approval_required") + ); + assert_eq!( + by_resource["schema.old"].reason.as_deref(), + Some("approval_required") + ); + assert_eq!( + by_resource["query.old.q"].reason.as_deref(), + Some("approval_required") + ); + // State intact; nothing destroyed without the artifact. + let state = read_state_json(dir.path()); + let resources = &state["applied_revision"]["resources"]; + assert_eq!(resources["graph.old"]["digest"], "3333"); + assert_eq!(resources["schema.old"]["digest"], "4444"); + assert_eq!(resources["query.old.q"]["digest"], "5555"); + } + + #[tokio::test] + async fn approve_writes_digest_bound_artifact() { + let dir = fixture(); + write_applyable_state(dir.path()); + // Seed a deletable subtree. + let state = read_state_json(dir.path()); + let graph_digest_str = state["applied_revision"]["resources"]["graph.knowledge"]["digest"] + .as_str() + .unwrap() + .to_string(); + let schema_digest_str = state["applied_revision"]["resources"]["schema.knowledge"] + ["digest"] + .as_str() + .unwrap() + .to_string(); + write_state_resources( + dir.path(), + &[ + ("graph.knowledge", graph_digest_str.as_str()), + ("schema.knowledge", schema_digest_str.as_str()), + ("graph.old", "3333"), + ("schema.old", "4444"), + ], + ); + + let out = approve_config_dir(dir.path(), "graph.old", "andrew").await; + assert!(out.ok, "{:?}", out.diagnostics); + let approval_id = out.approval_id.clone().unwrap(); + let artifact: serde_json::Value = serde_json::from_str( + &fs::read_to_string( + dir.path() + .join(CLUSTER_APPROVALS_DIR) + .join(format!("{approval_id}.json")), + ) + .unwrap(), + ) + .unwrap(); + assert_eq!(artifact["resource"], "graph.old"); + assert_eq!(artifact["operation"], "delete"); + assert_eq!(artifact["approved_by"], "andrew"); + assert_eq!(artifact["bound_before_digest"], "3333"); + assert!(artifact["bound_after_digest"].is_null()); + assert!(artifact["bound_config_digest"].is_string()); + assert!(artifact["consumed_at"].is_null()); + + // A non-gated address is refused. + let not_gated = approve_config_dir(dir.path(), "query.knowledge.find_person", "andrew").await; + assert!(!not_gated.ok); + assert!( + not_gated + .diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "approval_not_required") + ); + } + + #[tokio::test] + async fn stale_approval_is_ignored() { + let dir = fixture(); + write_applyable_state(dir.path()); + let state = read_state_json(dir.path()); + let graph_digest_str = state["applied_revision"]["resources"]["graph.knowledge"]["digest"] + .as_str() + .unwrap() + .to_string(); + let schema_digest_str = state["applied_revision"]["resources"]["schema.knowledge"] + ["digest"] + .as_str() + .unwrap() + .to_string(); + write_state_resources( + dir.path(), + &[ + ("graph.knowledge", graph_digest_str.as_str()), + ("schema.knowledge", schema_digest_str.as_str()), + ("graph.old", "3333"), + ], + ); + let approved = approve_config_dir(dir.path(), "graph.old", "andrew").await; + assert!(approved.ok, "{:?}", approved.diagnostics); + // The config moves after approval: the bound config digest no longer + // matches and the artifact authorizes nothing. + fs::write(dir.path().join("base.policy.yaml"), "rules: [] # moved\n").unwrap(); + + let out = apply_config_dir(dir.path()).await; + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "approval_stale"), + "{:?}", + out.diagnostics + ); + let by_resource: BTreeMap<&str, &PlanChange> = out + .changes + .iter() + .map(|change| (change.resource.as_str(), change)) + .collect(); + assert_eq!( + by_resource["graph.old"].reason.as_deref(), + Some("approval_required") + ); + let state = read_state_json(dir.path()); + assert_eq!( + state["applied_revision"]["resources"]["graph.old"]["digest"], + "3333" + ); + } + + #[tokio::test] + async fn compute_approvals_one_gate_per_subtree() { + let dir = fixture(); + write_applyable_state(dir.path()); + let state = read_state_json(dir.path()); + let g = state["applied_revision"]["resources"]["graph.knowledge"]["digest"] + .as_str() + .unwrap() + .to_string(); + let sc = state["applied_revision"]["resources"]["schema.knowledge"]["digest"] + .as_str() + .unwrap() + .to_string(); + write_state_resources( + dir.path(), + &[ + ("graph.knowledge", g.as_str()), + ("schema.knowledge", sc.as_str()), + ("graph.old", "3333"), + ("schema.old", "4444"), + ("query.old.q", "5555"), + ], + ); + let plan = plan_config_dir(dir.path()).await; + let gated: Vec<&str> = plan + .approvals_required + .iter() + .map(|gate| gate.resource.as_str()) + .collect(); + assert_eq!(gated, vec!["graph.old"], "{plan:?}"); + assert!(!plan.approvals_required[0].satisfied); + } + + #[tokio::test] + async fn apply_is_idempotent() { + let dir = fixture(); + write_applyable_state(dir.path()); + + let first = apply_config_dir(dir.path()).await; + assert!(first.ok, "{:?}", first.diagnostics); + assert!(first.state_written); + let state_after_first = fs::read_to_string(dir.path().join(CLUSTER_STATE_FILE)).unwrap(); + + let second = apply_config_dir(dir.path()).await; + assert!(second.ok, "{:?}", second.diagnostics); + assert!(second.changes.is_empty()); + assert_eq!(second.applied_count, 0); + assert!(second.converged); + assert!(!second.state_written); + let state_after_second = fs::read_to_string(dir.path().join(CLUSTER_STATE_FILE)).unwrap(); + assert_eq!(state_after_first, state_after_second); + assert_eq!(second.state_observations.state_revision, 2); + } + + #[tokio::test] + async fn apply_respects_held_lock() { + let dir = fixture(); + write_applyable_state(dir.path()); + write_lock_file(dir.path(), "held-lock", "plan"); + + let out = apply_config_dir(dir.path()).await; + assert!(!out.ok); + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "state_lock_held") + ); + // The held lock survives a refused apply, and nothing was written. + assert!(dir.path().join(CLUSTER_LOCK_FILE).exists()); + assert!(!dir.path().join(CLUSTER_RESOURCES_DIR).exists()); + let state = read_state_json(dir.path()); + assert_eq!(state["state_revision"], 1); + } + + #[tokio::test] + async fn apply_state_lock_false_bypasses_with_warning() { + let dir = fixture(); + fs::write( + dir.path().join(CLUSTER_CONFIG_FILE), + r#" +version: 1 +state: + backend: cluster + lock: false +graphs: + knowledge: + schema: ./people.pg + queries: + find_person: + file: ./people.gq +"#, + ) + .unwrap(); + write_applyable_state(dir.path()); + + let out = apply_config_dir(dir.path()).await; + assert!(out.ok, "{:?}", out.diagnostics); + assert!(out.state_written); + assert!(!out.state_observations.lock_acquired); + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "state_lock_disabled") + ); + assert!(!dir.path().join(CLUSTER_LOCK_FILE).exists()); + } + + #[tokio::test] + async fn apply_skips_existing_payload_blob() { + let dir = fixture(); + write_applyable_state(dir.path()); + let desired = validate_config_dir(dir.path()); + let query_digest = desired + .resource_digests + .get("query.knowledge.find_person") + .unwrap() + .clone(); + // Content-addressed blobs are trusted by name: an existing file is + // never rewritten. + let blob = query_payload_path(dir.path(), &query_digest); + fs::create_dir_all(blob.parent().unwrap()).unwrap(); + fs::write(&blob, "pre-existing").unwrap(); + + let out = apply_config_dir(dir.path()).await; + assert!(out.ok, "{:?}", out.diagnostics); + assert_eq!(fs::read_to_string(&blob).unwrap(), "pre-existing"); + } + + #[tokio::test] + async fn apply_invalid_config_fails_before_lock() { + let dir = fixture(); + fs::write( + dir.path().join(CLUSTER_CONFIG_FILE), + "version: 1\nnot_a_field: true\n", + ) + .unwrap(); + + let out = apply_config_dir(dir.path()).await; + assert!(!out.ok); + // Config errors bail before the lock or any state directory exists. + assert!(!dir.path().join(CLUSTER_STATE_DIR).exists()); + } + + /// When the state write fails after payloads landed, the output must + /// report the statuses actually on disk — not the unpersisted in-memory + /// mutations (phantom `applied` entries would mislead automation that + /// reads `resource_statuses` independently of `ok`). + #[cfg(unix)] + #[tokio::test] + async fn apply_state_write_failure_reports_persisted_statuses() { + use std::os::unix::fs::PermissionsExt; + + let dir = fixture(); + // lock: false so the only write into __cluster/ is state.json itself. + fs::write( + dir.path().join(CLUSTER_CONFIG_FILE), + r#" +version: 1 +state: + backend: cluster + lock: false +graphs: + knowledge: + schema: ./people.pg + queries: + find_person: + file: ./people.gq +"#, + ) + .unwrap(); + write_applyable_state(dir.path()); + // Pre-create the payload blob so the payload phase is a no-op and the + // failure lands exactly at the state write. + let desired = validate_config_dir(dir.path()); + let query_digest = desired + .resource_digests + .get("query.knowledge.find_person") + .unwrap(); + let blob = query_payload_path(dir.path(), query_digest); + fs::create_dir_all(blob.parent().unwrap()).unwrap(); + fs::write(&blob, QUERY).unwrap(); + + let state_dir = dir.path().join(CLUSTER_STATE_DIR); + fs::set_permissions(&state_dir, fs::Permissions::from_mode(0o555)).unwrap(); + // Running as root ignores permission bits; skip rather than flake. + if fs::write(state_dir.join("probe"), b"x").is_ok() { + let _ = fs::remove_file(state_dir.join("probe")); + fs::set_permissions(&state_dir, fs::Permissions::from_mode(0o755)).unwrap(); + eprintln!("skipping: permissions are not enforced (running as root)"); + return; + } + + let out = apply_config_dir(dir.path()).await; + fs::set_permissions(&state_dir, fs::Permissions::from_mode(0o755)).unwrap(); + + assert!(!out.ok); + assert!(!out.state_written); + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "state_write_error"), + "{:?}", + out.diagnostics + ); + // The seeded state has no statuses; the failed apply must not invent + // the in-memory `applied` ones it failed to persist. + assert!( + out.resource_statuses.is_empty(), + "unpersisted statuses leaked into output: {:?}", + out.resource_statuses + ); + } + + // ---- catalog payload verification (Stage 3B) ---- + + /// Converge a fixture dir and return the query blob path. + async fn converge_fixture(config_dir: &Path) -> std::path::PathBuf { + write_applyable_state(config_dir); + let out = apply_config_dir(config_dir).await; + assert!(out.ok && out.converged, "{:?}", out.diagnostics); + let desired = validate_config_dir(config_dir); + query_payload_path( + config_dir, + desired + .resource_digests + .get("query.knowledge.find_person") + .unwrap(), + ) + } + + #[tokio::test] + async fn status_reports_missing_payload_read_only() { + let dir = fixture(); + let blob = converge_fixture(dir.path()).await; + let state_before = fs::read_to_string(dir.path().join(CLUSTER_STATE_FILE)).unwrap(); + fs::remove_file(&blob).unwrap(); + + let out = status_config_dir(dir.path()); + assert!(out.ok, "{:?}", out.diagnostics); + assert!(out.diagnostics.iter().any(|diagnostic| { + diagnostic.code == "catalog_payload_missing" + && diagnostic.path == "query.knowledge.find_person" + })); + // Read-only: persisted statuses and state bytes untouched. + assert_eq!( + out.resource_statuses["query.knowledge.find_person"].status, + ResourceLifecycleStatus::Applied + ); + assert_eq!( + fs::read_to_string(dir.path().join(CLUSTER_STATE_FILE)).unwrap(), + state_before + ); + } + + #[tokio::test] + async fn refresh_removes_digest_and_drifts_on_missing_payload() { + let dir = fixture(); + init_derived_graph(dir.path()).await; + let blob = converge_fixture(dir.path()).await; + fs::remove_file(&blob).unwrap(); + + let out = refresh_config_dir(dir.path()).await; + assert!(out.ok, "{:?}", out.diagnostics); + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "catalog_payload_missing") + ); + let status = &out.resource_statuses["query.knowledge.find_person"]; + assert_eq!(status.status, ResourceLifecycleStatus::Drifted); + assert!(status.conditions.contains(&"payload_missing".to_string())); + let state = read_state_json(dir.path()); + assert!( + state["applied_revision"]["resources"] + .get("query.knowledge.find_person") + .is_none(), + "{state}" + ); + } + + #[tokio::test] + async fn refresh_drifts_on_corrupted_payload() { + let dir = fixture(); + init_derived_graph(dir.path()).await; + let blob = converge_fixture(dir.path()).await; + fs::write(&blob, "corrupted content").unwrap(); + + let out = refresh_config_dir(dir.path()).await; + assert!(out.ok, "{:?}", out.diagnostics); + let status = &out.resource_statuses["query.knowledge.find_person"]; + assert_eq!(status.status, ResourceLifecycleStatus::Drifted); + assert!(status.conditions.contains(&"payload_mismatch".to_string())); + let state = read_state_json(dir.path()); + assert!( + state["applied_revision"]["resources"] + .get("query.knowledge.find_person") + .is_none() + ); + } + + #[tokio::test] + async fn refresh_flags_unreadable_payload_as_error() { + let dir = fixture(); + init_derived_graph(dir.path()).await; + let blob = converge_fixture(dir.path()).await; + // A same-named directory yields a non-NotFound IO error portably. + fs::remove_file(&blob).unwrap(); + fs::create_dir(&blob).unwrap(); + + let out = refresh_config_dir(dir.path()).await; + assert!(!out.ok); + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "catalog_payload_read_error") + ); + let status = &out.resource_statuses["query.knowledge.find_person"]; + assert_eq!(status.status, ResourceLifecycleStatus::Error); + assert!(status.conditions.contains(&"payload_read_error".to_string())); + // Transient IO keeps the digest: no spurious republish. + let state = read_state_json(dir.path()); + assert!( + state["applied_revision"]["resources"] + .get("query.knowledge.find_person") + .is_some() + ); + } + + #[tokio::test] + async fn payload_drift_self_heals_through_refresh_plan_apply() { + let dir = fixture(); + init_derived_graph(dir.path()).await; + let blob = converge_fixture(dir.path()).await; + let original = fs::read_to_string(&blob).unwrap(); + fs::remove_file(&blob).unwrap(); + + let refresh = refresh_config_dir(dir.path()).await; + assert!(refresh.ok, "{:?}", refresh.diagnostics); + + let plan = plan_config_dir(dir.path()).await; + let query_change = plan + .changes + .iter() + .find(|change| change.resource == "query.knowledge.find_person") + .expect("plan must propose recreating the query"); + assert_eq!(query_change.operation, PlanOperation::Create); + assert_eq!(query_change.disposition, Some(ApplyDisposition::Applied)); + + let apply = apply_config_dir(dir.path()).await; + assert!(apply.ok && apply.converged, "{:?}", apply.diagnostics); + assert_eq!(fs::read_to_string(&blob).unwrap(), original); + + let status = status_config_dir(dir.path()); + assert!( + !status + .diagnostics + .iter() + .any(|diagnostic| diagnostic.code.starts_with("catalog_payload")), + "{:?}", + status.diagnostics + ); + } + + #[test] + fn verification_skips_graph_and_schema_resources() { + let dir = fixture(); + write_applyable_state(dir.path()); // graph + schema digests only, no blobs + + let out = status_config_dir(dir.path()); + assert!( + !out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code.starts_with("catalog_payload")), + "{:?}", + out.diagnostics + ); + } + + // ---- recovery sidecars + sweep (Stage 4A) ---- + + fn derived_graph_uri(config_dir: &Path, graph_id: &str) -> String { + display_path( + &config_dir + .join(CLUSTER_GRAPHS_DIR) + .join(format!("{graph_id}.omni")), + ) + } + + fn write_create_sidecar( + config_dir: &Path, + graph_id: &str, + desired_schema_digest: &str, + operation_id: &str, + ) -> PathBuf { + let dir = config_dir.join(CLUSTER_RECOVERIES_DIR); + fs::create_dir_all(&dir).unwrap(); + let path = dir.join(format!("{operation_id}.json")); + fs::write( + &path, + serde_json::to_string_pretty(&json!({ + "schema_version": 1, + "operation_id": operation_id, + "started_at": "1970-01-01T00:00:00Z", + "kind": "graph_create", + "graph_id": graph_id, + "graph_uri": derived_graph_uri(config_dir, graph_id), + "desired_schema_digest": desired_schema_digest, + })) + .unwrap(), + ) + .unwrap(); + path + } + + #[tokio::test] + async fn sweep_removes_sidecar_when_root_absent() { + let dir = fixture(); + write_applyable_state(dir.path()); + let sidecar = write_create_sidecar(dir.path(), "knowledge", "irrelevant", "01ROW1"); + + let out = apply_config_dir(dir.path()).await; + assert!(out.ok, "{:?}", out.diagnostics); + // Row 1: nothing moved; intent removed, run proceeds normally. + assert!(!sidecar.exists()); + assert!(out.converged); + } + + #[tokio::test] + async fn sweep_rolls_forward_completed_create() { + let dir = fixture(); + init_derived_graph(dir.path()).await; + write_state_resources(dir.path(), &[]); // state predates the create + let desired = validate_config_dir(dir.path()); + let schema_digest = desired.resource_digests["schema.knowledge"].clone(); + let sidecar = write_create_sidecar(dir.path(), "knowledge", &schema_digest, "01ROW4"); + + let out = apply_config_dir(dir.path()).await; + assert!(out.ok, "{:?}", out.diagnostics); + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "cluster_recovery_rolled_forward") + ); + // Row 4: ledger converged to observable reality, audit recorded, + // sidecar retired after the CAS landed. + let state = read_state_json(dir.path()); + assert_eq!( + state["applied_revision"]["resources"]["schema.knowledge"]["digest"], + schema_digest + ); + assert!( + state["recovery_records"] + .as_object() + .unwrap() + .values() + .any(|record| record["outcome"] == "rolled_forward" + && record["graph_id"] == "knowledge") + ); + assert!(!sidecar.exists()); + // With the graph rolled forward, the same run converges the catalog. + assert!(out.converged, "{out:?}"); + } + + #[tokio::test] + async fn sweep_completes_already_recorded_create() { + let dir = fixture(); + init_derived_graph(dir.path()).await; + write_applyable_state(dir.path()); // state already records graph+schema + let desired = validate_config_dir(dir.path()); + let sidecar = write_create_sidecar( + dir.path(), + "knowledge", + &desired.resource_digests["schema.knowledge"], + "01ROW2", + ); + + let out = apply_config_dir(dir.path()).await; + assert!(out.ok, "{:?}", out.diagnostics); + // Row 2: outcome was already durable; no audit entry, sidecar retired. + assert!(!sidecar.exists()); + let state = read_state_json(dir.path()); + assert!( + state["recovery_records"] + .as_object() + .is_none_or(|records| records.is_empty()), + "{state}" + ); + } + + #[tokio::test] + async fn sweep_keeps_sidecar_for_incomplete_root() { + let dir = fixture(); + write_applyable_state(dir.path()); + // A root that exists but cannot be opened: the engine's partial-init gap. + let root = dir.path().join(CLUSTER_GRAPHS_DIR).join("knowledge.omni"); + fs::create_dir_all(&root).unwrap(); + fs::write(root.join("_schema.pg"), "junk").unwrap(); + let sidecar = write_create_sidecar(dir.path(), "knowledge", "whatever", "01ROW5"); + + let out = apply_config_dir(dir.path()).await; + assert!(!out.ok); + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "graph_create_incomplete") + ); + // Row 5: never auto-delete; sidecar and root stay for the operator, + // and the Error status is persisted by the run's state write. + assert!(sidecar.exists()); + assert!(root.exists()); + let state = read_state_json(dir.path()); + assert_eq!(state["resource_statuses"]["graph.knowledge"]["status"], "error"); + assert!( + state["resource_statuses"]["graph.knowledge"]["conditions"] + .as_array() + .unwrap() + .iter() + .any(|condition| condition == "graph_create_incomplete") + ); + } + + #[tokio::test] + async fn sweep_flags_unexpected_schema_as_pending() { + let dir = fixture(); + write_state_resources(dir.path(), &[]); + // Live graph exists with a schema the sidecar never intended. + let graph_dir = dir.path().join(CLUSTER_GRAPHS_DIR); + fs::create_dir_all(&graph_dir).unwrap(); + Omnigraph::init( + &derived_graph_uri(dir.path(), "knowledge"), + "\nnode Other {\n name: String @key\n}\n", + ) + .await + .unwrap(); + let desired = validate_config_dir(dir.path()); + let sidecar = write_create_sidecar( + dir.path(), + "knowledge", + &desired.resource_digests["schema.knowledge"], + "01ROW6", + ); + + let out = apply_config_dir(dir.path()).await; + assert!(out.ok, "{:?}", out.diagnostics); // warning, not error + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "cluster_recovery_pending") + ); + // Row 6: refuse to guess; sidecar kept, Drifted persisted. + assert!(sidecar.exists()); + let state = read_state_json(dir.path()); + assert_eq!( + state["resource_statuses"]["graph.knowledge"]["status"], + "drifted" + ); + assert!( + state["resource_statuses"]["graph.knowledge"]["conditions"] + .as_array() + .unwrap() + .iter() + .any(|condition| condition == "actual_applied_state_pending") + ); + } + + #[tokio::test] + async fn apply_blocks_create_while_recovery_pending() { + let dir = fixture(); + write_state_resources(dir.path(), &[]); + // A kept (row 5) sidecar: partial root that cannot be opened. + let root = dir.path().join(CLUSTER_GRAPHS_DIR).join("knowledge.omni"); + fs::create_dir_all(&root).unwrap(); + fs::write(root.join("_schema.pg"), "junk").unwrap(); + let sidecar = write_create_sidecar(dir.path(), "knowledge", "whatever", "01PEND"); + + let out = apply_config_dir(dir.path()).await; + assert!(!out.ok); // row 5 is an error condition + let by_resource: BTreeMap<&str, &PlanChange> = out + .changes + .iter() + .map(|change| (change.resource.as_str(), change)) + .collect(); + // The pending recovery blocks the create and its dependents; the + // executor never attempts the init. + assert_eq!( + by_resource["graph.knowledge"].disposition, + Some(ApplyDisposition::Blocked) + ); + assert_eq!( + by_resource["graph.knowledge"].reason.as_deref(), + Some("cluster_recovery_pending") + ); + assert_eq!( + by_resource["query.knowledge.find_person"].reason.as_deref(), + Some("cluster_recovery_pending") + ); + assert_eq!( + by_resource["policy.base"].reason.as_deref(), + Some("cluster_recovery_pending") + ); + assert!(sidecar.exists()); + // The sweep's Error status is what persists — not a generic Blocked. + let state = read_state_json(dir.path()); + assert_eq!(state["resource_statuses"]["graph.knowledge"]["status"], "error"); + } + + #[tokio::test] + async fn plan_embeds_migration_preview_for_schema_update() { + let dir = fixture(); + init_derived_graph(dir.path()).await; + write_applyable_state(dir.path()); + fs::write( + dir.path().join("people.pg"), + "\nnode Person {\n name: String @key\n age: I32?\n bio: String?\n}\n", + ) + .unwrap(); + + let out = plan_config_dir(dir.path()).await; + assert!(out.ok, "{:?}", out.diagnostics); + let schema_change = out + .changes + .iter() + .find(|change| change.resource == "schema.knowledge") + .unwrap(); + let migration = schema_change.migration.as_ref().expect("preview embedded"); + assert!(migration.supported); + assert!( + serde_json::to_string(&migration.steps) + .unwrap() + .contains("add_property"), + "{migration:?}" + ); + } + + #[tokio::test] + async fn plan_warns_when_preview_unavailable() { + let dir = fixture(); + write_applyable_state(dir.path()); // digests recorded, but no live root + fs::write( + dir.path().join("people.pg"), + "\nnode Person {\n name: String @key\n age: I32?\n bio: String?\n}\n", + ) + .unwrap(); + + let out = plan_config_dir(dir.path()).await; + assert!(out.ok, "{:?}", out.diagnostics); + let schema_change = out + .changes + .iter() + .find(|change| change.resource == "schema.knowledge") + .unwrap(); + assert!(schema_change.migration.is_none()); + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "schema_preview_unavailable") + ); + } + + fn write_schema_apply_sidecar( + config_dir: &Path, + graph_id: &str, + desired_schema_digest: &str, + operation_id: &str, + ) -> PathBuf { + let dir = config_dir.join(CLUSTER_RECOVERIES_DIR); + fs::create_dir_all(&dir).unwrap(); + let path = dir.join(format!("{operation_id}.json")); + fs::write( + &path, + serde_json::to_string_pretty(&json!({ + "schema_version": 1, + "operation_id": operation_id, + "started_at": "1970-01-01T00:00:00Z", + "kind": "schema_apply", + "graph_id": graph_id, + "graph_uri": derived_graph_uri(config_dir, graph_id), + "desired_schema_digest": desired_schema_digest, + })) + .unwrap(), + ) + .unwrap(); + path + } + + const SCHEMA_V2: &str = "\nnode Person {\n name: String @key\n age: I32?\n bio: String?\n}\n"; + + #[tokio::test] + async fn sweep_retires_schema_sidecar_when_ledger_consistent() { + let dir = fixture(); + init_derived_graph(dir.path()).await; + write_applyable_state(dir.path()); // state digest == live digest + let sidecar = + write_schema_apply_sidecar(dir.path(), "knowledge", "never-applied", "01SROW1"); + + let out = apply_config_dir(dir.path()).await; + assert!(out.ok, "{:?}", out.diagnostics); + assert!(!sidecar.exists()); + let state = read_state_json(dir.path()); + assert!( + state["recovery_records"] + .as_object() + .is_none_or(|records| records.is_empty()) + ); + } + + #[tokio::test] + async fn sweep_rolls_forward_completed_schema_apply() { + let dir = fixture(); + init_derived_graph(dir.path()).await; + write_applyable_state(dir.path()); + // The schema apply completed on the graph out-of-process... + let graph_uri = derived_graph_uri(dir.path(), "knowledge"); + let db = Omnigraph::open(&graph_uri).await.unwrap(); + db.apply_schema(SCHEMA_V2).await.unwrap(); + // ...the desired config matches it, and the sidecar records the intent. + fs::write(dir.path().join("people.pg"), SCHEMA_V2).unwrap(); + let desired = validate_config_dir(dir.path()); + let v2_digest = desired.resource_digests["schema.knowledge"].clone(); + let sidecar = write_schema_apply_sidecar(dir.path(), "knowledge", &v2_digest, "01SROW3"); + + let out = apply_config_dir(dir.path()).await; + assert!(out.ok, "{:?}", out.diagnostics); + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "cluster_recovery_rolled_forward") + ); + assert!(!sidecar.exists()); + let state = read_state_json(dir.path()); + assert_eq!( + state["applied_revision"]["resources"]["schema.knowledge"]["digest"], + v2_digest + ); + assert!( + state["recovery_records"] + .as_object() + .unwrap() + .values() + .any(|record| record["kind"] == "schema_apply" + && record["outcome"] == "rolled_forward") + ); + assert!(out.converged, "{out:?}"); + } + + #[tokio::test] + async fn sweep_flags_unexpected_schema_apply_state_as_pending() { + let dir = fixture(); + init_derived_graph(dir.path()).await; // live = v1 + write_state_resources(dir.path(), &[("schema.knowledge", "stale-digest")]); + // Sidecar intended a digest that is neither live nor recorded. + let sidecar = + write_schema_apply_sidecar(dir.path(), "knowledge", "intended-digest", "01SROW6"); + + let out = apply_config_dir(dir.path()).await; + assert!(out.ok, "{:?}", out.diagnostics); // warnings only + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "cluster_recovery_pending") + ); + assert!(sidecar.exists()); + let state = read_state_json(dir.path()); + assert_eq!( + state["resource_statuses"]["schema.knowledge"]["status"], + "drifted" + ); + } + + #[tokio::test] + async fn sweep_keeps_schema_sidecar_for_unopenable_root() { + let dir = fixture(); + write_applyable_state(dir.path()); + let root = dir.path().join(CLUSTER_GRAPHS_DIR).join("knowledge.omni"); + fs::create_dir_all(&root).unwrap(); // exists, won't open + let sidecar = + write_schema_apply_sidecar(dir.path(), "knowledge", "whatever", "01SROWX"); + + let out = apply_config_dir(dir.path()).await; + assert!(out.ok, "{:?}", out.diagnostics); // warning: cannot verify + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "cluster_recovery_pending") + ); + assert!(sidecar.exists()); + } + + /// Seed: converged knowledge subtree + a stale `old` graph subtree with a + /// real directory on disk. + fn seed_deletable_state(config_dir: &Path) { + write_applyable_state(config_dir); + let state = read_state_json(config_dir); + let g = state["applied_revision"]["resources"]["graph.knowledge"]["digest"] + .as_str() + .unwrap() + .to_string(); + let sc = state["applied_revision"]["resources"]["schema.knowledge"]["digest"] + .as_str() + .unwrap() + .to_string(); + write_state_resources( + config_dir, + &[ + ("graph.knowledge", g.as_str()), + ("schema.knowledge", sc.as_str()), + ("graph.old", "3333"), + ("schema.old", "4444"), + ("query.old.q", "5555"), + ], + ); + let root = config_dir.join(CLUSTER_GRAPHS_DIR).join("old.omni"); + fs::create_dir_all(&root).unwrap(); + fs::write(root.join("_schema.pg"), "stale").unwrap(); + } + + #[tokio::test] + async fn apply_executes_approved_graph_delete() { + let dir = fixture(); + seed_deletable_state(dir.path()); + let approved = approve_config_dir(dir.path(), "graph.old", "andrew").await; + assert!(approved.ok, "{:?}", approved.diagnostics); + let approval_id = approved.approval_id.clone().unwrap(); + + let out = apply_config_dir(dir.path()).await; + assert!(out.ok, "{:?}", out.diagnostics); + assert!(out.converged, "{out:?}"); + let by_resource: BTreeMap<&str, &PlanChange> = out + .changes + .iter() + .map(|change| (change.resource.as_str(), change)) + .collect(); + assert_eq!(by_resource["graph.old"].disposition, Some(ApplyDisposition::Applied)); + assert_eq!(by_resource["schema.old"].disposition, Some(ApplyDisposition::Applied)); + assert_eq!(by_resource["query.old.q"].disposition, Some(ApplyDisposition::Applied)); + // The root is gone; the subtree is tombstoned out of the ledger. + assert!(!dir.path().join(CLUSTER_GRAPHS_DIR).join("old.omni").exists()); + let state = read_state_json(dir.path()); + let resources = state["applied_revision"]["resources"].as_object().unwrap(); + assert!(!resources.contains_key("graph.old")); + assert!(!resources.contains_key("schema.old")); + assert!(!resources.contains_key("query.old.q")); + assert_eq!(state["observations"]["graph.old"]["kind"], "tombstone"); + assert_eq!(state["observations"]["graph.old"]["approval_id"], approval_id); + // Approval consumed in BOTH stores: ledger summary + artifact file. + assert!(state["approval_records"][&approval_id]["consumed_at"].is_string()); + let artifact: serde_json::Value = serde_json::from_str( + &fs::read_to_string( + dir.path() + .join(CLUSTER_APPROVALS_DIR) + .join(format!("{approval_id}.json")), + ) + .unwrap(), + ) + .unwrap(); + assert!(artifact["consumed_at"].is_string(), "{artifact}"); + // Sidecar retired. + assert!( + fs::read_dir(dir.path().join(CLUSTER_RECOVERIES_DIR)) + .map(|mut entries| entries.next().is_none()) + .unwrap_or(true) + ); + // A consumed approval authorizes nothing further (idempotent re-apply). + let again = apply_config_dir(dir.path()).await; + assert!(again.ok && again.converged && !again.state_written, "{again:?}"); + } + + fn write_delete_sidecar( + config_dir: &Path, + graph_id: &str, + approval_id: Option<&str>, + operation_id: &str, + ) -> PathBuf { + let dir = config_dir.join(CLUSTER_RECOVERIES_DIR); + fs::create_dir_all(&dir).unwrap(); + let path = dir.join(format!("{operation_id}.json")); + fs::write( + &path, + serde_json::to_string_pretty(&json!({ + "schema_version": 1, + "operation_id": operation_id, + "started_at": "1970-01-01T00:00:00Z", + "kind": "graph_delete", + "graph_id": graph_id, + "graph_uri": derived_graph_uri(config_dir, graph_id), + "desired_schema_digest": "", + "approval_id": approval_id, + })) + .unwrap(), + ) + .unwrap(); + path + } + + #[tokio::test] + async fn sweep_retires_delete_sidecar_when_tombstoned() { + let dir = fixture(); + write_applyable_state(dir.path()); // no graph.old in state, no root + let sidecar = write_delete_sidecar(dir.path(), "old", None, "01DROW7"); + + let out = apply_config_dir(dir.path()).await; + assert!(out.ok, "{:?}", out.diagnostics); + assert!(!sidecar.exists()); + let state = read_state_json(dir.path()); + assert!( + state["recovery_records"] + .as_object() + .is_none_or(|records| records.is_empty()) + ); + } + + #[tokio::test] + async fn sweep_rolls_forward_completed_delete() { + let dir = fixture(); + seed_deletable_state(dir.path()); + // Approve, then simulate: root removed, state stale, sidecar present. + let approved = approve_config_dir(dir.path(), "graph.old", "andrew").await; + let approval_id = approved.approval_id.unwrap(); + fs::remove_dir_all(dir.path().join(CLUSTER_GRAPHS_DIR).join("old.omni")).unwrap(); + let sidecar = write_delete_sidecar(dir.path(), "old", Some(&approval_id), "01DROW7B"); + + let out = apply_config_dir(dir.path()).await; + assert!(out.ok, "{:?}", out.diagnostics); + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "cluster_recovery_rolled_forward") + ); + assert!(!sidecar.exists()); + let state = read_state_json(dir.path()); + assert!( + !state["applied_revision"]["resources"] + .as_object() + .unwrap() + .contains_key("graph.old") + ); + assert_eq!(state["observations"]["graph.old"]["kind"], "tombstone"); + assert!(state["approval_records"][&approval_id]["consumed_at"].is_string()); + assert!( + state["recovery_records"] + .as_object() + .unwrap() + .values() + .any(|record| record["kind"] == "graph_delete" + && record["outcome"] == "rolled_forward") + ); + // The artifact file is marked consumed post-CAS. + let artifact: serde_json::Value = serde_json::from_str( + &fs::read_to_string( + dir.path() + .join(CLUSTER_APPROVALS_DIR) + .join(format!("{approval_id}.json")), + ) + .unwrap(), + ) + .unwrap(); + assert!(artifact["consumed_at"].is_string()); + assert!(out.converged, "{out:?}"); + } + + #[tokio::test] + async fn sweep_reproposes_incomplete_delete() { + let dir = fixture(); + seed_deletable_state(dir.path()); // root present + let approved = approve_config_dir(dir.path(), "graph.old", "andrew").await; + assert!(approved.ok); + let sidecar = write_delete_sidecar(dir.path(), "old", approved.approval_id.as_deref(), "01DROW8"); + + // Row 8: the stale intent is retired with a warning, and the same run + // re-executes the still-approved delete to completion. + let out = apply_config_dir(dir.path()).await; + assert!(out.ok, "{:?}", out.diagnostics); + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "graph_delete_incomplete") + ); + assert!(!sidecar.exists()); + assert!(!dir.path().join(CLUSTER_GRAPHS_DIR).join("old.omni").exists()); + assert!(out.converged, "{out:?}"); + } + + // ---- policy bindings in the applied revision (5A) ---- + + #[tokio::test] + async fn apply_records_policy_bindings() { + let dir = fixture(); + write_applyable_state(dir.path()); + + let out = apply_config_dir(dir.path()).await; + assert!(out.ok && out.converged, "{:?}", out.diagnostics); + let state = read_state_json(dir.path()); + assert_eq!( + state["applied_revision"]["resources"]["policy.base"]["applies_to"], + serde_json::json!(["graph.knowledge"]), + "{state}" + ); + // Non-policy entries carry no bindings field at all. + assert!( + state["applied_revision"]["resources"]["query.knowledge.find_person"] + .get("applies_to") + .is_none() + ); + } + + #[tokio::test] + async fn binding_change_is_a_visible_plan_change() { + let dir = fixture(); + write_applyable_state(dir.path()); + let converge = apply_config_dir(dir.path()).await; + assert!(converge.converged, "{converge:?}"); + // Edit ONLY applies_to: the policy file digest is unchanged. + fs::write( + dir.path().join(CLUSTER_CONFIG_FILE), + r#" +version: 1 +metadata: + name: test +state: + backend: cluster + lock: true +graphs: + knowledge: + schema: ./people.pg + queries: + find_person: + file: ./people.gq +policies: + base: + file: ./base.policy.yaml + applies_to: [cluster, knowledge] +"#, + ) + .unwrap(); + + let plan = plan_config_dir(dir.path()).await; + let change = plan + .changes + .iter() + .find(|change| change.resource == "policy.base") + .expect("binding change must be visible in plan"); + assert!(change.binding_change); + assert_eq!(change.operation, PlanOperation::Update); + assert_eq!(change.before_digest, change.after_digest); + + let out = apply_config_dir(dir.path()).await; + assert!(out.ok && out.converged, "{out:?}"); + let state = read_state_json(dir.path()); + assert_eq!( + state["applied_revision"]["resources"]["policy.base"]["applies_to"], + serde_json::json!(["cluster", "graph.knowledge"]) + ); + // Idempotent: a second run sees no changes. + let again = apply_config_dir(dir.path()).await; + assert!(again.changes.is_empty() && !again.state_written, "{again:?}"); + } + + #[tokio::test] + async fn pre_5a_state_backfills_bindings() { + let dir = fixture(); + write_applyable_state(dir.path()); + let converge = apply_config_dir(dir.path()).await; + assert!(converge.converged, "{converge:?}"); + // Strip the bindings from the state entry (a pre-5A ledger). + let mut state: serde_json::Value = serde_json::from_str( + &fs::read_to_string(dir.path().join(CLUSTER_STATE_FILE)).unwrap(), + ) + .unwrap(); + state["applied_revision"]["resources"]["policy.base"] + .as_object_mut() + .unwrap() + .remove("applies_to"); + fs::write( + dir.path().join(CLUSTER_STATE_FILE), + serde_json::to_string_pretty(&state).unwrap(), + ) + .unwrap(); + + let plan = plan_config_dir(dir.path()).await; + assert!( + plan.changes + .iter() + .any(|change| change.resource == "policy.base" && change.binding_change), + "{plan:?}" + ); + let out = apply_config_dir(dir.path()).await; + assert!(out.ok && out.converged, "{out:?}"); + let healed = read_state_json(dir.path()); + assert_eq!( + healed["applied_revision"]["resources"]["policy.base"]["applies_to"], + serde_json::json!(["graph.knowledge"]) + ); + } + + #[tokio::test] + async fn bindings_survive_refresh() { + let dir = fixture(); + init_derived_graph(dir.path()).await; + write_applyable_state(dir.path()); + let converge = apply_config_dir(dir.path()).await; + assert!(converge.converged, "{converge:?}"); + + let refresh = refresh_config_dir(dir.path()).await; + assert!(refresh.ok, "{:?}", refresh.diagnostics); + let state = read_state_json(dir.path()); + assert_eq!( + state["applied_revision"]["resources"]["policy.base"]["applies_to"], + serde_json::json!(["graph.knowledge"]) + ); + } + + // ---- serving snapshot (5B read-only loader) ---- + + #[tokio::test] + async fn serving_snapshot_reads_converged_cluster() { + let dir = fixture(); + init_derived_graph(dir.path()).await; + write_applyable_state(dir.path()); + let converge = apply_config_dir(dir.path()).await; + assert!(converge.converged, "{converge:?}"); + + let snapshot = read_serving_snapshot(dir.path()).expect("converged cluster must serve"); + assert_eq!(snapshot.graphs.len(), 1); + assert_eq!(snapshot.graphs[0].graph_id, "knowledge"); + assert!(snapshot.graphs[0].root.ends_with("graphs/knowledge.omni")); + assert_eq!(snapshot.queries.len(), 1); + assert_eq!(snapshot.queries[0].name, "find_person"); + assert!(snapshot.queries[0].source.contains("query find_person")); + assert_eq!(snapshot.policies.len(), 1); + assert_eq!(snapshot.policies[0].applies_to, vec!["graph.knowledge"]); + assert!(snapshot.policies[0].blob_path.exists()); + } + + #[test] + fn serving_snapshot_refuses_missing_state() { + let dir = fixture(); + let err = read_serving_snapshot(dir.path()).unwrap_err(); + assert!( + err.iter().any(|diagnostic| diagnostic.code == "cluster_state_missing"), + "{err:?}" + ); + } + + #[tokio::test] + async fn serving_snapshot_refuses_pending_recovery() { + let dir = fixture(); + init_derived_graph(dir.path()).await; + write_applyable_state(dir.path()); + apply_config_dir(dir.path()).await; + write_schema_apply_sidecar(dir.path(), "knowledge", "whatever", "01SERVE"); + + let err = read_serving_snapshot(dir.path()).unwrap_err(); + assert!( + err.iter().any(|diagnostic| diagnostic.code == "cluster_recovery_pending"), + "{err:?}" + ); + } + + #[tokio::test] + async fn serving_snapshot_refuses_tampered_blob_and_stripped_bindings() { + let dir = fixture(); + init_derived_graph(dir.path()).await; + write_applyable_state(dir.path()); + apply_config_dir(dir.path()).await; + // Tamper with the query blob... + let snapshot = read_serving_snapshot(dir.path()).unwrap(); + let desired = validate_config_dir(dir.path()); + let query_digest = &desired.resource_digests["query.knowledge.find_person"]; + let blob = dir + .path() + .join(CLUSTER_RESOURCES_DIR) + .join("query/knowledge/find_person") + .join(format!("{query_digest}.gq")); + fs::write(&blob, "tampered").unwrap(); + // ...and strip the policy bindings (pre-5A ledger). + let mut state: serde_json::Value = serde_json::from_str( + &fs::read_to_string(dir.path().join(CLUSTER_STATE_FILE)).unwrap(), + ) + .unwrap(); + state["applied_revision"]["resources"]["policy.base"] + .as_object_mut() + .unwrap() + .remove("applies_to"); + fs::write( + dir.path().join(CLUSTER_STATE_FILE), + serde_json::to_string_pretty(&state).unwrap(), + ) + .unwrap(); + + let err = read_serving_snapshot(dir.path()).unwrap_err(); + assert!( + err.iter() + .any(|diagnostic| diagnostic.code == "catalog_payload_digest_mismatch"), + "{err:?}" + ); + assert!( + err.iter().any(|diagnostic| diagnostic.code == "policy_bindings_missing"), + "{err:?}" + ); + let _ = snapshot; // the pre-tamper read succeeded + } + + #[test] + fn serving_snapshot_refuses_empty_cluster() { + let dir = fixture(); + write_state_resources(dir.path(), &[]); // state exists, no graphs + + let err = read_serving_snapshot(dir.path()).unwrap_err(); + assert!( + err.iter().any(|diagnostic| diagnostic.code == "cluster_empty"), + "{err:?}" + ); + } + + // ---- query discovery (Terraform-style declaration) ---- + + #[test] + fn queries_directory_discovers_every_declaration() { + let dir = tempfile::tempdir().unwrap(); + fs::write(dir.path().join("people.pg"), "\nnode Person {\n name: String @key\n}\n").unwrap(); + fs::create_dir(dir.path().join("queries")).unwrap(); + fs::write( + dir.path().join("queries/people.gq"), + "\nquery find_person($name: String) {\n match { $p: Person { name: $name } }\n return { $p.name }\n}\n\nquery all_people() {\n match { $p: Person }\n return { $p.name }\n}\n", + ) + .unwrap(); + fs::write( + dir.path().join("queries/extra.gq"), + "\nquery count_people() {\n match { $p: Person }\n return { count($p) }\n}\n", + ) + .unwrap(); + fs::write(dir.path().join("queries/notes.txt"), "ignored").unwrap(); + fs::write( + dir.path().join("cluster.yaml"), + "version: 1\ngraphs:\n knowledge:\n schema: ./people.pg\n queries: ./queries/\n", + ) + .unwrap(); + + let out = validate_config_dir(dir.path()); + assert!(out.ok, "{:?}", out.diagnostics); + let names: Vec<&str> = out + .resource_digests + .keys() + .filter_map(|address| address.strip_prefix("query.knowledge.")) + .collect(); + assert_eq!(names, vec!["all_people", "count_people", "find_person"]); + } + + #[test] + fn queries_list_and_single_file_forms_discover() { + let dir = tempfile::tempdir().unwrap(); + fs::write(dir.path().join("people.pg"), "\nnode Person {\n name: String @key\n}\n").unwrap(); + fs::write( + dir.path().join("a.gq"), + "\nquery find_person($name: String) {\n match { $p: Person { name: $name } }\n return { $p.name }\n}\n", + ) + .unwrap(); + fs::write( + dir.path().join("b.gq"), + "\nquery all_people() {\n match { $p: Person }\n return { $p.name }\n}\n", + ) + .unwrap(); + fs::write( + dir.path().join("cluster.yaml"), + "version: 1\ngraphs:\n knowledge:\n schema: ./people.pg\n queries: [./a.gq, ./b.gq]\n", + ) + .unwrap(); + let out = validate_config_dir(dir.path()); + assert!(out.ok, "{:?}", out.diagnostics); + assert!(out.resource_digests.contains_key("query.knowledge.find_person")); + assert!(out.resource_digests.contains_key("query.knowledge.all_people")); + + // Single-file string form + fs::write( + dir.path().join("cluster.yaml"), + "version: 1\ngraphs:\n knowledge:\n schema: ./people.pg\n queries: ./a.gq\n", + ) + .unwrap(); + let out = validate_config_dir(dir.path()); + assert!(out.ok, "{:?}", out.diagnostics); + assert!(out.resource_digests.contains_key("query.knowledge.find_person")); + assert!(!out.resource_digests.contains_key("query.knowledge.all_people")); + } + + #[test] + fn query_discovery_rejects_duplicates_and_parse_errors() { + let dir = tempfile::tempdir().unwrap(); + fs::write(dir.path().join("people.pg"), "\nnode Person {\n name: String @key\n}\n").unwrap(); + let decl = "\nquery find_person($name: String) {\n match { $p: Person { name: $name } }\n return { $p.name }\n}\n"; + fs::write(dir.path().join("a.gq"), decl).unwrap(); + fs::write(dir.path().join("b.gq"), decl).unwrap(); + fs::write( + dir.path().join("cluster.yaml"), + "version: 1\ngraphs:\n knowledge:\n schema: ./people.pg\n queries: [./a.gq, ./b.gq]\n", + ) + .unwrap(); + let out = validate_config_dir(dir.path()); + assert!(!out.ok); + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "duplicate_query_name"), + "{:?}", + out.diagnostics + ); + + fs::write(dir.path().join("broken.gq"), "query {{{ nope").unwrap(); + fs::write( + dir.path().join("cluster.yaml"), + "version: 1\ngraphs:\n knowledge:\n schema: ./people.pg\n queries: ./broken.gq\n", + ) + .unwrap(); + let out = validate_config_dir(dir.path()); + assert!(!out.ok); + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "query_parse_error"), + "{:?}", + out.diagnostics + ); + } + + #[test] + fn status_warns_on_pending_recovery_sidecar() { + let dir = fixture(); + write_applyable_state(dir.path()); + write_create_sidecar(dir.path(), "knowledge", "irrelevant", "01STATUS"); + + let out = status_config_dir(dir.path()); + assert!(out.ok, "{:?}", out.diagnostics); + assert!( + out.diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "cluster_recovery_pending" + && diagnostic.severity == DiagnosticSeverity::Warning) + ); + } + + #[tokio::test] + async fn plan_annotates_apply_dispositions() { + let dir = fixture(); + let out = plan_config_dir(dir.path()).await; + assert!(out.ok, "{:?}", out.diagnostics); + let by_resource: BTreeMap<&str, &PlanChange> = out + .changes + .iter() + .map(|change| (change.resource.as_str(), change)) + .collect(); + // Stage 4A: graph/schema creates are executable, and dependents ride + // the same run — plan previews exactly that. + assert_eq!( + by_resource["graph.knowledge"].disposition, + Some(ApplyDisposition::Applied) + ); + assert_eq!( + by_resource["schema.knowledge"].disposition, + Some(ApplyDisposition::Applied) + ); + assert_eq!( + by_resource["query.knowledge.find_person"].disposition, + Some(ApplyDisposition::Applied) + ); + assert_eq!( + by_resource["policy.base"].disposition, + Some(ApplyDisposition::Applied) + ); + }