//! The recovery sweep: RFC-004's roll-forward-only sidecar //! classification (moved verbatim from lib.rs in the modularization). use super::*; /// Recovery sweep (RFC-004 §D3): runs at the start of every state-mutating /// cluster command, under the state lock, before the command's own work. /// Roll-forward-only — the engine's own sidecars make each graph-level /// operation atomic within the graph, so the cluster never rolls a graph /// back; it converges the ledger to observable reality or refuses loudly. /// Mutations ride the calling command's CAS-checked state write; completed /// sidecars are deleted only after that write lands. pub(crate) async fn sweep_recovery_sidecars( backend: &ClusterStore, state: &mut ClusterState, diagnostics: &mut Vec, ) -> SweepOutcome { let mut outcome = SweepOutcome::default(); for (path, sidecar) in backend.list_recovery_sidecars(diagnostics).await { match sidecar.kind { RecoverySidecarKind::GraphCreate => { sweep_graph_create_sidecar( backend, path, sidecar, state, diagnostics, &mut outcome, ) .await; } RecoverySidecarKind::SchemaApply => { sweep_schema_apply_sidecar(path, sidecar, state, diagnostics, &mut outcome).await; } RecoverySidecarKind::GraphDelete => { sweep_graph_delete_sidecar( backend, path, sidecar, state, diagnostics, &mut outcome, ) .await; } } } outcome } pub(crate) async fn sweep_graph_create_sidecar( backend: &ClusterStore, path: String, sidecar: RecoverySidecar, state: &mut ClusterState, diagnostics: &mut Vec, outcome: &mut SweepOutcome, ) { let graph_address = graph_address(&sidecar.graph_id); let schema_addr = schema_address(&sidecar.graph_id); // Row 1: nothing moved — the init never landed. The sidecar is pure // intent; retire it (deferred to the command's post-CAS cleanup, like // every other completed sidecar — a failed CAS simply re-sweeps it) and // let the command's own plan re-propose the create. if !backend.graph_root_exists(&sidecar.graph_uri).await { outcome.completed_sidecars.push(path); return; } match Omnigraph::open_read_only(&sidecar.graph_uri).await { Ok(db) => { let live_digest = sha256_hex(db.schema_source().as_bytes()); let recorded = state .applied_revision .resources .get(&schema_addr) .map(|resource| resource.digest.clone()); if recorded.as_deref() == Some(live_digest.as_str()) { // Row 2: crash fell between the state CAS and sidecar delete. outcome.completed_sidecars.push(path); } else if live_digest == sidecar.desired_schema_digest { // Row 4: the create completed on the graph; roll the cluster // state forward to observable reality. state.applied_revision.resources.insert( schema_addr.clone(), StateResource { digest: live_digest.clone(), applies_to: None, embedding_provider: None, embedding_profile: None, }, ); let query_digests = state_query_digests_for_graph(state, &sidecar.graph_id); let embedding_provider = state_graph_embedding_provider(state, &sidecar.graph_id); let embedding_provider_digest = state_embedding_provider_digest(state, embedding_provider.as_deref()); let composite = graph_digest( &sidecar.graph_id, Some(&live_digest), Some(&query_digests), embedding_provider.as_deref(), embedding_provider_digest.as_ref(), ); state.applied_revision.resources.insert( graph_address.clone(), StateResource { digest: composite, applies_to: None, embedding_provider, embedding_profile: None, }, ); set_resource_status_applied(state, &graph_address); set_resource_status_applied(state, &schema_addr); state.recovery_records.insert( sidecar.operation_id.clone(), json!({ "kind": "graph_create", "graph_id": sidecar.graph_id, "outcome": "rolled_forward", "recovered_at": now_rfc3339(), "actor": sidecar.actor, }), ); diagnostics.push(Diagnostic::warning( "cluster_recovery_rolled_forward", graph_address.clone(), "an interrupted graph create had completed on the graph; cluster state was rolled forward to match", )); outcome.completed_sidecars.push(path); } else { // Row 6: the graph moved to something the sidecar did not // intend. Refuse to guess; require refresh + operator re-plan. set_resource_status( state, &graph_address, ResourceLifecycleStatus::Drifted, "actual_applied_state_pending", "graph state does not match the interrupted operation; run `cluster refresh` and re-plan", ); set_resource_status( state, &schema_addr, ResourceLifecycleStatus::Drifted, "actual_applied_state_pending", "graph state does not match the interrupted operation; run `cluster refresh` and re-plan", ); diagnostics.push(Diagnostic::warning( "cluster_recovery_pending", graph_address.clone(), "an interrupted graph create left unexpected graph state; graph-moving work is blocked until repaired", )); outcome.pending_graphs.insert(sidecar.graph_id.clone()); } } Err(err) => { // Row 5: partial root (the engine's documented init gap). Never // auto-delete — reconciler deletes are the same data-loss class // as human deletes; the operator removes the root explicitly. set_resource_status( state, &graph_address, ResourceLifecycleStatus::Error, "graph_create_incomplete", "graph root exists but cannot be opened; remove the graph root and re-run `cluster apply`", ); set_resource_status( state, &schema_addr, ResourceLifecycleStatus::Error, "graph_create_incomplete", "graph root exists but cannot be opened; remove the graph root and re-run `cluster apply`", ); diagnostics.push(Diagnostic::error( "graph_create_incomplete", graph_address.clone(), format!( "graph root '{}' exists but cannot be opened ({err}); remove the graph root and re-run `cluster apply`", sidecar.graph_uri ), )); outcome.pending_graphs.insert(sidecar.graph_id.clone()); } } } pub(crate) async fn sweep_schema_apply_sidecar( path: String, sidecar: RecoverySidecar, state: &mut ClusterState, diagnostics: &mut Vec, outcome: &mut SweepOutcome, ) { let graph_address = graph_address(&sidecar.graph_id); let schema_addr = schema_address(&sidecar.graph_id); // Digest-based classification: robust to unrelated manifest movement; // the sidecar's version pins stay forensic. let live_digest = match Omnigraph::open_read_only(&sidecar.graph_uri).await { Ok(db) => sha256_hex(db.schema_source().as_bytes()), Err(err) => { // Cannot verify the interrupted operation — refuse to guess. diagnostics.push(Diagnostic::warning( "cluster_recovery_pending", graph_address.clone(), format!( "an interrupted schema apply cannot be verified (graph '{}' did not open: {err}); graph-moving work is blocked until repaired", sidecar.graph_uri ), )); outcome.pending_graphs.insert(sidecar.graph_id.clone()); return; } }; let recorded = state .applied_revision .resources .get(&schema_addr) .map(|resource| resource.digest.clone()); if recorded.as_deref() == Some(live_digest.as_str()) { // Ledger consistent with the live graph (the apply never landed, or // landed and was recorded): the sidecar is stale intent — retire it. outcome.completed_sidecars.push(path); } else if live_digest == sidecar.desired_schema_digest { // RFC-004 §D3 row 3: the schema apply completed on the graph; roll // the cluster state forward to observable reality. state.applied_revision.resources.insert( schema_addr.clone(), StateResource { digest: live_digest.clone(), applies_to: None, embedding_provider: None, embedding_profile: None, }, ); let query_digests = state_query_digests_for_graph(state, &sidecar.graph_id); let embedding_provider = state_graph_embedding_provider(state, &sidecar.graph_id); let embedding_provider_digest = state_embedding_provider_digest(state, embedding_provider.as_deref()); let composite = graph_digest( &sidecar.graph_id, Some(&live_digest), Some(&query_digests), embedding_provider.as_deref(), embedding_provider_digest.as_ref(), ); state.applied_revision.resources.insert( graph_address.clone(), StateResource { digest: composite, applies_to: None, embedding_provider, embedding_profile: None, }, ); set_resource_status_applied(state, &graph_address); set_resource_status_applied(state, &schema_addr); state.recovery_records.insert( sidecar.operation_id.clone(), json!({ "kind": "schema_apply", "graph_id": sidecar.graph_id, "outcome": "rolled_forward", "recovered_at": now_rfc3339(), "actor": sidecar.actor, }), ); diagnostics.push(Diagnostic::warning( "cluster_recovery_rolled_forward", graph_address.clone(), "an interrupted schema apply had completed on the graph; cluster state was rolled forward to match", )); outcome.completed_sidecars.push(path); } else { // Row 6: live schema is neither the recorded nor the desired digest. set_resource_status( state, &graph_address, ResourceLifecycleStatus::Drifted, "actual_applied_state_pending", "graph state does not match the interrupted operation; run `cluster refresh` and re-plan", ); set_resource_status( state, &schema_addr, ResourceLifecycleStatus::Drifted, "actual_applied_state_pending", "graph state does not match the interrupted operation; run `cluster refresh` and re-plan", ); diagnostics.push(Diagnostic::warning( "cluster_recovery_pending", graph_address.clone(), "an interrupted schema apply left unexpected graph state; graph-moving work is blocked until repaired", )); outcome.pending_graphs.insert(sidecar.graph_id.clone()); } } pub(crate) async fn sweep_graph_delete_sidecar( backend: &ClusterStore, path: String, sidecar: RecoverySidecar, state: &mut ClusterState, diagnostics: &mut Vec, outcome: &mut SweepOutcome, ) { let graph_address = graph_address(&sidecar.graph_id); if backend.graph_root_exists(&sidecar.graph_uri).await { // Row 8: the delete never completed. Prefix removal is idempotent and // works on partial roots, so the repair is simply the re-proposed, // still-approved delete on a later run — retire the stale intent. diagnostics.push(Diagnostic::warning( "graph_delete_incomplete", graph_address, "a previous graph delete did not complete; it will be re-proposed by plan and can be retried under its approval", )); outcome.completed_sidecars.push(path); return; } if !state .applied_revision .resources .contains_key(&graph_address) { // Row 7: already tombstoned (or never recorded); crash fell between // the state CAS and sidecar delete. outcome.completed_sidecars.push(path); return; } // Row 7b: the root is gone, the ledger is stale — roll forward the // tombstone, consume the approval the sidecar carries, audit. tombstone_graph_subtree( state, &sidecar.graph_id, sidecar.approval_id.as_deref(), sidecar.actor.as_deref(), ); state.recovery_records.insert( sidecar.operation_id.clone(), json!({ "kind": "graph_delete", "graph_id": sidecar.graph_id, "outcome": "rolled_forward", "recovered_at": now_rfc3339(), "actor": sidecar.actor, }), ); if let Some(approval_id) = &sidecar.approval_id { record_approval_consumed(state, approval_id, &sidecar.operation_id); outcome.consumed_approvals.push(approval_id.clone()); } diagnostics.push(Diagnostic::warning( "cluster_recovery_rolled_forward", graph_address, "an interrupted graph delete had completed on disk; cluster state was rolled forward to match", )); outcome.completed_sidecars.push(path); } /// Remove a graph's subtree (graph, schema, queries) from the ledger and /// leave a tombstone observation. Idempotent. pub(crate) fn tombstone_graph_subtree( state: &mut ClusterState, graph_id: &str, approval_id: Option<&str>, actor: Option<&str>, ) { let graph_addr = graph_address(graph_id); let schema_addr = schema_address(graph_id); let query_prefix = format!("query.{graph_id}."); state.applied_revision.resources.remove(&graph_addr); state.applied_revision.resources.remove(&schema_addr); state .applied_revision .resources .retain(|address, _| !address.starts_with(&query_prefix)); state.resource_statuses.remove(&graph_addr); state.resource_statuses.remove(&schema_addr); state .resource_statuses .retain(|address, _| !address.starts_with(&query_prefix)); state.observations.insert( graph_addr, json!({ "kind": "tombstone", "deleted_at": now_rfc3339(), "approval_id": approval_id, "actor": actor, }), ); } /// Record approval consumption in the state ledger. The artifact FILE is /// rewritten with consumed_at only after the state write lands, so a failed /// CAS leaves the approval valid for the retry. pub(crate) fn record_approval_consumed( state: &mut ClusterState, approval_id: &str, operation_id: &str, ) { state.approval_records.insert( approval_id.to_string(), json!({ "consumed_at": now_rfc3339(), "consumed_by_operation": operation_id, }), ); } /// Mark approval artifact files consumed on disk (post-CAS). pub(crate) async fn mark_approvals_consumed(backend: &ClusterStore, approval_ids: &[String]) { if approval_ids.is_empty() { return; } let mut sink = Vec::new(); for (_, mut artifact) in backend.list_approval_artifacts(&mut sink).await { if approval_ids.contains(&artifact.approval_id) && artifact.consumed_at.is_none() { artifact.consumed_at = Some(now_rfc3339()); let _ = backend.write_approval_artifact(&artifact).await; } } } /// Read-only commands report pending sidecars without acting on them. pub(crate) async fn warn_pending_recovery_sidecars( backend: &ClusterStore, diagnostics: &mut Vec, ) { for location in backend.list_recovery_sidecar_locations(diagnostics).await { diagnostics.push(Diagnostic::warning( "cluster_recovery_pending", location, "a recovery sidecar from an interrupted apply is pending; the next state-mutating command will classify it", )); } }