MR-771: demote Run to direct-publish via expected_table_versions CAS

mutate_as and load now write directly to target tables and call the
publisher once at the end with per-table expected versions; the Run
state machine, _graph_runs.lance writers, __run__ staging branches,
and server /runs/* endpoints are removed. Multi-statement mutations
remain atomic at the manifest level via an in-memory MutationStaging
accumulator that gives read-your-writes within a query and a single
publish at the end. Concurrent-writer conflicts surface as
ExpectedVersionMismatch (HTTP 409 manifest_conflict) instead of the
old DivergentUpdate merge shape. Documents one known limitation in
docs/runs.md: a multi-statement mid-query failure where op-N writes
a Lance fragment and op-N+1 fails leaves Lance HEAD ahead of the
manifest until a follow-up introduces per-table Lance branches.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Ragnor Comerford 2026-04-30 08:52:50 +02:00
parent 4e5374a85e
commit 35be20cb05
No known key found for this signature in database
28 changed files with 1188 additions and 3216 deletions

View file

@ -5,7 +5,7 @@ use std::path::PathBuf;
use clap::{Arg, ArgAction, Args, CommandFactory, FromArgMatches, Parser, Subcommand, ValueEnum};
use color_eyre::eyre::{Result, bail};
use omnigraph::db::{Omnigraph, ReadTarget, RunId, SnapshotId};
use omnigraph::db::{Omnigraph, ReadTarget, SnapshotId};
use omnigraph::loader::LoadMode;
use omnigraph_compiler::query::parser::parse_query;
use omnigraph_compiler::schema::parser::parse_schema;
@ -18,9 +18,8 @@ use omnigraph_server::api::{
BranchCreateOutput, BranchCreateRequest, BranchDeleteOutput, BranchListOutput,
BranchMergeOutput, BranchMergeRequest, ChangeOutput, ChangeRequest, CommitListOutput,
CommitOutput, ErrorOutput, ExportRequest, IngestOutput, IngestRequest, ReadOutput, ReadRequest,
RunListOutput, RunOutput, SchemaApplyOutput, SchemaApplyRequest, SchemaOutput, SnapshotOutput,
SnapshotTableOutput, commit_output, ingest_output, read_output, run_output,
schema_apply_output, snapshot_payload,
SchemaApplyOutput, SchemaApplyRequest, SchemaOutput, SnapshotOutput, SnapshotTableOutput,
commit_output, ingest_output, read_output, schema_apply_output, snapshot_payload,
};
use omnigraph_server::{
AliasCommand, OmnigraphConfig, PolicyAction, PolicyDecision, PolicyEngine, PolicyRequest,
@ -143,11 +142,6 @@ enum Command {
#[arg(long = "table")]
table_keys: Vec<String>,
},
/// Run operations
Run {
#[command(subcommand)]
command: RunCommand,
},
/// Commit history operations
Commit {
#[command(subcommand)]
@ -370,60 +364,6 @@ enum QueryCommand {
},
}
#[derive(Debug, Subcommand)]
enum RunCommand {
/// List transactional runs
List {
/// Repo URI
uri: Option<String>,
#[arg(long)]
target: Option<String>,
#[arg(long)]
config: Option<PathBuf>,
#[arg(long)]
json: bool,
},
/// Show a transactional run
Show {
/// Repo URI
#[arg(long)]
uri: Option<String>,
#[arg(long)]
target: Option<String>,
#[arg(long)]
config: Option<PathBuf>,
run_id: String,
#[arg(long)]
json: bool,
},
/// Publish a transactional run
Publish {
/// Repo URI
#[arg(long)]
uri: Option<String>,
#[arg(long)]
target: Option<String>,
#[arg(long)]
config: Option<PathBuf>,
run_id: String,
#[arg(long)]
json: bool,
},
/// Abort a transactional run
Abort {
/// Repo URI
#[arg(long)]
uri: Option<String>,
#[arg(long)]
target: Option<String>,
#[arg(long)]
config: Option<PathBuf>,
run_id: String,
#[arg(long)]
json: bool,
},
}
#[derive(Debug, Subcommand)]
enum CommitCommand {
/// List graph commits
@ -1214,42 +1154,6 @@ fn print_change_human(output: &ChangeOutput) {
}
}
fn print_run_list_human(runs: &[RunOutput]) {
for run in runs {
println!(
"{} {} target={} branch={}{}",
run.run_id,
run.status,
run.target_branch,
run.run_branch,
run.actor_id
.as_deref()
.map(|actor| format!(" actor={}", actor))
.unwrap_or_default()
);
}
}
fn print_run_human(run: &RunOutput) {
println!("run_id: {}", run.run_id);
println!("status: {}", run.status);
println!("target_branch: {}", run.target_branch);
println!("run_branch: {}", run.run_branch);
println!("base_snapshot_id: {}", run.base_snapshot_id);
println!("base_manifest_version: {}", run.base_manifest_version);
if let Some(actor_id) = &run.actor_id {
println!("actor_id: {}", actor_id);
}
if let Some(operation_hash) = &run.operation_hash {
println!("operation_hash: {}", operation_hash);
}
if let Some(snapshot_id) = &run.published_snapshot_id {
println!("published_snapshot_id: {}", snapshot_id);
}
println!("created_at: {}", run.created_at);
println!("updated_at: {}", run.updated_at);
}
fn print_commit_list_human(commits: &[CommitOutput]) {
for commit in commits {
let branch = commit.manifest_branch.as_deref().unwrap_or("main");
@ -2190,133 +2094,6 @@ async fn main() -> Result<()> {
.await?;
}
}
Command::Run { command } => match command {
RunCommand::List {
uri,
target,
config,
json,
} => {
let config = load_cli_config(config.as_ref())?;
let bearer_token =
resolve_remote_bearer_token(&config, uri.as_deref(), target.as_deref())?;
let uri = resolve_uri(&config, uri, target.as_deref())?;
let runs = if is_remote_uri(&uri) {
remote_json::<RunListOutput>(
&http_client,
Method::GET,
remote_url(&uri, "/runs"),
None,
bearer_token.as_deref(),
)
.await?
.runs
} else {
let db = Omnigraph::open(&uri).await?;
db.list_runs()
.await?
.iter()
.map(run_output)
.collect::<Vec<_>>()
};
if json {
print_json(&RunListOutput { runs })?;
} else {
print_run_list_human(&runs);
}
}
RunCommand::Show {
uri,
target,
config,
run_id,
json,
} => {
let config = load_cli_config(config.as_ref())?;
let bearer_token =
resolve_remote_bearer_token(&config, uri.as_deref(), target.as_deref())?;
let uri = resolve_uri(&config, uri, target.as_deref())?;
let run = if is_remote_uri(&uri) {
remote_json::<RunOutput>(
&http_client,
Method::GET,
remote_url(&uri, &format!("/runs/{}", run_id)),
None,
bearer_token.as_deref(),
)
.await?
} else {
let db = Omnigraph::open(&uri).await?;
run_output(&db.get_run(&RunId::new(run_id)).await?)
};
if json {
print_json(&run)?;
} else {
print_run_human(&run);
}
}
RunCommand::Publish {
uri,
target,
config,
run_id,
json,
} => {
let config = load_cli_config(config.as_ref())?;
let bearer_token =
resolve_remote_bearer_token(&config, uri.as_deref(), target.as_deref())?;
let uri = resolve_uri(&config, uri, target.as_deref())?;
let run = if is_remote_uri(&uri) {
remote_json::<RunOutput>(
&http_client,
Method::POST,
remote_url(&uri, &format!("/runs/{}/publish", run_id)),
Some(serde_json::json!({})),
bearer_token.as_deref(),
)
.await?
} else {
let mut db = Omnigraph::open(&uri).await?;
db.publish_run(&RunId::new(run_id.clone())).await?;
run_output(&db.get_run(&RunId::new(run_id)).await?)
};
if json {
print_json(&run)?;
} else {
print_run_human(&run);
}
}
RunCommand::Abort {
uri,
target,
config,
run_id,
json,
} => {
let config = load_cli_config(config.as_ref())?;
let bearer_token =
resolve_remote_bearer_token(&config, uri.as_deref(), target.as_deref())?;
let uri = resolve_uri(&config, uri, target.as_deref())?;
let run = if is_remote_uri(&uri) {
remote_json::<RunOutput>(
&http_client,
Method::POST,
remote_url(&uri, &format!("/runs/{}/abort", run_id)),
Some(serde_json::json!({})),
bearer_token.as_deref(),
)
.await?
} else {
let mut db = Omnigraph::open(&uri).await?;
run_output(&db.abort_run(&RunId::new(run_id)).await?)
};
if json {
print_json(&run)?;
} else {
print_run_human(&run);
}
}
},
Command::Read {
uri,
legacy_uri,

View file

@ -29,7 +29,7 @@ rules:
- id: admins-promote
allow:
actors: { group: admins }
actions: [branch_merge, run_publish]
actions: [branch_merge]
target_branch_scope: protected
"#;
@ -1905,119 +1905,7 @@ fn cli_fails_for_invalid_merge_requests() {
);
}
#[test]
fn run_list_and_show_report_published_runs() {
let temp = tempdir().unwrap();
let repo = repo_path(temp.path());
init_repo(&repo);
load_fixture(&repo);
let list_output = output_success(cli().arg("run").arg("list").arg(&repo).arg("--json"));
let list_payload: Value = serde_json::from_slice(&list_output.stdout).unwrap();
let runs = list_payload["runs"].as_array().unwrap();
assert_eq!(runs.len(), 1);
assert_eq!(runs[0]["status"], "published");
let run_id = runs[0]["run_id"].as_str().unwrap();
let show_output = output_success(
cli()
.arg("run")
.arg("show")
.arg("--uri")
.arg(&repo)
.arg(run_id)
.arg("--json"),
);
let show_payload: Value = serde_json::from_slice(&show_output.stdout).unwrap();
assert_eq!(show_payload["run_id"], run_id);
assert_eq!(show_payload["status"], "published");
assert_eq!(show_payload["target_branch"], "main");
}
#[test]
fn run_list_can_resolve_uri_from_config() {
let temp = tempdir().unwrap();
let repo = repo_path(temp.path());
let config = temp.path().join("omnigraph.yaml");
init_repo(&repo);
load_fixture(&repo);
write_config(&config, &local_yaml_config(&repo));
let output = output_success(
cli()
.arg("run")
.arg("list")
.arg("--config")
.arg(&config)
.arg("--json"),
);
let payload: Value = serde_json::from_slice(&output.stdout).unwrap();
assert_eq!(payload["runs"].as_array().unwrap().len(), 1);
}
#[test]
fn run_publish_promotes_manual_running_run() {
let runtime = tokio::runtime::Runtime::new().unwrap();
let temp = tempdir().unwrap();
let repo = repo_path(temp.path());
init_repo(&repo);
load_fixture(&repo);
let run_id = runtime.block_on(begin_manual_run(&repo, "main"));
let publish_output = output_success(
cli()
.arg("run")
.arg("publish")
.arg("--uri")
.arg(&repo)
.arg(&run_id)
.arg("--json"),
);
let payload: Value = serde_json::from_slice(&publish_output.stdout).unwrap();
assert_eq!(payload["run_id"], run_id);
assert_eq!(payload["status"], "published");
assert!(payload["published_snapshot_id"].is_string());
let runtime = tokio::runtime::Runtime::new().unwrap();
runtime.block_on(async {
let db = Omnigraph::open(repo.to_str().unwrap()).await.unwrap();
let result = db
.query(
ReadTarget::branch("main"),
include_str!("../../omnigraph/tests/fixtures/test.gq"),
"get_person",
&omnigraph_compiler::ir::ParamMap::from([(
"name".to_string(),
omnigraph_compiler::query::ast::Literal::String("Eve".to_string()),
)]),
)
.await
.unwrap();
assert_eq!(result.num_rows(), 1);
});
}
#[test]
fn run_abort_marks_manual_running_run_aborted() {
let runtime = tokio::runtime::Runtime::new().unwrap();
let temp = tempdir().unwrap();
let repo = repo_path(temp.path());
init_repo(&repo);
load_fixture(&repo);
let run_id = runtime.block_on(begin_manual_run(&repo, "main"));
let abort_output = output_success(
cli()
.arg("run")
.arg("abort")
.arg("--uri")
.arg(&repo)
.arg(&run_id)
.arg("--json"),
);
let payload: Value = serde_json::from_slice(&abort_output.stdout).unwrap();
assert_eq!(payload["run_id"], run_id);
assert_eq!(payload["status"], "aborted");
}
// MR-771: `omnigraph run list/show/publish/abort` subcommands removed
// alongside the run state machine. Direct-to-target writes leave nothing
// for these CLIs to manage. Audit history is now visible via
// `omnigraph commit list` reading the commit graph.

View file

@ -8,8 +8,6 @@ use std::thread::sleep;
use std::time::Duration;
use assert_cmd::Command;
use omnigraph::db::Omnigraph;
use omnigraph::loader::LoadMode;
use reqwest::blocking::Client;
use serde_json::Value;
use tempfile::{TempDir, tempdir};
@ -223,21 +221,6 @@ pub fn spawn_server_with_config_env(config: &Path, envs: &[(&str, &str)]) -> Tes
spawn_server_process(command)
}
pub async fn begin_manual_run(repo: &Path, target_branch: &str) -> String {
let mut db = Omnigraph::open(repo.to_str().unwrap()).await.unwrap();
let run = db
.begin_run(target_branch, Some("cli-test-run"))
.await
.unwrap();
db.load(
&run.run_branch,
r#"{"type":"Person","data":{"name":"Eve","age":29}}"#,
LoadMode::Append,
)
.await
.unwrap();
run.run_id.as_str().to_string()
}
pub struct SystemRepo {
_temp: TempDir,

View file

@ -2,12 +2,7 @@ mod support;
use std::env;
use std::fs;
use std::process::Stdio;
use std::thread::sleep;
use std::time::Duration;
use omnigraph::db::Omnigraph;
use omnigraph::loader::LoadMode;
use reqwest::blocking::Client;
use serde_json::Value;
@ -33,7 +28,7 @@ rules:
- id: admins-promote
allow:
actors: { group: admins }
actions: [branch_merge, run_publish]
actions: [branch_merge]
target_branch_scope: protected
"#;
@ -117,42 +112,6 @@ fn snapshot_table_row_count_at(repo: &std::path::Path, table_key: &str) -> u64 {
.unwrap()
}
fn wait_for_running_run(repo: &SystemRepo) -> String {
let runtime = tokio::runtime::Runtime::new().unwrap();
for _ in 0..200 {
let running = runtime.block_on(async {
let db = Omnigraph::open(repo.path().to_str().unwrap())
.await
.unwrap();
db.list_runs()
.await
.unwrap()
.into_iter()
.find(|run| run.target_branch == "main" && run.status.as_str() == "running")
.map(|run| run.run_id.to_string())
});
if let Some(run_id) = running {
return run_id;
}
sleep(Duration::from_millis(50));
}
panic!("timed out waiting for running run");
}
fn bulk_people_jsonl(count: usize) -> String {
let mut rows = String::new();
for index in 0..count {
rows.push_str(&format!(
r#"{{"type":"Person","data":{{"name":"Bulk{:05}","age":{}}}}}"#,
index,
20 + (index % 50)
));
rows.push('\n');
}
rows
}
fn gemini_base_url() -> String {
env::var("OMNIGRAPH_GEMINI_BASE_URL")
.ok()
@ -348,15 +307,17 @@ fn local_cli_end_to_end_branch_change_merge_flow() {
assert_eq!(main_read["row_count"], 1);
assert_eq!(main_read["rows"][0]["p.name"], "Zoe");
let runs_payload = parse_stdout_json(&output_success(
cli().arg("run").arg("list").arg(repo.path()).arg("--json"),
// MR-771: `omnigraph run list` removed. Audit visible via commit list.
let commits_payload = parse_stdout_json(&output_success(
cli()
.arg("commit")
.arg("list")
.arg(repo.path())
.arg("--branch")
.arg("main")
.arg("--json"),
));
let runs = runs_payload["runs"].as_array().unwrap();
assert!(runs.len() >= 2);
assert!(
runs.iter()
.any(|run| run["target_branch"] == "feature" && run["status"] == "published")
);
assert!(commits_payload["commits"].as_array().unwrap().len() >= 2);
}
#[test]
@ -636,17 +597,8 @@ fn local_cli_failed_load_keeps_target_state_unchanged() {
snapshot_table_row_count(&repo, "edge:Knows"),
knows_rows_before
);
let runs_payload = parse_stdout_json(&output_success(
cli().arg("run").arg("list").arg(repo.path()).arg("--json"),
));
assert!(
runs_payload["runs"]
.as_array()
.unwrap()
.iter()
.any(|run| run["target_branch"] == "main" && run["status"] == "failed")
);
// MR-771: failed loads no longer leave a RunRecord. The atomicity
// guarantee is verified above (target tables are unchanged).
}
#[test]
@ -679,17 +631,9 @@ fn local_cli_failed_change_keeps_target_state_unchanged() {
.arg("--json"),
));
assert_eq!(friends_payload["row_count"], 2);
let runs_payload = parse_stdout_json(&output_success(
cli().arg("run").arg("list").arg(repo.path()).arg("--json"),
));
assert!(
runs_payload["runs"]
.as_array()
.unwrap()
.iter()
.any(|run| run["target_branch"] == "main" && run["status"] == "failed")
);
// MR-771: failed mutations no longer leave a RunRecord. The atomicity
// guarantee is verified above (the friends_of read above shows main
// unchanged).
}
#[test]
@ -997,102 +941,12 @@ query vector_search($q: String) {
assert_eq!(result["rows"][0]["d.slug"], "alpha-doc");
}
#[test]
fn local_cli_transactional_load_drift_fails_without_partial_publish() {
let repo = SystemRepo::loaded();
let large_data = repo.write_jsonl("system-large-load.jsonl", &bulk_people_jsonl(250_000));
let person_rows_before = snapshot_table_row_count(&repo, "node:Person");
let mut load = cli_process();
load.arg("load")
.arg("--data")
.arg(&large_data)
.arg("--mode")
.arg("merge")
.arg(repo.path())
.stdout(Stdio::piped())
.stderr(Stdio::piped());
let child = load.spawn().unwrap();
let run_id = wait_for_running_run(&repo);
tokio::runtime::Runtime::new().unwrap().block_on(async {
let mut db = Omnigraph::open(repo.path().to_str().unwrap())
.await
.unwrap();
let interloper = db
.begin_run("main", Some("system-test-interloper"))
.await
.unwrap();
db.load(
interloper.run_branch.as_str(),
r#"{"type":"Person","data":{"name":"Interloper","age":41}}"#,
LoadMode::Append,
)
.await
.unwrap();
db.publish_run(&interloper.run_id).await.unwrap();
});
let output = child.wait_with_output().unwrap();
assert!(
!output.status.success(),
"load unexpectedly succeeded\nstdout:\n{}\nstderr:\n{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
let stderr = String::from_utf8(output.stderr).unwrap();
assert!(
stderr.contains("advanced during transactional load")
|| stderr.contains("version drift")
|| stderr.contains("retry"),
"unexpected load failure: {stderr}"
);
let run_payload = parse_stdout_json(&output_success(
cli()
.arg("run")
.arg("show")
.arg("--uri")
.arg(repo.path())
.arg(&run_id)
.arg("--json"),
));
assert_eq!(run_payload["status"], "failed");
assert_eq!(
snapshot_table_row_count(&repo, "node:Person"),
person_rows_before + 1
);
let interloper = parse_stdout_json(&output_success(
cli()
.arg("read")
.arg(repo.path())
.arg("--query")
.arg(fixture("test.gq"))
.arg("--name")
.arg("get_person")
.arg("--params")
.arg(r#"{"name":"Interloper"}"#)
.arg("--json"),
));
assert_eq!(interloper["row_count"], 1);
let bulk_row = parse_stdout_json(&output_success(
cli()
.arg("read")
.arg(repo.path())
.arg("--query")
.arg(fixture("test.gq"))
.arg("--name")
.arg("get_person")
.arg("--params")
.arg(r#"{"name":"Bulk00000"}"#)
.arg("--json"),
));
assert_eq!(bulk_row["row_count"], 0);
}
// MR-771: the publisher CAS conflict shape is verified end-to-end at the
// engine level in `crates/omnigraph/tests/runs.rs::concurrent_writers_one_succeeds_one_gets_expected_version_mismatch`
// and at the HTTP boundary in
// `crates/omnigraph-server/tests/server.rs::change_conflict_returns_manifest_conflict_409`.
// The pre-MR-771 CLI-level race was timing-dependent; with direct-publish
// the surface is the same engine path the unit test already covers.
#[test]
fn local_cli_policy_tooling_is_end_to_end_while_local_writes_stay_unenforced() {

View file

@ -33,7 +33,7 @@ rules:
- id: admins-promote
allow:
actors: { group: admins }
actions: [branch_merge, run_publish]
actions: [branch_merge]
target_branch_scope: protected
"#;
@ -192,30 +192,9 @@ query insert_person($name: String, $age: I32) {
assert_eq!(local_verify["row_count"], 1);
assert_eq!(local_verify["rows"][0]["p.name"], "Mina");
let manual_run = tokio::runtime::Runtime::new()
.unwrap()
.block_on(begin_manual_run(repo.path(), "main"));
let publish_payload = parse_stdout_json(&output_success(
cli()
.arg("run")
.arg("publish")
.arg("--config")
.arg(&config)
.arg(&manual_run)
.arg("--json"),
));
assert_eq!(publish_payload["run_id"], manual_run);
assert_eq!(publish_payload["status"], "published");
let runs_payload = parse_stdout_json(&output_success(
cli()
.arg("run")
.arg("list")
.arg("--config")
.arg(&config)
.arg("--json"),
));
assert!(runs_payload["runs"].as_array().unwrap().len() >= 2);
// MR-771: `run publish` / `run list` removed. Direct-to-target writes
// already landed via the change call above; the commit graph is now
// the audit surface (verified separately by `commit list`).
}
#[test]